static void write_file( char *target, int rank, MPI_Info *info ) { MPI_File wfh; MPI_Status mpi_stat; int mpi_ret; int i; char buffer[OBJ_SIZE]; if ( debug ) printf( "%d writing file %s\n", rank, target ); if( (mpi_ret = MPI_File_open(MPI_COMM_WORLD, target, MPI_MODE_WRONLY | MPI_MODE_CREATE, *info, &wfh ) ) != MPI_SUCCESS ) { fatal_error( mpi_ret, NULL, "open for write" ); } for( i = 0; i < NUM_OBJS; i++ ) { MPI_Offset offset = get_offset( rank, NUM_OBJS, OBJ_SIZE, i ); fill_buffer( buffer, OBJ_SIZE, rank, offset ); if ( debug ) printf( "%s", buffer ); if ( (mpi_ret = MPI_File_write_at_all( wfh, offset, buffer, OBJ_SIZE, MPI_CHAR, &mpi_stat ) ) != MPI_SUCCESS ) { fatal_error( mpi_ret, &mpi_stat, "write" ); } } if ( debug ) print_hints( rank, &wfh ); if( (mpi_ret = MPI_File_close( &wfh ) ) != MPI_SUCCESS ) { fatal_error( mpi_ret, NULL, "close for write" ); } if ( debug ) printf( "%d wrote file %s\n", rank, target ); }
int main(int argc, char ** argv) { MPI_Info info = MPI_INFO_NULL; MPI_File fh; MPI_Offset off=0; MPI_Status status; int errcode; int i, rank, errs=0, toterrs, buffer[BUFSIZE], buf2[BUFSIZE]; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Info_create(&info); MPI_Info_set(info, "romio_cb_write", "enable"); MPI_Info_set(info, "cb_nodes", "1"); for (i=0; i<BUFSIZE; i++) { buffer[i] = 10000+rank; } off = rank*sizeof(buffer); errcode = MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_WRONLY|MPI_MODE_CREATE, info, &fh); if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open"); errcode = MPI_File_write_at_all(fh, off, buffer, BUFSIZE, MPI_INT, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_write_at_all"); errcode = MPI_File_close(&fh); if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close"); errcode = MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, info, &fh); if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open"); errcode = MPI_File_read_at_all(fh, off, buf2, BUFSIZE, MPI_INT, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read_at_all"); errcode = MPI_File_close(&fh); if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close"); for (i=0; i<BUFSIZE; i++) { if (buf2[i] != 10000+rank) errs++; } MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); if (rank == 0) { if( toterrs > 0) { fprintf( stderr, "Found %d errors\n", toterrs ); } else { fprintf( stdout, " No Errors\n" ); } } MPI_Info_free(&info); MPI_Finalize(); return 0; }
void FORTRAN_API mpi_file_write_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf, int *count,MPI_Datatype *datatype, MPI_Status *status, int *ierr ) { MPI_File fh_c; fh_c = MPI_File_f2c(*fh); *ierr = MPI_File_write_at_all(fh_c,*offset,buf,*count,*datatype,status); }
static int verify_type(char *filename, MPI_Datatype type, int64_t expected_extent, int do_coll) { int rank, canary; MPI_Count tsize; int compare=-1; int errs=0, toterrs=0; MPI_Status status; MPI_File fh; MPI_Comm_rank(MPI_COMM_WORLD, &rank); CHECK( MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); CHECK( MPI_File_set_view(fh, rank*sizeof(int), MPI_BYTE, type, "native", MPI_INFO_NULL)); MPI_Type_size_x(type, &tsize); canary=rank+1000000; /* skip over first instance of type */ if (do_coll) { CHECK( MPI_File_write_at_all(fh, tsize, &canary, 1, MPI_INT, &status)); } else { CHECK( MPI_File_write_at(fh, tsize, &canary, 1, MPI_INT, &status)); } CHECK( MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL)); if (do_coll) { CHECK( MPI_File_read_at_all(fh, expected_extent/sizeof(int)+rank, &compare, 1, MPI_INT, &status)); } else { CHECK( MPI_File_read_at(fh, expected_extent/sizeof(int)+rank, &compare, 1, MPI_INT, &status)); } if (compare != canary) errs=1; MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); MPI_File_close(&fh); if (toterrs) { printf("%d: got %d expected %d\n", rank, compare, canary); /* keep file if there's an error */ } else { if (rank == 0) MPI_File_delete(filename, MPI_INFO_NULL); } return (toterrs); }
void mpi_file_write_at_all_(MPI_Fint *fh,MPI_Offset *offset,void *buf, int *count,MPI_Fint *datatype, MPI_Status *status, int *ierr ) { MPI_File fh_c; MPI_Datatype datatype_c; fh_c = MPI_File_f2c(*fh); datatype_c = MPI_Type_f2c(*datatype); *ierr = MPI_File_write_at_all(fh_c,*offset,buf,*count,datatype_c,status); }
static MPI_Offset writeToMPI(const std::vector<T>& data, MPI_File f, MPI_Offset base, MPI_Comm comm) { MPI_Offset offset = 0, nbytes = data.size()*sizeof(T); MPI_Check( MPI_Exscan(&nbytes, &offset, 1, MPI_OFFSET, MPI_SUM, comm)); MPI_Check( MPI_File_write_at_all(f, base + offset, data.data(), nbytes, MPI_CHAR, MPI_STATUS_IGNORE)); MPI_Offset ntotal = 0; MPI_Check( MPI_Allreduce(&nbytes, &ntotal, 1, MPI_OFFSET, MPI_SUM, comm) ); return ntotal; }
/* * mpi_io_shared * * creates a single-shared-file * writes with independent-io * reads with independent-io * writes with collective-io * reads with collective-io */ int mpi_io_shared (char *path, int size, int rank) { MPI_File fh; char filepath[512]; MPI_Offset offset; MPI_Status status; void *buf; int bufcount = BYTES_PER_RANK; int rc; buf = malloc(bufcount); if (!buf) { return 0; } memset(buf, 0xa, bufcount); sprintf(filepath, "%s/%s", path, "cp-bench-mpio-shared"); rc = MPI_File_open(MPI_COMM_WORLD, filepath, (MPI_MODE_CREATE|MPI_MODE_RDWR|MPI_MODE_DELETE_ON_CLOSE), MPI_INFO_NULL, &fh); MPI_CHECK(rc,"MPI_File_open"); /* Indep Write */ offset = rank * bufcount; rc = MPI_File_write_at(fh,offset,buf,bufcount,MPI_BYTE,&status); MPI_CHECK(rc,"MPI_File_write_at"); MPI_Barrier(MPI_COMM_WORLD); /* Indep Read */ offset = ((rank+1)%size) * bufcount; rc = MPI_File_read_at(fh,offset,buf,bufcount,MPI_BYTE,&status); MPI_CHECK(rc,"MPI_File_read_at"); /* Collective Write */ offset = rank * bufcount; rc = MPI_File_write_at_all(fh, offset, buf, bufcount, MPI_BYTE, &status); MPI_CHECK(rc,"MPI_File_write_at_all"); /* Collective Read */ offset = ((rank+1)%size) * bufcount; rc = MPI_File_read_at_all(fh, offset, buf, bufcount, MPI_BYTE, &status); MPI_CHECK(rc,"MPI_File_read_at_all"); rc = MPI_File_close(&fh); MPI_CHECK(rc,"MPI_File_close"); free(buf); return 1; }
FORT_DLL_SPEC void FORT_CALL mpi_file_write_at_all_ ( MPI_Fint *v1, MPI_Offset *v2, void*v3, MPI_Fint *v4, MPI_Fint *v5, MPI_Fint *v6, MPI_Fint *ierr ){ #ifdef MPI_MODE_RDONLY #ifndef HAVE_MPI_F_INIT_WORKS_WITH_C if (MPIR_F_NeedInit){ mpirinitf_(); MPIR_F_NeedInit = 0; } #endif if (v6 == MPI_F_STATUS_IGNORE) { v6 = (MPI_Fint*)MPI_STATUS_IGNORE; } *ierr = MPI_File_write_at_all( MPI_File_f2c(*v1), (MPI_Offset)*v2, v3, (int)*v4, (MPI_Datatype)(*v5), (MPI_Status *)v6 ); #else *ierr = MPI_ERR_INTERN; #endif }
/* determine whether to dump buffer to file * should only be called at GVT! */ void st_buffer_write(int end_of_sim, int type) { MPI_Offset offset = prev_offsets[type]; MPI_File *fh = &buffer_fh[type]; int write_to_file = 0; int my_write_size = 0; int i; int write_sizes[tw_nnodes()]; tw_clock start_cycle_time = tw_clock_read(); my_write_size = g_st_buffer[type]->count; MPI_Allgather(&my_write_size, 1, MPI_INT, &write_sizes[0], 1, MPI_INT, MPI_COMM_ROSS); if (end_of_sim) write_to_file = 1; else { for (i = 0; i < tw_nnodes(); i++) { if ((double) write_sizes[i] / g_st_buffer_size >= g_st_buffer_free_percent / 100.0) write_to_file = 1; } } if (write_to_file) { for (i = 0; i < tw_nnodes(); i++) { if (i < g_tw_mynode) offset += write_sizes[i]; prev_offsets[type] += write_sizes[i]; } //printf("rank %ld writing %d bytes at offset %lld (prev_offsets[ANALYSIS_LP] = %lld)\n", g_tw_mynode, my_write_size, offset, prev_offsets[type]); // dump buffer to file MPI_Status status; g_tw_pe[0]->stats.s_stat_comp += tw_clock_read() - start_cycle_time; start_cycle_time = tw_clock_read(); MPI_File_write_at_all(*fh, offset, st_buffer_read_ptr(g_st_buffer[type]), my_write_size, MPI_BYTE, &status); g_tw_pe[0]->stats.s_stat_write += tw_clock_read() - start_cycle_time; // reset the buffer g_st_buffer[type]->write_pos = 0; g_st_buffer[type]->read_pos = 0; g_st_buffer[type]->count = 0; buffer_overflow_warned = 0; } else g_tw_pe[0]->stats.s_stat_comp += tw_clock_read() - start_cycle_time; }
void final_checkpoint(grid_parms grid, checkpoint_handle *check,double t1, double t2){ MPI_Status status; MPI_Offset disp; double diff = t2-t1; disp = grid.rank*sizeof(double); CHECK(MPI_File_write_at_all(check->elapsed_time, disp, &diff, 1, MPI_DOUBLE, &status)); MPI_Barrier(grid.universe); MPI_File_close(&check->Time); MPI_File_close(&check->logptr); MPI_File_close(&check->ci); MPI_File_close(&check->cj); MPI_File_close(&check->si); MPI_File_close(&check->sj); MPI_File_close(&check->vi); MPI_File_close(&check->vj); MPI_File_close(&check->wi); MPI_File_close(&check->Ii); MPI_File_close(&check->Ij); MPI_File_close(&check->cpCi); MPI_File_close(&check->cpCj); MPI_File_close(&check->cpVi); MPI_File_close(&check->cpVj); MPI_File_close(&check->cpIi); MPI_File_close(&check->cpIj); MPI_File_close(&check->elapsed_time); MPI_File_close(&check->jplc); MPI_File_close(&check->time_profiling); MPI_File_close(&check->async_calls); MPI_File_close(&check->async_wait); MPI_File_close(&check->barrier_before_comm); MPI_File_close(&check->map_function); MPI_File_close(&check->single_cell_fluxes); MPI_File_close(&check->coupling_fluxes); MPI_File_close(&check->solver); MPI_File_close(&check->writer_func); MPI_File_close(&check->derivative_calls); MPI_File_close(&check->itter_count); MPI_File_close(&check->coords); }
JNIEXPORT void JNICALL Java_mpi_File_writeAtAll( JNIEnv *env, jobject jthis, jlong fh, jlong fileOffset, jobject buf, jboolean db, jint off, jint count, jlong jType, jint bType, jlongArray stat) { MPI_Datatype type = (MPI_Datatype)jType; void *ptr; ompi_java_buffer_t *item; ompi_java_getReadPtr(&ptr, &item, env, buf, db, off, count, type, bType); MPI_Status status; int rc = MPI_File_write_at_all((MPI_File)fh, (MPI_Offset)fileOffset, ptr, count, (MPI_Datatype)type, &status); ompi_java_exceptionCheck(env, rc); ompi_java_releaseReadPtr(ptr, item, buf, db); ompi_java_status_set(env, stat, &status); }
int lemonWriteLatticeParallelMapped(LemonWriter *writer, void *data, MPI_Offset siteSize, int const *latticeDims, int const *mapping) { int written; int error; MPI_Status status; LemonSetup setup; error = lemonClearWriterState(writer); if (error != LEMON_SUCCESS) return error; lemonSetupIOTypes(&setup, writer->cartesian, siteSize, latticeDims, mapping); /* Install the data organization we worked out above on the file as a view */ MPI_Barrier(writer->cartesian); MPI_File_set_view(*writer->fp, writer->off + writer->pos, setup.etype, setup.ftype, "native", MPI_INFO_NULL); /* Blast away! */ MPI_File_write_at_all(*writer->fp, writer->pos, data, setup.localVol, setup.etype, &status); MPI_File_sync(*writer->fp); MPI_Barrier(writer->cartesian); writer->pos += setup.totalVol * siteSize; /* We should reset the shared file pointer, in an MPI_BYTE based view... */ MPI_Barrier(writer->cartesian); MPI_File_set_view(*writer->fp, 0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL); /* Free up the resources we claimed for this operation. */ lemonFreeIOTypes(&setup); MPI_Get_count(&status, MPI_BYTE, &written); if (written != siteSize * setup.localVol) { fprintf(stderr, "[LEMON] Node %d reports in lemonWriteLatticeParallel:\n" " Could not write the required amount of data.\n", writer->my_rank); return LEMON_ERR_WRITE; } return LEMON_SUCCESS; }
void mpi_file_write_at_all_f(MPI_Fint *fh, MPI_Offset *offset, char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *status, MPI_Fint *ierr) { MPI_File c_fh = MPI_File_f2c(*fh); MPI_Datatype c_type = MPI_Type_f2c(*datatype); MPI_Status *c_status; #if OMPI_SIZEOF_FORTRAN_INTEGER != SIZEOF_INT MPI_Status c_status2; #endif /* See if we got MPI_STATUS_IGNORE */ if (OMPI_IS_FORTRAN_STATUS_IGNORE(status)) { c_status = MPI_STATUS_IGNORE; } else { /* If sizeof(int) == sizeof(INTEGER), then there's no translation necessary -- let the underlying functions write directly into the Fortran status */ #if OMPI_SIZEOF_FORTRAN_INTEGER == SIZEOF_INT c_status = (MPI_Status *) status; #else c_status = &c_status2; #endif } *ierr = OMPI_FINT_2_INT(MPI_File_write_at_all(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf), OMPI_FINT_2_INT(*count), c_type, c_status)); #if OMPI_SIZEOF_FORTRAN_INTEGER != SIZEOF_INT if (MPI_SUCCESS == OMPI_FINT_2_INT(*ierr) && MPI_STATUS_IGNORE != c_status) { MPI_Status_c2f(c_status, status); } #endif }
void write_restart(field *temperature, parallel_data *parallel, int iter) { MPI_File fp; int disp, size; // open the file and write the dimensions MPI_File_open(parallel->comm, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fp); if (parallel->rank == 0) { MPI_File_write(fp, &temperature->nx_full, 1, MPI_INT, MPI_STATUS_IGNORE); MPI_File_write(fp, &temperature->ny_full, 1, MPI_INT, MPI_STATUS_IGNORE); MPI_File_write(fp, &iter, 1, MPI_INT, MPI_STATUS_IGNORE); } // size of the local data including the outermost ghost row if at the // top or the bottom of the full grid if ((parallel->rank == 0) || (parallel->rank == parallel->size - 1)) { size = (temperature->nx + 2) * (temperature->ny + 1); } else { size = (temperature->nx + 2) * temperature->ny; } // point each MPI task to the correct part of the file disp = 3 * sizeof(int); if (parallel->rank > 0) { disp += (1 + parallel->rank * temperature->ny) * (temperature->nx + 2) * sizeof(double); } // write data simultaneously from all processes MPI_File_write_at_all(fp, disp, &temperature->data[0][0], size, MPI_DOUBLE, MPI_STATUS_IGNORE); // close up shop MPI_File_close(&fp); }
void ompi_file_write_at_all_f(MPI_Fint *fh, MPI_Offset *offset, char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *status, MPI_Fint *ierr) { int c_ierr; MPI_File c_fh = MPI_File_f2c(*fh); MPI_Datatype c_type = MPI_Type_f2c(*datatype); OMPI_FORTRAN_STATUS_DECLARATION(c_status,c_status2) OMPI_FORTRAN_STATUS_SET_POINTER(c_status,c_status2,status) c_ierr = MPI_File_write_at_all(c_fh, (MPI_Offset) *offset, OMPI_F2C_BOTTOM(buf), OMPI_FINT_2_INT(*count), c_type, c_status); if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr); OMPI_FORTRAN_STATUS_RETURN(c_status,c_status2,status,c_ierr) }
void checkpoint_timing_data(grid_parms grid, checkpoint_handle* check, double tnow, time_stamps t_stamp,int itteration){ MPI_Status status; MPI_Offset disp,disp_write; int n = 11; double buffer[n]; buffer[0] = tnow; buffer[1] = t_stamp.diff_async_comm_calls; buffer[2] = t_stamp.diff_async_comm_calls_wait; buffer[3] = t_stamp.diff_barrier_in_solver_before_comm; buffer[4] = t_stamp.diff_map_function; buffer[5] = t_stamp.diff_single_cell_fluxes; buffer[6] = t_stamp.diff_coupling_fluxes; buffer[7] = t_stamp.diff_solver; buffer[8] = t_stamp.diff_write; buffer[9] = (double) (t_stamp.computeDerivatives_call_counter); buffer[10] = (double) (itteration); int write_element_count, time_offset_in_file; write_element_count = 1; time_offset_in_file = itteration* write_element_count * grid.tasks * sizeof(double); disp_write = time_offset_in_file + (grid.rank * write_element_count * sizeof(double)); CHECK(MPI_File_write_at_all(check->time_profiling, disp_write, &buffer[0], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->async_calls, disp_write, &buffer[1], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->async_wait, disp_write, &buffer[2], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->barrier_before_comm, disp_write, &buffer[3], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->map_function, disp_write, &buffer[4], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->single_cell_fluxes, disp_write, &buffer[5], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->coupling_fluxes, disp_write, &buffer[6], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->solver, disp_write, &buffer[7], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->writer_func, disp_write, &buffer[8], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->derivative_calls, disp_write, &buffer[9], 1, MPI_DOUBLE, &status)); CHECK(MPI_File_write_at_all(check->itter_count, disp_write, &buffer[10], 1, MPI_DOUBLE, &status)); }
static int test_mpio_special_collective(char *filename) { int mpi_size, mpi_rank; MPI_File fh; MPI_Datatype etype,buftype,filetype; char mpi_err_str[MPI_MAX_ERROR_STRING]; int mpi_err_strlen; int mpi_err; char writedata[2]; char *buf; int i; int count,bufcount; int blocklens[2]; MPI_Aint offsets[2]; MPI_Offset mpi_off; MPI_Status mpi_stat; int retcode; MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); retcode = 0; /* create MPI data type */ etype = MPI_BYTE; if(mpi_rank == 0 || mpi_rank == 1) { count = DIMSIZE; bufcount = 1; } else { count = 0; bufcount = 0; } blocklens[0] = count; offsets[0] = mpi_rank*count; blocklens[1] = count; offsets[1] = (mpi_size+mpi_rank)*count; if(count !=0) { if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&filetype)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&buftype)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&buftype))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } } else { filetype = MPI_BYTE; buftype = MPI_BYTE; } /* Open a file */ if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDWR | MPI_MODE_CREATE , MPI_INFO_NULL, &fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_open failed (%s)\n", mpi_err_str); return 1; } /* each process writes some data */ for (i=0; i < 2*DIMSIZE; i++) writedata[i] = mpi_rank*DIMSIZE + i; mpi_off = 0; if((mpi_err = MPI_File_set_view(fh, mpi_off, MPI_BYTE, filetype, "native", MPI_INFO_NULL)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_set_view failed (%s)\n", mpi_err_str); return 1; } buf = writedata; if ((mpi_err = MPI_File_write_at_all(fh, mpi_off, buf, bufcount, buftype, &mpi_stat)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n", (long) mpi_off, bufcount, mpi_err_str); return 1; }; if ((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_close failed. \n"); return 1; }; mpi_err = MPI_Barrier(MPI_COMM_WORLD); #ifdef H5_MPI_SPECIAL_COLLECTIVE_IO_WORKS if(retcode != 0) { if(mpi_rank == 0) { printf("special collective IO is NOT working at this platform\n"); printf("Go back to hdf5/config and find the corresponding\n"); printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n"); printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n"); printf(" at the end of the file.\n"); printf(" Please report to [email protected] about this problem.\n"); } retcode = 1; } #else if(retcode == 0) { if(mpi_rank == 0) { printf(" This is NOT an error, What it really says is\n"); printf("special collective IO is WORKING at this platform\n"); printf(" Go back to hdf5/config and find the corresponding \n"); printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n"); printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n"); printf(" at the end of the file.\n"); printf("Please report to [email protected] about this problem.\n"); } retcode = 1; } #endif return retcode; }
int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, const char *msg, int verbose) { MPI_Datatype typevec, newtype, t[3]; int *buf, i, b[3], errcode, errors=0; MPI_File fh; MPI_Aint d[3]; MPI_Status status; int SIZE = (STARTING_SIZE/nprocs)*nprocs; MPI_Info info; if (mynod==0 && verbose) fprintf(stderr, "%s\n", msg); buf = (int *) malloc(SIZE*sizeof(int)); if (buf == NULL) { perror("test_file"); MPI_Abort(MPI_COMM_WORLD, -1); } if (cb_hosts != NULL ) { MPI_Info_create(&info); MPI_Info_set(info, "cb_config_list", cb_hosts); } else { info = MPI_INFO_NULL; } MPI_Type_vector(SIZE/nprocs, 1, nprocs, MPI_INT, &typevec); b[0] = b[1] = b[2] = 1; d[0] = 0; d[1] = mynod*sizeof(int); d[2] = SIZE*sizeof(int); t[0] = MPI_LB; t[1] = typevec; t[2] = MPI_UB; MPI_Type_struct(3, b, d, t, &newtype); MPI_Type_commit(&newtype); MPI_Type_free(&typevec); if (!mynod) { if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n"); MPI_File_delete(filename, info); } MPI_Barrier(MPI_COMM_WORLD); errcode = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); if (errcode != MPI_SUCCESS) { handle_error(errcode, "MPI_File_open"); } MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info); for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE); errcode = MPI_File_write_all(fh, buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) { handle_error(errcode, "nc mem - nc file: MPI_File_write_all"); } MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<SIZE; i++) buf[i] = -1; errcode = MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) { handle_error(errcode, "nc mem - nc file: MPI_File_read_at_all"); } /* the verification for N compute nodes is tricky. Say we have 3 * processors. * process 0 sees: 0 -1 -1 3 -1 -1 ... * process 1 sees: -1 34 -1 -1 37 -1 ... * process 2 sees: -1 -1 68 -1 -1 71 ... */ /* verify those leading -1s exist if they should */ for (i=0; i<mynod; i++ ) { if ( buf[i] != -1 ) { if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]); errors++; } } /* now the modulo games are hairy. processor 0 sees real data in the 0th, * 3rd, 6th... elements of the buffer (assuming nprocs==3 ). proc 1 sees * the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */ for(/* 'i' set in above loop */; i<SIZE; i++) { if ( ((i-mynod)%nprocs) && buf[i] != -1) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); errors++; } if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE) ) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod,i,SIZE)); errors++; } } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n"); MPI_File_delete(filename, info); } MPI_Barrier(MPI_COMM_WORLD); MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE); errcode = MPI_File_write_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "nc mem - c file: MPI_File_write_at_all"); MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<SIZE; i++) buf[i] = -1; errcode = MPI_File_read_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "nc mem - c file: MPI_File_read_at_all"); /* just like as above */ for (i=0; i<mynod; i++ ) { if ( buf[i] != -1 ) { if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]); errors++; } } for(/* i set in above loop */; i<SIZE; i++) { if ( ((i-mynod)%nprocs) && buf[i] != -1) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); errors++; } if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE)) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod,i,SIZE) ); errors++; } } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { if(verbose) fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n"); MPI_File_delete(filename, info); } MPI_Barrier(MPI_COMM_WORLD); MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info); for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod, i, SIZE); errcode = MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "c mem - nc file: MPI_File_write_all"); MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<SIZE; i++) buf[i] = -1; errcode = MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "c mem - nc file: MPI_File_read_at_all"); /* same crazy checking */ for (i=0; i<SIZE; i++) { if (buf[i] != SEEDER(mynod, i, SIZE)) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod, i, SIZE)); errors++; } } MPI_File_close(&fh); MPI_Type_free(&newtype); free(buf); if (info != MPI_INFO_NULL) MPI_Info_free(&info); return errors; }
int main(int argc, char **argv) { int *buf, i, mynod, nprocs, len, b[3]; int errs = 0, toterrs; MPI_Aint d[3]; MPI_File fh; MPI_Status status; char *filename; MPI_Datatype typevec, newtype, t[3]; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &mynod); if (nprocs != 2) { fprintf(stderr, "Run this program on two processes\n"); MPI_Abort(MPI_COMM_WORLD, 1); } /* process 0 takes the file name as a command-line argument and broadcasts it to other processes */ if (!mynod) { i = 1; while ((i < argc) && strcmp("-fname", *argv)) { i++; argv++; } if (i >= argc) { fprintf(stderr, "\n*# Usage: noncontig_coll -fname filename\n\n"); MPI_Abort(MPI_COMM_WORLD, 1); } argv++; len = strlen(*argv); filename = (char *) malloc(len + 1); strcpy(filename, *argv); MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD); } else { MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); filename = (char *) malloc(len + 1); MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD); } buf = (int *) malloc(SIZE * sizeof(int)); MPI_Type_vector(SIZE / 2, 1, 2, MPI_INT, &typevec); b[0] = b[1] = b[2] = 1; d[0] = 0; d[1] = mynod * sizeof(int); d[2] = SIZE * sizeof(int); t[0] = MPI_LB; t[1] = typevec; t[2] = MPI_UB; MPI_Type_struct(3, b, d, t, &newtype); MPI_Type_commit(&newtype); MPI_Type_free(&typevec); if (!mynod) { #if VERBOSE fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n"); #endif MPI_File_delete(filename, MPI_INFO_NULL); } MPI_Barrier(MPI_COMM_WORLD); MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL)); for (i = 0; i < SIZE; i++) buf[i] = i + mynod * SIZE; MPI_CHECK(MPI_File_write_all(fh, buf, 1, newtype, &status)); MPI_Barrier(MPI_COMM_WORLD); for (i = 0; i < SIZE; i++) buf[i] = -1; MPI_CHECK(MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status)); for (i = 0; i < SIZE; i++) { if (!mynod) { if ((i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } if (!(i % 2) && (buf[i] != i)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i); } } else { if ((i % 2) && (buf[i] != i + mynod * SIZE)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i + mynod * SIZE); } if (!(i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } } } MPI_CHECK(MPI_File_close(&fh)); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { #if VERBOSE fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n"); #endif MPI_File_delete(filename, MPI_INFO_NULL); } MPI_Barrier(MPI_COMM_WORLD); MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); for (i = 0; i < SIZE; i++) buf[i] = i + mynod * SIZE; MPI_CHECK(MPI_File_write_at_all(fh, mynod * (SIZE / 2) * sizeof(int), buf, 1, newtype, &status)); MPI_Barrier(MPI_COMM_WORLD); for (i = 0; i < SIZE; i++) buf[i] = -1; MPI_CHECK(MPI_File_read_at_all(fh, mynod * (SIZE / 2) * sizeof(int), buf, 1, newtype, &status)); for (i = 0; i < SIZE; i++) { if (!mynod) { if ((i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } if (!(i % 2) && (buf[i] != i)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i); } } else { if ((i % 2) && (buf[i] != i + mynod * SIZE)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i + mynod * SIZE); } if (!(i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } } } MPI_CHECK(MPI_File_close(&fh)); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { #if VERBOSE fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n"); #endif MPI_File_delete(filename, MPI_INFO_NULL); } MPI_Barrier(MPI_COMM_WORLD); MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL)); for (i = 0; i < SIZE; i++) buf[i] = i + mynod * SIZE; MPI_CHECK(MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status)); MPI_Barrier(MPI_COMM_WORLD); for (i = 0; i < SIZE; i++) buf[i] = -1; MPI_CHECK(MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status)); for (i = 0; i < SIZE; i++) { if (!mynod) { if (buf[i] != i) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i); } } else { if (buf[i] != i + mynod * SIZE) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i + mynod * SIZE); } } } MPI_CHECK(MPI_File_close(&fh)); MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (mynod == 0) { if (toterrs > 0) { fprintf(stderr, "Found %d errors\n", toterrs); } else { fprintf(stdout, " No Errors\n"); } } MPI_Type_free(&newtype); free(buf); free(filename); MPI_Finalize(); return 0; }
static int test_mpio_special_collective(char *filename) { int mpi_size, mpi_rank; MPI_File fh; MPI_Datatype etype,buftype,filetype; char mpi_err_str[MPI_MAX_ERROR_STRING]; int mpi_err_strlen; int mpi_err; char writedata[2*DIMSIZE]; char filerep[7] = "native"; int i; int count,bufcount; int blocklens[2]; MPI_Aint offsets[2]; MPI_Offset mpi_off = 0; MPI_Status mpi_stat; int retcode = 0; MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); /* create MPI data type */ etype = MPI_BYTE; if(mpi_rank == 0 || mpi_rank == 1) { count = DIMSIZE; bufcount = 1; } /* end if */ else { count = 0; bufcount = 0; } /* end else */ blocklens[0] = count; offsets[0] = mpi_rank*count; blocklens[1] = count; offsets[1] = (mpi_size+mpi_rank)*count; if(count !=0) { if((mpi_err = MPI_Type_hindexed(2, blocklens, offsets, etype, &filetype)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } /* end if */ if((mpi_err = MPI_Type_commit(&filetype)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } /* end if */ if((mpi_err = MPI_Type_hindexed(2, blocklens, offsets, etype, &buftype)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } /* end if */ if((mpi_err = MPI_Type_commit(&buftype)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } /* end if */ } /* end if */ else { filetype = MPI_BYTE; buftype = MPI_BYTE; } /* end else */ /* Open a file */ if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fh)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_open failed (%s)\n", mpi_err_str); return 1; } /* end if */ /* each process writes some data */ for (i=0; i < 2*DIMSIZE; i++) writedata[i] = (char)(mpi_rank*DIMSIZE + i); /* Set the file view */ if((mpi_err = MPI_File_set_view(fh, mpi_off, MPI_BYTE, filetype, filerep, MPI_INFO_NULL)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_set_view failed (%s)\n", mpi_err_str); return 1; } /* end if */ /* Collectively write into the file */ if ((mpi_err = MPI_File_write_at_all(fh, mpi_off, writedata, bufcount, buftype, &mpi_stat)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n", (long) mpi_off, bufcount, mpi_err_str); return 1; } /* end if */ /* Close the file */ if ((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_close failed. \n"); return 1; } /* end if */ /* Perform a barrier */ mpi_err = MPI_Barrier(MPI_COMM_WORLD); if(retcode != 0) { if(mpi_rank == 0) { printf("special collective IO is NOT working at this platform\n"); printf(" Please report to [email protected] about this problem.\n"); } /* end if */ retcode = 1; } /* end if */ return retcode; } /* test_mpio_special_collective */
int main(int argc, char** argv) { MPI_File fh; MPI_Datatype file_type, mem_type; int *data = NULL; int *verify = NULL; int data_size = DATA_SIZE; int i, j,k, nr_errors=0; MPI_Aint disp[BLK_COUNT]; int block_lens[BLK_COUNT]; char* filename = "unnamed.dat"; MPI_Init (&argc, &argv); disp[0] = (MPI_Aint)(PAD); disp[1] = (MPI_Aint)(data_size*1 + PAD); disp[2] = (MPI_Aint)(data_size*2 + PAD); block_lens[0] = data_size; block_lens[1] = data_size; block_lens[2] = data_size; data = malloc(data_size); verify = malloc(data_size*BLK_COUNT + HEADER + PAD); for(i=0 ; i<data_size/sizeof(int) ; i++) data[i] = i; MPI_Type_create_hindexed_block(BLK_COUNT, data_size, disp, MPI_BYTE, &file_type); MPI_Type_commit(&file_type); MPI_Type_create_hvector(BLK_COUNT, data_size, 0, MPI_BYTE, &mem_type); MPI_Type_commit(&mem_type); if( 1 < argc ) filename = argv[1]; CHECK(MPI_File_open (MPI_COMM_WORLD, filename, MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE, MPI_INFO_NULL, &fh) != 0); CHECK(MPI_File_set_view(fh, HEADER, MPI_BYTE, file_type, "native", MPI_INFO_NULL)); /* write everything */ CHECK(MPI_File_write_at_all (fh, 0, data, 1, mem_type, MPI_STATUS_IGNORE)); /* verify */ CHECK(MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL)); CHECK(MPI_File_read_at_all(fh, 0, verify, (HEADER+PAD+BLK_COUNT*DATA_SIZE)/sizeof(int), MPI_INT, MPI_STATUS_IGNORE)); /* header and block padding should have no data */ for (i=0; i<(HEADER+PAD)/sizeof(int); i++) { if (verify[i] != 0) { nr_errors++; fprintf(stderr, "expected 0, read %d\n", verify[i]); } } /* blocks are replicated */ for (j=0; j<BLK_COUNT; j++ ) { for (k=0; k<(DATA_SIZE/sizeof(int)); k++) { if (verify[(HEADER+PAD)/sizeof(int) + k + j*(DATA_SIZE/sizeof(int))] != data[k]) { nr_errors++; fprintf(stderr, "expcted %d, read %d\n", data[k], verify[(HEADER+PAD)/sizeof(int) + k + j*(DATA_SIZE/sizeof(int))]); } i++; } } MPI_File_close(&fh); MPI_Type_free (&mem_type); MPI_Type_free(&file_type); if (nr_errors == 0) printf(" No Errors\n"); MPI_Finalize (); free(data); return 0; }
int main(int argc, char *argv[]){ int i; int n_ranks; int my_rank; int data_bytes; int buffer_size; int n_iterations; char *filename_to_write=NULL; int *rank_buffer=NULL; MPI_Info my_Info=MPI_INFO_NULL; MPI_Status my_MPI_Status; MPI_File **archive_file_MPI=NULL; int file_result; long int total_data_transfer; int total_elapsed_time_s; float total_transferred_gb; float transfer_speed_gb_s; long int stage_archive_file_offset; char *archive_filenames[MAX_N_ARCHIVE_FILES]; int n_archive_files=(1); int archive_filename_length; time_t time_before,time_after; // int my_target_file; int rank_ranges[MAX_N_ARCHIVE_FILES][3]; int n_ranks_in_archive_file[MAX_N_ARCHIVE_FILES]; MPI_Group world_group; MPI_Group sub_group[MAX_N_ARCHIVE_FILES]; MPI_Comm sub_comm[MAX_N_ARCHIVE_FILES]; int my_communicator; long int total_archive_file_size; // fprintf(stderr,"parfu_write_test beginning\n"); if(argc < 6){ fprintf(stderr,"usage: \n"); fprintf(stderr," parfu_write_test <dat_bytes> <buf_bytes> <file_to_write> <n_iterations> <# arch files>\n"); MPI_Finalize(); return -1; } data_bytes=atoi(argv[1]); buffer_size=atoi(argv[2]); filename_to_write=argv[3]; n_iterations=atoi(argv[4]); n_archive_files=atoi(argv[5]); if(n_archive_files<1 || n_archive_files>MAX_N_ARCHIVE_FILES){ fprintf(stderr," you specified %d archive files! Must be >0 or <%d\n", n_archive_files,MAX_N_ARCHIVE_FILES); } MPI_Init(NULL,NULL); MPI_Comm_size(MPI_COMM_WORLD,&n_ranks); MPI_Comm_rank(MPI_COMM_WORLD,&my_rank); if(my_rank==0){ fprintf(stderr," Data payload: %d bytes.\n",data_bytes); fprintf(stderr," Buffer size: %d bytes.\n",buffer_size); fprintf(stderr," Writing to file: >%s<\n",filename_to_write); fprintf(stderr," Performing %d iterations\n",n_iterations); } // all MPI stuff below // set up multiple file output for(i=0;i<MAX_N_ARCHIVE_FILES;i++){ archive_filenames[i]=NULL; } // Creating the sub-group communicators file_result=MPI_Comm_group(MPI_COMM_WORLD,&world_group); if(file_result != MPI_SUCCESS){ fprintf(stderr,"rank %d MPI_Comm_group to get master returned %d!\n",my_rank,file_result); } for(i=0;i<n_archive_files;i++){ rank_ranges[0][0] = i; rank_ranges[0][1] = ((n_ranks/n_archive_files)*n_archive_files) + i; if(rank_ranges[0][1] >= n_ranks){ rank_ranges[0][1] -= n_archive_files; } rank_ranges[0][2] = n_archive_files; if(my_rank == 0){ fprintf(stderr,"triple [%02d]: %4d %4d %4d\n", i,rank_ranges[0][0],rank_ranges[0][1],rank_ranges[0][2]); } n_ranks_in_archive_file[i]=((rank_ranges[0][1] - rank_ranges[0][0]) / n_archive_files)+1; file_result=MPI_Group_range_incl(world_group,1,rank_ranges,sub_group+i); if(file_result != MPI_SUCCESS){ fprintf(stderr,"rank %d MPI_Group_range_incl() returned %d\n", my_rank,file_result); } } // sub groups created; now create the sub-communicators for(i=0;i<n_archive_files;i++){ MPI_Comm_create(MPI_COMM_WORLD,sub_group[i],sub_comm+i); if(file_result != MPI_SUCCESS){ fprintf(stderr,"rank_%d MPI_Comm_create() returned %d\n", my_rank,file_result); } } my_communicator = my_rank % n_archive_files; archive_filename_length = strlen(filename_to_write) + 10; for(i=0;i<n_archive_files;i++){ if((archive_filenames[i]= (char*)malloc(sizeof(char)*archive_filename_length))==NULL){ fprintf(stderr,"Could not allocate archive_filename member # %d!\n",i); MPI_Finalize(); return -4; } sprintf(archive_filenames[i],"%s__%02d",filename_to_write,i); } // for(i=0; if(my_rank==0){ fprintf(stderr,"Writing to %d archive files:\n",n_archive_files); for(i=0;i<n_archive_files;i++){ fprintf(stderr," %s\n",archive_filenames[i]); } } // allocate transfer buffer if((rank_buffer=(void*)malloc(buffer_size))==NULL){ fprintf(stderr,"rank %d failed to allocate buffer!\n",my_rank); MPI_Finalize(); return -3; } // fill buffer with numbers for(i=0;i<(data_bytes/sizeof(int));i++){ rank_buffer[i]=(i*22)+7; } // All the ranks have a buffer ready to go // now get the collective file(s) set up for writing. if((archive_file_MPI=(MPI_File**)malloc(sizeof(MPI_File*)*n_archive_files))==NULL){ fprintf(stderr,"rank %d could not allocate array for archive file pointers!\n",my_rank); MPI_Finalize(); return 75; } for(i=0;i<n_archive_files;i++){ if((archive_file_MPI[i]=(MPI_File*)malloc(sizeof(MPI_File)))==NULL){ fprintf(stderr,"rank %d could not allocate MPI file pointer number %d!!\n",my_rank,i); return 76; } } /* for(i=0;i<n_archive_files;i++){ file_result=MPI_File_open(MPI_COMM_WORLD, archive_filenames[i], MPI_MODE_WRONLY | MPI_MODE_CREATE , my_Info, archive_file_MPI[i]); if(file_result != MPI_SUCCESS){ fprintf(stderr,"MPI_File_open for archive buffer: returned error! Rank %d file >%s<\n", my_rank, archive_filenames[i]); return 3; } } */ // all files open THEIR file in THEIR communicator total_archive_file_size = ((long int)(n_ranks_in_archive_file[my_communicator])) * ((long int)(n_iterations)) * ((long int)(buffer_size)); file_result=MPI_File_open(sub_comm[my_communicator], archive_filenames[my_communicator], MPI_MODE_WRONLY | MPI_MODE_CREATE , my_Info, archive_file_MPI[my_communicator]); if(file_result != MPI_SUCCESS){ fprintf(stderr,"MPI_File_open for archive buffer: returned error! Rank %d file >%s< comm %d\n", my_rank, archive_filenames[my_communicator],my_communicator); MPI_Finalize(); return 3; } file_result=MPI_File_set_size((*(archive_file_MPI[my_communicator])),total_archive_file_size); if(file_result != MPI_SUCCESS){ fprintf(stderr,"MPI_File_set_size for archive buffer: returned error! Rank %d file >%s< comm %d\n", my_rank, archive_filenames[my_communicator],my_communicator); MPI_Finalize(); return 4; } // file(s) is(are) open on all ranks. // time to do a whole mess of writing to it(them). MPI_Barrier(MPI_COMM_WORLD); if(my_rank==0){ fprintf(stderr,"About to begin data writing loop.\n"); time(&time_before); } /* if(n_archive_files>1){ my_target_file = my_rank % n_archive_files; } else{ my_target_file=0; } */ for(i=0;i<n_iterations;i++){ stage_archive_file_offset = ((long int)( ((long int)i) * (((long int)(n_ranks/n_archive_files)) * ((long int)buffer_size)) )) + ((long int)(( (my_rank/n_archive_files) * buffer_size))); // file_result=MPI_File_write_at_all(*archive_file_MPI,stage_archive_file_offset,rank_buffer, // data_bytes,MPI_CHAR,&my_MPI_Status); // file_result=MPI_File_write_at_all(*archive_file_MPI,stage_archive_file_offset,rank_buffer, // data_bytes,MPI_CHAR,&my_MPI_Status); file_result=MPI_File_write_at_all((*(archive_file_MPI[my_communicator])),stage_archive_file_offset,rank_buffer, data_bytes,MPI_CHAR,&my_MPI_Status); if(file_result != MPI_SUCCESS){ fprintf(stderr,"rank %d i=%d got %d from MPI_File_write_at_all\n",my_rank,i,file_result); fprintf(stderr,"failed in i=%d communicator %d!!\n",i,my_communicator); MPI_Finalize(); return 77; } if(my_rank==0 && (!(i%20))){ fprintf(stderr,"."); } } // for(i=0.... if(my_rank==0) fprintf(stderr,"\n"); // MPI_File_close((*(ar MPI_Barrier(MPI_COMM_WORLD); if(my_rank==0){ time(&time_after); total_data_transfer = ((long int)data_bytes) * ((long int)n_ranks) * ((long int)n_iterations); total_elapsed_time_s = time_after - time_before; total_transferred_gb = ((float)(total_data_transfer))/1.0e9; fprintf(stderr,"total_time: %d seconds to transfer %3.4f GB\n", total_elapsed_time_s,total_transferred_gb); transfer_speed_gb_s = ( total_transferred_gb / ((float)total_elapsed_time_s) ); fprintf(stderr,"transfer speed: %3.4f GB/s\n",transfer_speed_gb_s); } // all MPI stuff above MPI_Finalize(); return 0; }
//------------------------------------------------------------------------------ // Function to output non-magnetic atomic positions to disk //------------------------------------------------------------------------------ void atoms_non_magnetic(){ //------------------------------------------------------------ // Determine non magnetic atoms to be outputted to coord list //------------------------------------------------------------ // array of atom numbers to be outputted std::vector<uint64_t> atom_list(0); // get output bounds const double minB[3] = {atoms_output_min[0] * cs::system_dimensions[0], atoms_output_min[1] * cs::system_dimensions[1], atoms_output_min[2] * cs::system_dimensions[2]}; const double maxB[3] = {atoms_output_max[0] * cs::system_dimensions[0], atoms_output_max[1] * cs::system_dimensions[1], atoms_output_max[2] * cs::system_dimensions[2]}; // Determine non magnetic atoms to be outputted to coord list for (uint64_t atom = 0; atom < cs::non_magnetic_atoms_array.size(); atom++){ const double cc[3] = {cs::non_magnetic_atoms_array[atom].x, cs::non_magnetic_atoms_array[atom].y, cs::non_magnetic_atoms_array[atom].z}; // check atom within output bounds if ( (cc[0] >= minB[0]) && (cc[0] <= maxB[0]) ){ if ( (cc[1] >= minB[1]) && (cc[1] <= maxB[1]) ){ if ( (cc[2] >= minB[2]) && (cc[2] <= maxB[2]) ){ atom_list.push_back(atom); //non-magnetic atoms } } } } //------------------------------------------------ // Create temporary buffers for atom information //------------------------------------------------ uint64_t num_local_atoms = atom_list.size(); uint64_t num_total_atoms = 0; // number of atoms across all processors #ifdef MPICF // calculate number of atoms to be output on all processors MPI_Allreduce(&num_local_atoms, &num_total_atoms, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); #else num_total_atoms = num_local_atoms; #endif std::vector<int> atom_type_buffer(num_local_atoms); for(unsigned int atom = 0; atom < num_local_atoms; atom++) atom_type_buffer[atom] = cs::non_magnetic_atoms_array[ atom_list[atom] ].mat; std::vector<int> atom_category_buffer(num_local_atoms); for(unsigned int atom = 0; atom < num_local_atoms; atom++) atom_category_buffer[atom] = cs::non_magnetic_atoms_array[ atom_list[atom] ].cat; std::vector<double> atom_coord_buffer(3*num_local_atoms); for(unsigned int atom = 0; atom < num_local_atoms; atom++){ const uint64_t atom_id = atom_list[atom]; // get atom array index atom_coord_buffer[3*atom + 0] = cs::non_magnetic_atoms_array[atom_id].x; atom_coord_buffer[3*atom + 1] = cs::non_magnetic_atoms_array[atom_id].y; atom_coord_buffer[3*atom + 2] = cs::non_magnetic_atoms_array[atom_id].z; } //------------------------------------------ // Output Meta Data from root process //------------------------------------------ // set number of files // const int files = config::internal::num_io_groups; // unused variable if(config::internal::mode != legacy && vmpi::my_rank == 0){ config::internal::write_non_magnetic_meta(num_total_atoms); } //------------------------------------------ // Output coordinate data //------------------------------------------ // Determine output filename std::stringstream file_sstr; // set simple file name for single file output if(config::internal::num_io_groups == 1) file_sstr << "non-magnetic-atoms.data"; // otherwise set indexed files else file_sstr << "non-magnetic-atoms-" << std::setfill('0') << std::setw(6) << config::internal::io_group_id << ".data"; // convert string stream to string std::string filename = file_sstr.str(); // Calculate number of bytes to be written to disk const double data_size = double(num_total_atoms) * 1.0e-9 * (3.0*double(sizeof(double) + 2.0*double(sizeof(int)) ) ); // Output informative messages of actual data size to be outputed to disk (in binary mode) zlog << zTs() << "Total non-magnetic data filesize: " << 1000.0 * data_size << " MB" << std::endl; // Output informative message to log file on root process zlog << zTs() << "Outputting non-magnetic atomic coordinate file to disk "; // Variable for calculating output bandwidth double io_time = 1.0e-12; //----------------------------------------------------- // Parallel mode output //----------------------------------------------------- #ifdef MPICF // Determine io mode and call appropriate function for data switch(config::internal::mode){ // legacy case config::internal::legacy: break; case config::internal::mpi_io:{ vutil::vtimer_t timer; // instantiate timer MPI_File fh; // MPI file handle MPI_Status status; // MPI io status // convert filename to character string for output char *cfilename = (char*)filename.c_str(); // Open file on all processors MPI_File_open(MPI_COMM_WORLD, cfilename, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fh); // write number of atoms on root process if(vmpi::my_rank == 0) MPI_File_write(fh, &num_total_atoms, 1, MPI_UINT64_T, &status); // Calculate local byte offsets since MPI-IO is simple and doesn't update the file handle pointer after I/O MPI_Offset type_offset = config::internal::linear_offset + sizeof(uint64_t); MPI_Offset category_offset = config::internal::linear_offset + num_total_atoms * sizeof(int) + sizeof(uint64_t); MPI_Offset data_offset = config::internal::buffer_offset + 2 * num_total_atoms * sizeof(int) + sizeof(uint64_t); timer.start(); // start timer // Write data to disk MPI_File_write_at_all(fh, type_offset, &atom_type_buffer[0], atom_type_buffer.size(), MPI_INT, &status); MPI_File_write_at_all(fh, category_offset, &atom_category_buffer[0], atom_category_buffer.size(), MPI_INT, &status); MPI_File_write_at_all(fh, data_offset, &atom_coord_buffer[0], atom_coord_buffer.size(), MPI_DOUBLE, &status); timer.stop(); // Stop timer // Calculate elapsed time io_time = timer.elapsed_time(); // Close file MPI_File_close(&fh); break; } case config::internal::fpprocess: io_time = write_coord_data(filename, atom_coord_buffer, atom_type_buffer, atom_category_buffer); break; case config::internal::fpnode:{ // Gather data from all processors in io group std::vector<int> collated_atom_type_buffer(0); collate_int_data(atom_type_buffer, collated_atom_type_buffer); std::vector<int> collated_atom_category_buffer(0); collate_int_data(atom_category_buffer, collated_atom_category_buffer); std::vector<double> collated_atom_coord_buffer(0); collate_double_data(atom_coord_buffer, collated_atom_coord_buffer); // output data on master io processes if(config::internal::io_group_master) io_time = write_coord_data(filename, collated_atom_coord_buffer, collated_atom_type_buffer, collated_atom_category_buffer); // find longest time in all io nodes double max_io_time = 0.0; // calculate actual bandwidth on root process MPI_Reduce(&io_time, &max_io_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); io_time = max_io_time; break; } } #else //----------------------------------------------------- // Serial mode output (ignores most io directives) //----------------------------------------------------- // if new output (not legacy) then output non magnetic atoms if(config::internal::mode != config::internal::legacy) io_time = write_coord_data(filename, atom_coord_buffer, atom_type_buffer, atom_category_buffer); #endif // Output bandwidth to log file zlog << data_size/io_time << " GB/s in " << io_time << " s" << std::endl; return; }