int main(int argc, char **argv)
{
int *buf, i, rank, nints, len;
char *filename, *tmp;
int errs=0, toterrs;
MPI_File fh;
MPI_Status status[NR_NBOPS];
MPI_Request request[NR_NBOPS];
int errcode = 0;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: async -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE);
nints = SIZE/sizeof(int);
for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
/* each process opens a separate file called filename.'myrank' */
tmp = (char *) malloc(len+10);
strcpy(tmp, filename);
sprintf(filename, "%s.%d", tmp, rank);
errcode = MPI_File_open(MPI_COMM_SELF, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
for (i=0; i<NR_NBOPS; i++) {
errcode = MPI_File_iwrite_at(fh, nints/NR_NBOPS*i,
buf+(nints/NR_NBOPS*i), nints/NR_NBOPS, MPI_INT, &(request[i]));
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_iwrite");
}
}
MPI_Waitall(NR_NBOPS, request, status);
MPI_File_close(&fh);
/* reopen the file and read the data back */
for (i=0; i<nints; i++) buf[i] = 0;
errcode = MPI_File_open(MPI_COMM_SELF, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
for (i=0; i<NR_NBOPS; i++) {
errcode = MPI_File_iread_at(fh, nints/NR_NBOPS*i,
buf+(nints/NR_NBOPS*i), nints/NR_NBOPS, MPI_INT, &(request[i]));
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
}
MPI_Waitall(NR_NBOPS, request, status);
MPI_File_close(&fh);
/* check if the data read is correct */
for (i=0; i<nints; i++) {
if (buf[i] != (rank*100000 + i)) {
errs++;
fprintf(stderr, "Process %d: error, read %d, should be %d\n", rank, buf[i], rank*100000+i);
}
}
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (rank == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
free(buf);
free(filename);
free(tmp);
MPI_Finalize();
return 0;
}
PetscErrorCode VecView_Seq_Binary(Vec xin,PetscViewer viewer)
{
PetscErrorCode ierr;
int fdes;
PetscInt n = xin->map->n,classid=VEC_FILE_CLASSID;
FILE *file;
const PetscScalar *xv;
#if defined(PETSC_HAVE_MPIIO)
PetscBool isMPIIO;
#endif
PetscBool skipHeader;
PetscViewerFormat format;
PetscFunctionBegin;
/* Write vector header */
ierr = PetscViewerBinaryGetSkipHeader(viewer,&skipHeader);CHKERRQ(ierr);
if (!skipHeader) {
ierr = PetscViewerBinaryWrite(viewer,&classid,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscViewerBinaryWrite(viewer,&n,1,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
}
/* Write vector contents */
#if defined(PETSC_HAVE_MPIIO)
ierr = PetscViewerBinaryGetUseMPIIO(viewer,&isMPIIO);CHKERRQ(ierr);
if (!isMPIIO) {
#endif
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
ierr = VecGetArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,(void*)xv,n,PETSC_SCALAR,PETSC_FALSE);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscViewerGetFormat(viewer,&format);CHKERRQ(ierr);
if (format == PETSC_VIEWER_BINARY_MATLAB) {
MPI_Comm comm;
FILE *info;
const char *name;
ierr = PetscObjectGetName((PetscObject)xin,&name);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetInfoPointer(viewer,&info);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,info,"#--- begin code written by PetscViewerBinary for MATLAB format ---#\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,info,"#$$ Set.%s = PetscBinaryRead(fd);\n",name);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,info,"#--- end code written by PetscViewerBinary for MATLAB format ---#\n\n");CHKERRQ(ierr);
}
#if defined(PETSC_HAVE_MPIIO)
} else {
MPI_Offset off;
MPI_File mfdes;
PetscMPIInt lsize;
ierr = PetscMPIIntCast(n,&lsize);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIODescriptor(viewer,&mfdes);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIOOffset(viewer,&off);CHKERRQ(ierr);
ierr = MPI_File_set_view(mfdes,off,MPIU_SCALAR,MPIU_SCALAR,(char*)"native",MPI_INFO_NULL);CHKERRQ(ierr);
ierr = VecGetArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = MPIU_File_write_all(mfdes,(void*)xv,lsize,MPIU_SCALAR,MPI_STATUS_IGNORE);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscViewerBinaryAddMPIIOOffset(viewer,n*sizeof(PetscScalar));CHKERRQ(ierr);
}
#endif
ierr = PetscViewerBinaryGetInfoPointer(viewer,&file);CHKERRQ(ierr);
if (file) {
if (((PetscObject)xin)->prefix) {
ierr = PetscFPrintf(PETSC_COMM_SELF,file,"-%svecload_block_size %D\n",((PetscObject)xin)->prefix,PetscAbs(xin->map->bs));CHKERRQ(ierr);
} else {
ierr = PetscFPrintf(PETSC_COMM_SELF,file,"-vecload_block_size %D\n",PetscAbs(xin->map->bs));CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int main(int argc, char *argv[]) {
int my_id, nprocs;
int mpi_dims[4];
int period[4] = {0, 0, 0, 0};
int coords[4];
int dimsf[4] = {nbands, gpts, gpts, gpts};
int count[4];
int offset[4];
int ndims = 4;
double t0, t1;
#ifdef PAPI
PAPI_dmem_info_t dmem;
double mem1, mem2, mem1_max, mem2_max, mem1_ave, mem2_ave;
int papi_err;
#endif
double *my_data;
MPI_Comm cart_comm;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
assert(argc == 5);
for (int i=1; i < argc; i++)
mpi_dims[i-1] = atoi(argv[i]);
assert(mpi_dims[0] * mpi_dims[1] * mpi_dims[2] * mpi_dims[3] == nprocs);
MPI_Cart_create(MPI_COMM_WORLD, 4, mpi_dims, period, 0, &cart_comm);
MPI_Comm_rank(cart_comm, &my_id);
MPI_Cart_coords(cart_comm, my_id, 4, coords);
assert(nbands % mpi_dims[0] == 0);
for (int i=1; i < 4; i++)
assert(gpts % mpi_dims[i] == 0);
int total_size = nbands*gpts*gpts*gpts;
count[0] = nbands / mpi_dims[0];
offset[0] = coords[0] * count[0];
int data_size = count[0];
for (int i=1; i < 4; i++)
{
count[i] = gpts/mpi_dims[i];
offset[i] = coords[i] * count[i];
data_size *= count[i];
}
my_data = (double *) malloc(data_size * sizeof(double));
for (int i=0; i < data_size; i++)
my_data[i] = my_id;
MPI_Info info;
MPI_File fp;
MPI_Datatype filetype;
// MPI_Info_set(info, "cb_nodes", "64");
MPI_Barrier(MPI_COMM_WORLD);
#ifdef PAPI
papi_err = PAPI_get_dmem_info(&dmem);
if (papi_err != PAPI_OK)
printf("PAPI_ERR\n");
mem1 = (double)dmem.size / 1024.0;
MPI_Reduce(&mem1, &mem1_max, 1, MPI_DOUBLE, MPI_MAX, 0, cart_comm);
MPI_Reduce(&mem1, &mem1_ave, 1, MPI_DOUBLE, MPI_SUM, 0, cart_comm);
mem1_ave /= nprocs;
#endif
t0 = MPI_Wtime();
MPI_File_open(MPI_COMM_WORLD, "test.dat",
MPI_MODE_CREATE|MPI_MODE_WRONLY,
MPI_INFO_NULL, &fp);
MPI_Type_create_subarray(ndims, dimsf, count, offset, MPI_ORDER_C,
MPI_DOUBLE, &filetype);
MPI_Type_commit(&filetype);
MPI_File_set_view(fp, 0, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL);
MPI_File_write_all(fp, my_data, data_size, MPI_DOUBLE, MPI_STATUS_IGNORE);
MPI_Type_free(&filetype);
MPI_File_close(&fp);
MPI_Barrier(MPI_COMM_WORLD);
t1 = MPI_Wtime();
#ifdef PAPI
papi_err = PAPI_get_dmem_info(&dmem);
if (papi_err != PAPI_OK)
printf("PAPI_ERR\n");
mem2 = (double)dmem.size/ 1024.0;
MPI_Reduce(&mem2, &mem2_max, 1, MPI_DOUBLE, MPI_MAX, 0, cart_comm);
MPI_Reduce(&mem2, &mem2_ave, 1, MPI_DOUBLE, MPI_SUM, 0, cart_comm);
mem2_ave /= nprocs;
#endif
if (my_id == 0)
{
printf("IO time %f (%f) MB %f s\n",
total_size * 8/(1024.0*1024.0),
data_size * 8/(1024.0*1024.0), t1-t0);
#ifdef PAPI
printf("Memory usage max (ave): %f (%f) %f (%f) \n",
mem1_max, mem1_ave, mem2_max, mem2_ave);
#endif
}
MPI_Finalize();
}
static int test_mpio_derived_dtype(char *filename) {
MPI_File fh;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
int i;
int nerrors = 0; /* number of errors */
MPI_Datatype etype,filetype;
MPI_Datatype adv_filetype,bas_filetype[2];
MPI_Datatype etypenew, filetypenew;
MPI_Offset disp;
MPI_Status Status;
MPI_Aint adv_disp[2];
MPI_Aint offsets[1];
int blocklens[1],adv_blocklens[2];
int count,outcount;
int retcode;
int mpi_rank,mpi_size;
char buf[3],outbuf[3] = {0};
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
retcode = 0;
for(i=0;i<3;i++)
buf[i] = i+1;
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
disp = 0;
etype = MPI_BYTE;
count = 1;
blocklens[0] = 1;
offsets[0] = 0;
if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
count = 1;
blocklens[0]=1;
offsets[0] = 1;
if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetypenew))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetypenew))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
outcount = 2;
adv_blocklens[0] = 1;
adv_blocklens[1] = 1;
adv_disp[0] = 0;
adv_disp[1] = 1;
bas_filetype[0] = filetype;
bas_filetype[1] = filetypenew;
if((mpi_err= MPI_Type_struct(outcount,adv_blocklens,adv_disp,bas_filetype,&adv_filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_struct failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&adv_filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_set_view(fh,disp,etype,adv_filetype,"native",MPI_INFO_NULL))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_write(fh,buf,3,MPI_BYTE,&Status))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_write failed (%s)\n", mpi_err_str);
return 1;
;
}
if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_open(MPI_COMM_WORLD,filename,MPI_MODE_RDONLY,MPI_INFO_NULL,&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_set_view(fh,0,MPI_BYTE,MPI_BYTE,"native",MPI_INFO_NULL))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_read(fh,outbuf,3,MPI_BYTE,&Status))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_read failed (%s)\n", mpi_err_str);
return 1;
}
if(outbuf[2]==2) {
retcode = 0;
}
else {
/* if(mpi_rank == 0) {
printf("complicated derived datatype is NOT working at this platform\n");
printf("go back to hdf5/config and find the corresponding\n");
printf("configure-specific file and change ?????\n");
}
*/
retcode = -1;
}
if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed (%s)\n", mpi_err_str);
return 1;
}
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
if(retcode == -1) {
if(mpi_rank == 0) {
printf("Complicated derived datatype is NOT working at this platform\n");
printf(" Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
return retcode;
}
void IOserver::start()
{
//starting IO server (Sinc line)
MPI_Status status,status2;
MPI_Request send_statut_request;
int flag=1;
int control;
int count=0;
int fileID;
int fileCounter;
int newFilenameLength;
int len;
bool getingDataFlag;
int getingData[IO_ClientSize_];
serverOn_flag=true;
while(serverOn_flag)
{
//cout<<count<<endl;
ostreamFile_flag=false;
if(IO_Rank_==0)
{
//cout<<count<<endl;
//count++;
//send non-blocking for status server free. (Sync line)
serverReady_flag=true;
MPI_Isend(&serverReady_flag,1,MPI::BOOL,1,SERVER_STATE_TAG,syncLineComm_,&send_statut_request);
//test if an ostream need to be open or if the server need to be stoped (Sync line) blocking rec
MPI_Recv(&control,1,MPI::INT,1,SERVER_CONTROL_TAG,syncLineComm_,&status);
}
MPI_Bcast(&control,1,MPI::INT,0,IO_Comm_);
//MPI_Bcast(&ostreamFile_flag,1,MPI::BOOL,0,IO_Comm_);
if(control==CONTROL_STOP)
{
//if(IO_Rank_==0)cout<<"stop server"<<endl;
serverOn_flag=false;
}
if(control==CONTROL_OPEN_OSTREAM)
{
//if(IO_Rank_==0)cout<<"ostream"<<endl;
ostreamFile_flag=true;
serverReady_flag=false;
if(IO_Rank_==0)MPI_Isend(&serverReady_flag,1,MPI::BOOL,1,SERVER_STATE_TAG,syncLineComm_,&send_statut_request);
fileCounter=0;
while(ostreamFile_flag)
{
//cout<<"rank: "<<IO_Rank_<<", ostream open"<<endl;
//test if there is a new file (Sync line)
if(IO_Rank_==0)MPI_Recv(&control,1,MPI::INT,1,IO_FILE_CONTROL_TAG,syncLineComm_,&status);
MPI_Bcast(&control,1,MPI::INT,0,IO_Comm_);
if(control==CONTROL_CLOSE_OSTREAM)
{
//if(IO_Rank_==0)cout<<"stop ostream"<<endl;
ostreamFile_flag=false;
}
if(control==CONTROL_CREATE_FILE)
{
//create new file ID
if(fileCounter>= MAX_FILE_NUMBER)
{
if(IO_Rank_==0)cout<<"Already to much file"<<endl;
fileID=FILE_FAIL;
}
else
{
fileID=fileCounter;
//if(IO_Rank_==0)cout<<"IO_server creating new file : "<<fileID<<endl;
}
if(IO_Rank_==0)MPI_Ssend(&fileID,1,MPI::INT,1,IO_FILE_CONTROL_FILEID_TAG,syncLineComm_);
if(fileID != FILE_FAIL)
{
if(IO_Rank_==0)
{
MPI_Probe(1,IO_FILE_CONTROL_FILENAME_TAG,syncLineComm_,&status);
MPI_Get_count(&status,MPI::CHAR,&newFilenameLength);
}
MPI_Bcast(&newFilenameLength,1,MPI::INT,0,IO_Comm_);
char * filename;
filename = (char*)malloc(newFilenameLength*sizeof(char));
if(IO_Rank_==0)MPI_Recv(filename,newFilenameLength,MPI::CHAR,1,IO_FILE_CONTROL_FILENAME_TAG,syncLineComm_,&status2);
MPI_Bcast(filename,newFilenameLength,MPI::CHAR,0,IO_Comm_);
files[fileID].filename=filename;
free(filename);
files[fileID].type=FILETYPE_UNSTRUCTURED; // no structured file implemented!!!!
//cout<< files[fileID].filename <<endl;
if(fileID==0)files[fileID].data=dataBuffer;
else files[fileID].data=&(files[fileID-1].data[files[fileID-1].size]);
files[fileID].size=0;
fileCounter++;
if(files[fileID].type==FILETYPE_UNSTRUCTURED)
{
for(int i=0;i<IO_ClientSize_;i++)getingData[i]=1;
getingDataFlag=true;
while(getingDataFlag)
{
MPI_Iprobe(MPI_ANY_SOURCE,fileID, masterClientComm_,&flag,&status);
if(flag==true)
{
char * send;
send=&(files[fileID].data[files[fileID].size]);
int size;
MPI_Get_count(&status,MPI::CHAR,&size);
MPI_Recv(send,size,MPI::CHAR,status.MPI_SOURCE,fileID,masterClientComm_,&status2);
files[fileID].size+=size;
}
else
{
MPI_Iprobe(MPI_ANY_SOURCE,IO_FILE_CONTROL_CLOSE_TAG,masterClientComm_ ,&flag,&status);
if(flag==true)
{
//cout<<"file closed"<<endl;
int tempFileID;
int total=0;
int source = status.MPI_SOURCE;
//cout<<source<<endl;
MPI_Recv(&tempFileID,1,MPI::INT,source,IO_FILE_CONTROL_CLOSE_TAG,masterClientComm_,&status2);
//cout<<source<<endl;
getingData[source-1]=0;
for(int i=0;i<IO_ClientSize_;i++)total+=getingData[i];
if(total==0)getingDataFlag=false;
//cout<<total<<endl;
}
}
}
//cout<<"file closed"<<endl;
MPI_Barrier(IO_Comm_);
}
MPI_Barrier(IO_Comm_);
}
}
}//close stream and write
if(fileCounter!=0)
{
for(int i=0;i < fileCounter;i++)
{
//cout<<"writing file: "<< i<<endl;
MPI_File ofile;
long file_Offset=0;
long temp_long=0;
string str_fname;
str_fname = files[i].filename + int2string(IO_Node_,999)+".dat";
char * fname = &(str_fname[0]);
for(int k=0;k<(IO_NodeSize_-1); k++)
{
if(IO_NodeRank_==k)
{
temp_long = file_Offset + files[i].size;
MPI_Send(&temp_long,1,MPI_LONG, k+1 , 0, IO_NodeComm_ );
}
if(IO_NodeRank_==k+1)
{
MPI_Recv( &file_Offset, 1, MPI_LONG, k, 0, IO_NodeComm_, &status);
}
}
if(files[i].size!=0)
{
MPI_File_open(IO_NodeComm_,fname,MPI_MODE_WRONLY | MPI_MODE_CREATE,MPI_INFO_NULL,&ofile);
MPI_File_set_view(ofile,file_Offset,MPI_CHAR,MPI_CHAR,(char*)"native",MPI_INFO_NULL);
MPI_File_write_all(ofile,files[i].data,files[i].size,MPI_CHAR,&status);
MPI_File_close(&ofile);
}
}
}
fileCounter=0;
MPI_Barrier(IO_Comm_);
}
}
}
int main( int argc, char *argv[] )
{
int opt;
extern char *optarg;
extern int optind;
int is_output_timing=0, is_print_usage = 0;
int _debug=0, use_gen_file = 0, use_actsto = 0, use_normalsto=0;
char *token;
MPI_Offset disp, offset, file_size;
MPI_Datatype etype, ftype, buftype;
int errs = 0;
int size, rank, i, count;
char *fname = NULL;
double *buf;
MPI_File fh;
MPI_Comm comm;
MPI_Status status;
int64_t nitem = 0;
int fsize = 0, type_size;
double stime, etime, iotime, comptime, elapsed_time;
double max_iotime, max_comptime;
double max, min, sum=0.0, global_sum;
MPI_Init( &argc, &argv );
comm = MPI_COMM_WORLD;
MPI_Comm_size( comm, &size );
MPI_Comm_rank( comm, &rank );
while ( (opt=getopt(argc,argv,"i:s:godhxt"))!= EOF) {
switch (opt) {
case 'i': fname = optarg;
break;
case 'o': is_output_timing = 1;
break;
case 'g': use_gen_file = 1;
break;
case 'd': _debug = 1;
break;
case 'h': is_print_usage = 1;
break;
case 's':
token = strtok(optarg, ":");
//if (rank == 0) printf("token=%s\n", token);
if(token == NULL) {
if (rank == 0) printf("1: Wrong file size format!\n");
MPI_Finalize();
exit(1);
}
fsize = atoi(token);
token = strtok(NULL, ":");
//if (rank == 0) printf("token=%s\n", token);
if(token == NULL) {
if (rank == 0) printf("2: Wrong file size format!\n");
MPI_Finalize();
exit(1);
}
if(*token != 'm' && *token != 'g') {
if (rank == 0) printf("3: Wrong file size format!\n");
MPI_Finalize();
exit(1);
}
if (rank ==0) printf("fsize = %d (%s)\n", fsize, (*token=='m'?"MB":"GB"));
if (fsize == 0)
nitem = 0;
else {
MPI_Type_size(MPI_DOUBLE, &type_size);
nitem = fsize*1024; /* KB */
nitem = nitem*1024; /* MB */
if(*token == 'g') {
//if(rank == 0) printf("data in GB\n");
nitem = nitem*1024; /* GB */
}
nitem = nitem/type_size;
//printf("nitem=%lld\n", nitem);
nitem = nitem/size; /* size means comm size */
}
if (rank == 0) printf("nitem = %d\n", nitem);
break;
case 'x': use_actsto = 1;
break;
case 't': use_normalsto = 1;
break;
default: is_print_usage = 1;
break;
}
}
if (fname == NULL || is_print_usage == 1 || nitem == 0) {
if (rank == 0) usage(argv[0]);
MPI_Finalize();
exit(1);
}
int sizeof_mpi_offset;
sizeof_mpi_offset = (int)(sizeof(MPI_Offset)); // 8
//if (rank == 0) printf ("size_of_mpi_offset=%d\n", sizeof_mpi_offset);
if(use_normalsto == 1 && use_actsto == 1) {
if(rank == 0)
printf("Can't test both: either normalsto or actsto\n");
MPI_Finalize();
exit(1);
}
#if 0
if(use_actsto == 1) {
if (size != 1) {
if(rank == 0)
printf("active storage should be run with only 1 process!!!\n");
MPI_Finalize();
exit(1);
}
}
#endif
/* initialize random seed: */
srand(time(NULL));
if(use_gen_file == 1) {
int t, result;
MPI_File_open( comm, fname, MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fh );
/* Set the file view */
disp = rank * nitem * type_size;
printf("%d: disp = %lld\n", rank, disp);
etype = MPI_DOUBLE;
ftype = MPI_DOUBLE;
result = MPI_File_set_view(fh, disp, etype, ftype, "native", MPI_INFO_NULL);
if(result != MPI_SUCCESS)
sample_error(result, "MPI_File_set_view");
buf = (double *)malloc( nitem * sizeof(double) );
if (buf == NULL) {
if(rank == 0) printf("malloc() failed\n");
MPI_Finalize();
exit(1);
}
buf[0] = rand()%4096;
if(rank==0) printf("%lf\n", buf[0]);
max = min = sum = buf[0];
for(i=1; i<nitem; i++) {
t = rand()%4096;
if (t>max) max = t;
if (t<min) min = t;
sum += t;
buf[i] = t;
if (i<10 && rank == 0) printf("%lf\n", buf[i]);
}
if(rank == 0) {
printf("MPI_Type_size(MPI_DOUBLE)=%d\n", type_size);
printf ("max=%lf, min=%lf, sum=%lf\n", max, min, sum);
}
stime = MPI_Wtime();
/* Write to file */
MPI_File_write_all( fh, buf, nitem, MPI_DOUBLE, &status );
etime = MPI_Wtime();
iotime = etime - stime;
printf("%d: iotime (write) = %10.4f\n", rank, iotime);
MPI_Get_count( &status, MPI_DOUBLE, &count );
//printf("count = %lld\n", count);
if (count != nitem) {
fprintf( stderr, "%d: Wrong count (%lld) on write\n", rank, count );
fflush(stderr);
/* exit */
MPI_Finalize();
exit(1);
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if(rank == 0) printf("File is written\n\n");
}
double *tmp = (double *)malloc( nitem * sizeof(double) );
memset (tmp, 0, nitem*sizeof(double));
if(use_normalsto == 1) {
MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh );
/* Read nothing (check status) */
memset( &status, 0xff, sizeof(MPI_Status) );
offset = rank * nitem * type_size;
/* start I/O */
stime = MPI_Wtime();
MPI_File_read_at(fh, offset, tmp, nitem, MPI_DOUBLE, &status);
etime = MPI_Wtime();
/* end I/O */
iotime = etime - stime;
if(_debug==1) printf("%d: iotime = %10.4f\n", rank, iotime);
MPI_Reduce(&iotime, &max_iotime, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
sum = 0.0; /* reset sum */
/* start computation */
stime = MPI_Wtime();
for(i=0; i<nitem; i++) {
sum += tmp[i];
}
MPI_Reduce(&sum, &global_sum, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
etime = MPI_Wtime();
/* end computation */
comptime = etime - stime;
if(_debug==1) printf("%d: comptime = %10.4f\n", rank, comptime);
MPI_Reduce(&comptime, &max_comptime, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if(rank == 0) {
elapsed_time = max_comptime + max_iotime;
printf("<<Result (SUM) with normal read>>\n"
"SUM = %10.4f \n"
"Computation time = %10.4f sec\n"
"I/O time = %10.4f sec\n"
"total time = %10.4f sec\n\n",
global_sum, max_comptime, max_iotime, elapsed_time);
}
MPI_File_close(&fh);
}
#if 0
if(use_actsto == 1) {
#if 0
/* MPI_MAX */
MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh );
stime = MPI_Wtime();
MPI_File_read_at_ex( fh, offset, tmp, nitem, MPI_DOUBLE, MPI_MAX, &status );
etime = MPI_Wtime();
elapsed_time = etime-stime;
printf ("<<Result with active storage>>\n"
"max=%lf (in %10.4f sec)\n", tmp[0], elapsed_time);
MPI_File_close(&fh);
/* MPI_MIN */
MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh );
stime = MPI_Wtime();
MPI_File_read_at_ex( fh, offset, tmp, nitem, MPI_DOUBLE, MPI_MIN, &status );
etime = MPI_Wtime();
elapsed_time = etime - stime;
printf ("min=%lf (in %10.4f sec)\n", tmp[0], elapsed_time);
MPI_File_close(&fh);
#endif
/* MPI_SUM */
MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh );
memset(&status, 0xff, sizeof(MPI_Status));
offset = rank * nitem * type_size;
stime = MPI_Wtime();
MPI_File_read_at_ex( fh, offset, tmp, nitem, MPI_DOUBLE, MPI_SUM, &status );
etime = MPI_Wtime();
elapsed_time = etime - stime;
printf ("<<Result with active storage>>\n"
"sum=%lf (in %10.4f sec)\n", tmp[0], elapsed_time);
MPI_File_close( &fh );
}
#endif
MPI_Barrier(MPI_COMM_WORLD);
if (use_gen_file == 1) free( buf );
free( tmp );
MPI_Finalize();
return errs;
}
int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, char *msg, int verbose)
{
MPI_Datatype typevec, newtype, t[3];
int *buf, i, b[3], errcode, errors=0;
MPI_File fh;
MPI_Aint d[3];
MPI_Status status;
int SIZE = (STARTING_SIZE/nprocs)*nprocs;
MPI_Info info;
if (mynod==0 && verbose) fprintf(stderr, "%s\n", msg);
buf = (int *) malloc(SIZE*sizeof(int));
if (buf == NULL) {
perror("test_file");
MPI_Abort(MPI_COMM_WORLD, -1);
}
if (cb_hosts != NULL ) {
MPI_Info_create(&info);
MPI_Info_set(info, "cb_config_list", cb_hosts);
} else {
info = MPI_INFO_NULL;
}
MPI_Type_vector(SIZE/nprocs, 1, nprocs, MPI_INT, &typevec);
b[0] = b[1] = b[2] = 1;
d[0] = 0;
d[1] = mynod*sizeof(int);
d[2] = SIZE*sizeof(int);
t[0] = MPI_LB;
t[1] = typevec;
t[2] = MPI_UB;
MPI_Type_struct(3, b, d, t, &newtype);
MPI_Type_commit(&newtype);
MPI_Type_free(&typevec);
if (!mynod) {
if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n");
MPI_File_delete(filename, info);
}
MPI_Barrier(MPI_COMM_WORLD);
errcode = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
errcode = MPI_File_write_all(fh, buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "nc mem - nc file: MPI_File_write_all");
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
errcode = MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "nc mem - nc file: MPI_File_read_at_all");
}
/* the verification for N compute nodes is tricky. Say we have 3
* processors.
* process 0 sees: 0 -1 -1 3 -1 -1 ...
* process 1 sees: -1 34 -1 -1 37 -1 ...
* process 2 sees: -1 -1 68 -1 -1 71 ... */
/* verify those leading -1s exist if they should */
for (i=0; i<mynod; i++ ) {
if ( buf[i] != -1 ) {
if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
errors++;
}
}
/* now the modulo games are hairy. processor 0 sees real data in the 0th,
* 3rd, 6th... elements of the buffer (assuming nprocs==3 ). proc 1 sees
* the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */
for(/* 'i' set in above loop */; i<SIZE; i++) {
if ( ((i-mynod)%nprocs) && buf[i] != -1) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
errors++;
}
if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE) ) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], SEEDER(mynod,i,SIZE));
errors++;
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n");
MPI_File_delete(filename, info);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE);
errcode = MPI_File_write_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int),
buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "nc mem - c file: MPI_File_write_at_all");
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
errcode = MPI_File_read_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int),
buf, 1, newtype, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "nc mem - c file: MPI_File_read_at_all");
/* just like as above */
for (i=0; i<mynod; i++ ) {
if ( buf[i] != -1 ) {
if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]);
errors++;
}
}
for(/* i set in above loop */; i<SIZE; i++) {
if ( ((i-mynod)%nprocs) && buf[i] != -1) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
errors++;
}
if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE)) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], SEEDER(mynod,i,SIZE) );
errors++;
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
if(verbose) fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n");
MPI_File_delete(filename, info);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod, i, SIZE);
errcode = MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "c mem - nc file: MPI_File_write_all");
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<SIZE; i++) buf[i] = -1;
errcode = MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status);
if (errcode != MPI_SUCCESS)
handle_error(errcode, "c mem - nc file: MPI_File_read_at_all");
/* same crazy checking */
for (i=0; i<SIZE; i++) {
if (buf[i] != SEEDER(mynod, i, SIZE)) {
if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod, i, SIZE));
errors++;
}
}
MPI_File_close(&fh);
MPI_Type_free(&newtype);
free(buf);
if (info != MPI_INFO_NULL) MPI_Info_free(&info);
return errors;
}
/* *************************** *
* Main computational kernel *
* *************************** */
int correlationKernel(int rank,
int size,
double* dataMatrixX,
double* dataMatrixY,
int columns,
int rows,
char *out_filename,
int distance_flag) {
int local_check = 0, global_check = 0;
int i = 0, j, taskNo;
int err, count = 0;
unsigned long long fair_chunk = 0, coeff_count = 0;
unsigned int init_and_cleanup_loop_iter=0;
unsigned long long cor_cur_size = 0;
double start_time, end_time;
// Variables needed by the Indexed Datatype
MPI_Datatype coeff_index_dt;
MPI_File fh;
int *blocklens, *indices;
MPI_Status stat;
MPI_Comm comm = MPI_COMM_WORLD;
// Master processor keeps track of tasks
if (rank == 0) {
// Make sure everything will work fine even if there are
// less genes than available workers (there are size-1 workers
// master does not count)
if ( (size-1) > rows )
init_and_cleanup_loop_iter = rows+1;
else
init_and_cleanup_loop_iter = size;
// Start timer
start_time = MPI_Wtime();
// Send out initial tasks (remember you have size-1 workers, master does not count)
for (i=1; i<init_and_cleanup_loop_iter; i++) {
taskNo = i-1;
err = MPI_Send(&taskNo, 1, MPI_INT, i, 0, comm);
}
// Terminate any processes that were not working due to the fact
// that the number of rows where less than the actual available workers
for(i=init_and_cleanup_loop_iter; i < size; i++) {
PROF(rank, "\nPROF_idle : Worker %d terminated due to insufficient work load", i);
err = -1;
err = MPI_Send(&err, 1, MPI_INT, i, 0, comm);
}
// Wait for workers to finish their work assignment and ask for more
for (i=init_and_cleanup_loop_iter-1; i<rows; i++) {
err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat);
// Check taskNo to make sure everything is ok. Negative means there is problem
// thus terminate gracefully all remaining working workers
if ( taskNo < 0 ) {
// Reduce by one because one worker is already terminated
init_and_cleanup_loop_iter--;
// Break and cleanup
break;
}
// The sending processor is ready to work:
// It's ID is in stat.MPI_SOURCE
// Send it the current task (i)
err = MPI_Send(&i, 1, MPI_INT, stat.MPI_SOURCE, 0, comm);
}
// Clean up processors
for (i=1; i<init_and_cleanup_loop_iter; i++) {
// All tasks complete - shutdown workers
err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat);
// If process failed then it will not be waiting to receive anything
// We have to ignore the send because it will deadlock
if ( taskNo < 0 )
continue;
err = -1;
err = MPI_Send(&err, 1, MPI_INT, stat.MPI_SOURCE, 0, comm);
}
// Master is *always* OK
local_check = 0;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Check failed, abort
if ( global_check != 0 ) {
return -1;
}
// Stop timer
end_time = MPI_Wtime();
PROF(rank, "\nPROF_comp (workers=%d) : Time taken by correlation coefficients computations : %g\n", size-1, end_time - start_time);
// Start timer
start_time = MPI_Wtime();
// Master process must call MPI_File_set_view as well, it's a collective call
// Open the file handler
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
// Create the file view
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
// Write data to disk
MPI_File_write_all(fh, &cor[0], 0, MPI_DOUBLE, &stat);
// Stop timer
end_time = MPI_Wtime();
PROF(rank, "\nPROF_write (workers=%d) : Time taken for global write-file : %g\n", size-1, end_time - start_time);
} else {
// Compute how many workers will share the work load
// Two scenarios exist:
// (1) more OR equal number of workers and rows exist
// (2) more rows than workers
if ( (size-1) > rows ) {
// For this scenario each worker will get exaclty one work asssignment.
// There is not going to be any other work so it only compute "rows" number
// of coefficients
fair_chunk = rows;
cor_cur_size = fair_chunk;
} else {
// For this scenario we are going to allocate space equal to a fair
// distribution of work assignments *plus* an extra amount of space to
// cover any load imbalancing. This amount is expressed as a percentage
// of the fair work distribution (see on top, 20% for now)
// Plus 1 to round it up or just add some extra space, both are fine
fair_chunk = (rows / (size-1)) + 1;
DEBUG("fair_chunk %d \n", fair_chunk);
// We can use "j" as temporary variable.
// Plus 1 to avoid getting 0 from the multiplication.
j = (fair_chunk * MEM_PERC) + 1;
cor_cur_size = (fair_chunk + j) * rows;
DEBUG("cor_cur_size %lld \n", cor_cur_size);
}
// Allocate memory
DEBUG("cor_cur_size %lld \n", cor_cur_size);
long long double_size = sizeof(double);
DEBUG("malloc size %lld \n", (double_size * cor_cur_size));
cor = (double *)malloc(double_size * cor_cur_size);
blocklens = (int *)malloc(sizeof(int) * rows);
indices = (int *)malloc(sizeof(int) * rows);
mean_value_vectorX = (double *)malloc(sizeof(double) * rows);
Sxx_vector = (double *)malloc(sizeof(double) * rows);
mean_value_vectorY = (double *)malloc(sizeof(double) * rows);
Syy_vector = (double *)malloc(sizeof(double) * rows);
// Check that all memory is successfully allocated
if ( ( cor == NULL ) || ( blocklens == NULL ) || ( indices == NULL ) ||
( mean_value_vectorX == NULL ) || ( Sxx_vector == NULL ) ||
( mean_value_vectorY == NULL ) || ( Syy_vector == NULL ) ) {
ERR("**ERROR** : Memory allocation failed on worker process %d. Aborting.\n", rank);
// Free allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
// We have to receive a work assignment first and then terminate
// otherwise the master will deadlock trying to give work to this worker
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
return -1;
}
// Compute necessary parameters for Pearson method
// (this will transform the values of the input array to more meaningful data
// and save us from a lot of redundant computations)
compute_parameters(dataMatrixX, dataMatrixY, rows, columns);
// Main loop for workers. They get work from master, compute coefficients,
// save them to their *local* vector and ask for more work
for(;;) {
// Get work
err = 0;
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
// If received task is -1, function is terminated
if ( taskNo == -1 ) break;
// Check if there is enough memory to store the new coefficients, if not reallocate
// the current memory and expand it by MEM_PERC of the approximated size
if ( cor_cur_size < (coeff_count + rows) ) {
PROF(0, "\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank);
DEBUG("\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank);
// Use j as temporary again. Add two (or any other value) to avoid 0.
// (two is just a random value, you can put any value really...)
j = (fair_chunk * MEM_PERC) + 2;
cor_cur_size += (j * rows);
// Reallocate and check
cor = (double *)realloc(cor, sizeof(double) * cor_cur_size);
if ( cor == NULL ) {
ERR("**ERROR** : Memory re-allocation failed on worker process %d. Aborting.\n", rank);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
return -1;
}
}
// Compute the correlation coefficients
if(dataMatrixY != NULL) {
for (j=0; j < rows; j++) {
cor[coeff_count] = pearson_XY(dataMatrixX, dataMatrixY, j, taskNo, columns);
coeff_count++;
}
} else {
for (j=0; j < rows; j++) {
// Set main diagonal to 1
if ( j == taskNo ) {
cor[coeff_count] = 1.0;
coeff_count++;
continue;
}
cor[coeff_count] = pearson(dataMatrixX, taskNo, j, columns);
coeff_count++;
}
}
// The value of blocklens[] represents the number of coefficients on each
// row of the corellation array
blocklens[count] = rows;
// The value of indices[] represents the offset of each row in the data file
indices[count] = (taskNo * rows);
count++;
// Give the master the taskID
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
}
// There are two possibilities
// (a) everything went well and all workers finished ok
// (b) some processes finished ok but one or more of the remaining working workers failed
// To make sure all is well an all-reduce will be performed to sync all workers and guarantee success
// before moving on to write the output file
// This worker is OK
local_check = 0;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Check failed
if ( global_check != 0 ) {
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
return -1;
}
PROF(0, "\nPROF_stats (thread %3d) : Fair chunk of work : %d \t\t Allocated : %d \t\t Computed : %d\n",
rank, fair_chunk, cor_cur_size, coeff_count);
// If the distance_flag is set, then transform all correlation coefficients to distances
if ( distance_flag == 1 ) {
for(j=0; j < coeff_count; j++) {
cor[j] = 1 - cor[j];
}
}
// Create and commit the Indexed datatype *ONLY* if there are data available
if ( coeff_count != 0 ) {
MPI_Type_indexed(count, blocklens, indices, MPI_DOUBLE, &coeff_index_dt);
MPI_Type_commit(&coeff_index_dt);
}
// Open the file handler
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
// Create the file view
if ( coeff_count != 0 ) {
MPI_File_set_view(fh, 0, MPI_DOUBLE, coeff_index_dt, "native", MPI_INFO_NULL);
} else {
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
}
// Write data to disk
// TODO coeff_count cannot be greater than max int (for use in the MPI_File_write_all call).
// A better fix should be possible, for now throw error.
DEBUG("\ncoeff_count is %lld\n", coeff_count);
DEBUG("\INT_MAX is %d\n", INT_MAX);
if(coeff_count>INT_MAX)
{
ERR("**ERROR** : Could not run as the chunks of data are too large. Try running again with more MPI processes.\n");
// Free allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
// We have to receive a work assignment first and then terminate
// otherwise the master will deadlock trying to give work to this worker
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
return -1;
}
DEBUG("\nWriting %d to disk\n", coeff_count);
MPI_File_write_all(fh, &cor[0], coeff_count, MPI_DOUBLE, &stat);
if (coeff_count != 0 )
MPI_Type_free(&coeff_index_dt);
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
}
DEBUG("\nAbout to write to disk %d\n", rank);
MPI_File_sync( fh ) ; // Causes all previous writes to be transferred to the storage device
DEBUG("\nWritten to disk %d\n",rank);
// MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine.
DEBUG("\nAfter barrier \n", rank);
// Close file handler
MPI_File_close(&fh);
DEBUG("\nAfter file closed /n");
// MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine.
DEBUG("\nAbout to return from kernel /n");
return 0;
}
void parallel_readwrite(char *file_name, void *dump_buffer,
int type_of_file, int is_write, long long offset)
{
#if MPI && DO_PARALLEL_WRITE
MPI_File fh;
MPI_Status status;
MPI_Datatype mpi_elementary_type, mpi_file_type;
int file_open_error, file_write_error ;
int error_string_length;
char error_string[BUFSIZ];
MPI_Offset file_size;
int count;
void *mpi_buffer;
size_t mpi_buffer_size;
int mode;
MPI_Offset mpi_offset;
MPI_Barrier(MPI_COMM_WORLD);
if (is_write) {
mode = MPI_MODE_CREATE | MPI_MODE_WRONLY | MPI_MODE_APPEND;
}
else {
mode = MPI_MODE_RDONLY;
}
file_open_error = MPI_File_open(MPI_COMM_WORLD, file_name,
mode,
MPI_INFO_NULL, &fh);
if (file_open_error != MPI_SUCCESS) {
MPI_Error_string(file_open_error, error_string,
&error_string_length);
fprintf(stderr, "parallel_readwrite(): error opening file: %3d: %s\n", mpi_rank, error_string);
MPI_Abort(MPI_COMM_WORLD, file_open_error);
/* It is still OK to abort, because we have failed to
open the file. */
}
else {
// if (i_am_the_master)
// chmod(file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (offset < 0L) {
if(is_write) {
MPI_File_get_position(fh, &mpi_offset);
offset = mpi_offset;
}
else {
offset = 0L;
}
}
MPI_Barrier(MPI_COMM_WORLD);
//differentiate data type and buffers involved based on file type
if( DUMP_FILE == type_of_file ) {
mpi_elementary_type = MPI_DUMP_TYPE;
mpi_file_type = dump_file_type;
mpi_buffer = (void*)dump_buffer;
mpi_buffer_size = dump_buffer_size;
}
else if( GDUMP_FILE == type_of_file){
mpi_elementary_type = MPI_GDUMP_TYPE;
mpi_file_type = gdump_file_type;
mpi_buffer = (void*)gdump_buffer;
mpi_buffer_size = gdump_buffer_size;
}
else if( GDUMP2_FILE == type_of_file){
mpi_elementary_type = MPI_GDUMP2_TYPE;
mpi_file_type = gdump2_file_type;
mpi_buffer = (void*)gdump2_buffer;
mpi_buffer_size = gdump2_buffer_size;
}
else if( RDUMP_FILE == type_of_file){
mpi_elementary_type = MPI_RDUMP_TYPE;
mpi_file_type = rdump_file_type;
mpi_buffer = (void*)rdump_buffer;
mpi_buffer_size = rdump_buffer_size;
}
else if( FDUMP_FILE == type_of_file){
mpi_elementary_type = MPI_FDUMP_TYPE;
mpi_file_type = fdump_file_type;
mpi_buffer = (void*)fdump_buffer;
mpi_buffer_size = fdump_buffer_size;
}
else {
if(i_am_the_master)
fprintf(stderr, "Unknown file type %d\n", type_of_file);
MPI_File_close(&fh);
MPI_Finalize();
exit(2);
}
MPI_File_set_view(fh, offset, mpi_elementary_type, mpi_file_type, "native", MPI_INFO_NULL);
if (is_write) {
file_write_error =
MPI_File_write_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type,
&status);
}
else {
file_write_error =
MPI_File_read_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type,
&status);
}
if (file_write_error != MPI_SUCCESS) {
MPI_Error_string(file_write_error, error_string,
&error_string_length);
fprintf(stderr, "parallel_readwrite(): error %s file: %3d: %s\n",
(is_write)?("writing"):("reading"), mpi_rank, error_string);
MPI_File_close(&fh);
//if (i_am_the_master) MPI_File_delete(file_name, MPI_INFO_NULL);
MPI_Finalize();
exit(1);
}
// MPI_Get_count(&status, MPI_FLOAT, &count);
// MPI_File_get_size(fh, &file_size);
// if(1) {
// printf("%3d: wrote %d floats, expected to write %lld floats\n", mpi_rank, count, (long long int)dump_buffer_size);
// printf("%3d: file size is %lld bytes, header-related offset is %lld\n", mpi_rank, file_size, offset);
// }
MPI_File_close(&fh);
}
#endif
}
int main(int argc, char *argv[])
{
char *filename;
int i, len, nprocs, amode, err, nerrs = 0;
int blen[2], disp[2];
MPI_Datatype etype, filetype;
MPI_File fh;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (nprocs != 1) {
fprintf(stderr, "Run this program on 1 process\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
len = 8;
filename = (char *) malloc(len + 10);
strcpy(filename, "testfile");
} else {
argv++;
len = (int) strlen(*argv);
filename = (char *) malloc(len + 1);
strcpy(filename, *argv);
}
MPI_File_delete(filename, MPI_INFO_NULL);
amode = MPI_MODE_RDWR | MPI_MODE_CREATE;
err = MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh);
CHECK_ERROR(err, nerrs);
/* create etype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with overlaps */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create filetype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with overlaps */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_close(&fh);
CHECK_ERROR(err, nerrs);
/* open the file for read only */
amode = MPI_MODE_RDONLY;
err = MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh);
CHECK_ERROR(err, nerrs);
/* create etype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with overlaps (should be OK for read-only) */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_ERROR(err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create filetype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with overlaps (should be OK for read-only) */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_ERROR(err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_close(&fh);
CHECK_ERROR(err, nerrs);
if (nerrs == 0)
printf(" No Errors\n");
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char *argv[] ) {
double time1, time2;
time1 = MPI_Wtime();
int rank, processors;
int j; // number of iterations
int k; // number of iterations to perform before creating a checkpoint
int l; // number of random samples per grid point
int checkpoint_resume = 0; // 1 = resume from last checkpoint
int c; // used to hold a character
int i=0, row = 0, col = 0, pln = 0; // array iterators
char ***local_array;
char **local_array_2nd;
char *local_array_pointer;
char ***local_array_copy;
char **local_array_copy_2nd;
char *local_array_copy_pointer;
char ***temp, *temp_pointer;
int file_open_error;
int command_line_incomplete = 0;
int grid_size[3] = {0,0,0};
int proc_size[3] = {0,0,0};
int local_size[3] = {0,0,0};
int remainder_size[3] = {0,0,0};
int coords[3] = {0,0,0};
int start_indices[3] = {0,0,0};
int periods[3] = {0,0,0};
int mem_size[3] = {0,0,0};
MPI_Status status;
MPI_Datatype filetype, memtype;
MPI_File fh;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &processors);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// Interpret the command line arguments --------------------------------
if (rank == 0) {
if (argc < 6 || argc > 8) {
fputs("usage: x y z j k l r\n", stderr);
fputs("where: x,y,z = x, y and z dimensions\n", stderr);
fputs(" j = how many times the game of life is played\n", stderr);
fputs(" k = checkpoint every k iterations\n", stderr);
fputs(" l = number of random samples per grid point\n", stderr);
fputs(" r = resume from the last checkpoint\n", stderr);
fputs(INITIAL, stderr);
fputs(" must be present.\n", stderr);
fputs(CHECKPOINT, stderr);
fputs(" must be present if resuming from the last checkpoint.\n", stderr);
exit(EXIT_FAILURE);
}
}
j = (int) strtol(argv[4], NULL, 10);
k = (int) strtol(argv[5], NULL, 10);
l = (int) strtol(argv[6], NULL, 10);
if ( argc == 7 )
if ( argv[6][0] == 'r' )
checkpoint_resume = 1;
if (rank == 0)
printf("%d iterations \ncheckpoint every %d iterations \n%d samples per grid point \ncheckpoint resume = %d\n", j,k,l,checkpoint_resume);
grid_size[0] = (int) strtol(argv[1], NULL, 10);
grid_size[1] = (int) strtol(argv[2], NULL, 10);
grid_size[2] = (int) strtol(argv[3], NULL, 10);
if (rank==0) printf("grid_size: %d, %d, %d\n", grid_size[0], grid_size[1], grid_size[2]);
MPI_Dims_create(processors, 3, proc_size);
if (rank==0) printf("proc_size: %d, %d, %d\n", proc_size[0], proc_size[1], proc_size[2]);
local_size[0] = grid_size[0] / proc_size[0];
local_size[1] = grid_size[1] / proc_size[1];
local_size[2] = grid_size[2] / proc_size[2];
if (rank==0) printf("local_size: %d, %d, %d\n", local_size[0], local_size[1], local_size[2]);
remainder_size[0] = grid_size[0] % proc_size[0];
remainder_size[1] = grid_size[1] % proc_size[1];
remainder_size[2] = grid_size[2] % proc_size[2];
if (rank==0) printf("remainder_size: %d, %d, %d\n", remainder_size[0], remainder_size[1], remainder_size[2]);
if (remainder_size[0] != 0 || remainder_size[1] != 0 || remainder_size[2] != 0) {
fputs("remainder size != 0, check your dimensions", stderr);
MPI_Finalize();
exit(EXIT_FAILURE);
}
MPI_Comm comm;
MPI_Cart_create(MPI_COMM_WORLD, 3, proc_size, periods, 0, &comm);
MPI_Comm_rank(comm, &rank);
MPI_Cart_coords(comm, rank, 3, coords);
start_indices[0] = coords[0] * local_size[0];
start_indices[1] = coords[1] * local_size[1];
start_indices[2] = coords[2] * local_size[2];
/* printf("A coords R%d: (%d, %d, %d) (%d, %d, %d)\n", rank, coords[0], coords[1], coords[2], start_indices[0], start_indices[1], start_indices[2]);*/
fflush(stdout);
// create the file type ---------------------------------------------------
MPI_Type_create_subarray(3, grid_size, local_size, start_indices, MPI_ORDER_C, MPI_CHAR, &filetype);
MPI_Type_commit(&filetype);
// create a local memory type with ghost rows -----------------------------
mem_size[0] = local_size[0] + 2;
mem_size[1] = local_size[1] + 2;
mem_size[2] = local_size[2] + 2;
start_indices[0] = start_indices[1] = start_indices[2] = 1;
MPI_Type_create_subarray(3, mem_size, local_size, start_indices, MPI_ORDER_C, MPI_CHAR, &memtype);
MPI_Type_commit(&memtype);
// find my neighbors ------------------------------------------------------
int nxminus, nxplus, nyminus, nyplus, nzminus, nzplus, tag = 333, *neighbors;
// Neighbors Array: row- col- col+ row+ plane- plane+
neighbors = (int *) malloc(6 * sizeof(int));
for(i=0; i<6; i++)
neighbors[i] = rank;
MPI_Cart_shift(comm, 0, 1, &nxminus, &nxplus);
MPI_Cart_shift(comm, 1, 1, &nyminus, &nyplus);
MPI_Cart_shift(comm, 2, 1, &nzminus, &nzplus);
// printf(" %d sending south to %d receiving from %d \n",rank,nxplus,nxminus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nxplus, tag,
&(neighbors[0]), 1, MPI_INT, nxminus, tag, comm, &status);
// printf(" %d sending North to %d receiving from %d \n",rank,nxminus,nxplus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nxminus, tag,
&(neighbors[3]), 1, MPI_INT, nxplus, tag, comm, &status);
// printf(" %d sending East to %d receiving from %d \n",rank,nyplus,nyminus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nyplus, tag,
&neighbors[1], 1, MPI_INT, nyminus, tag, comm, &status);
// printf(" %d sending West to %d receiving from %d \n",rank,nyminus,nyplus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nyminus, tag,
&neighbors[2], 1, MPI_INT, nyplus, tag, comm, &status);
// printf(" %d sending backwards to %d receiving from %d \n",rank,nzplus,nzminus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nzplus, tag,
&(neighbors[4]), 1, MPI_INT, nzminus, tag, comm, &status);
// printf(" %d sending forward to %d receiving from %d \n",rank,nzminus,nzplus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nzminus, tag,
&(neighbors[5]), 1, MPI_INT, nzplus, tag, comm, &status);
/* printf("neighboors R%d : (row-) %d (col-) %d (col+) %d (row+) %d (plane-) %d (plane+) %d\n",rank,neighbors[0],neighbors[1],neighbors[2],neighbors[3],neighbors[4],neighbors[5]);*/
fflush(stdout);
//init_sprng(1,time(0),SPRNG_DEFAULT);
srand((unsigned int)time(NULL));
// Open the initial condition (checkpoint or not) ----------------------
if ( checkpoint_resume ) {
file_open_error =
MPI_File_open(MPI_COMM_WORLD, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh,0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
}
else {
file_open_error =
MPI_File_open(MPI_COMM_WORLD, INITIAL, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh,0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
}
if (file_open_error != MPI_SUCCESS) {
if (checkpoint_resume)
fputs(CHECKPOINT, stderr);
else
fputs(INITIAL, stderr);
fputs(" could not be opened.\n", stderr);
exit(EXIT_FAILURE);
}
// Allocate and Populate the local array ----------------------------------
local_array_copy_pointer = (char *) malloc(mem_size[0] * mem_size[1] * mem_size[2] * sizeof(char));
local_array_copy_2nd = (char **) malloc(mem_size[0] * mem_size[1] * sizeof(char*));
local_array_copy = (char ***) malloc(mem_size[0] * sizeof(char*));
for(i = 0; i < mem_size[0] * mem_size[1]; i++)
local_array_copy_2nd[i] = &local_array_copy_pointer[i * mem_size[2]];
for(i = 0; i < mem_size[0]; i++)
local_array_copy[i] = &local_array_copy_2nd[i * mem_size[1]];
local_array_pointer = (char *) malloc(mem_size[0] * mem_size[1] * mem_size[2] * sizeof(char));
local_array_2nd = (char **) malloc(mem_size[0] * mem_size[1] * sizeof(char*));
local_array = (char ***) malloc(mem_size[0] * sizeof(char*));
for(i = 0; i < mem_size[0] * mem_size[1]; i++)
local_array_2nd[i] = &local_array_pointer[i * mem_size[2]];
for(i = 0; i < mem_size[0]; i++)
local_array[i] = &local_array_2nd[i * mem_size[1]];
// if (rank==0) printf("Malloc complete\n");
for(row=0; row<mem_size[0]; row++) {
for(col=0; col<mem_size[1]; col++) {
for(pln=0; pln<mem_size[2]; pln++) {
local_array[row][col][pln] = local_array_copy[row][col][pln] = '0';
}
}
}
// if (rank==0) printf("Setup complete\n");
MPI_File_read_all(fh, local_array_pointer, 1, memtype, &status);
if (rank==0) printf("File Read\n");
// if (rank==0) {
// for(row=0; row<mem_size[0]; row++) {
// for(col=0; col<mem_size[1]; col++) {
// for(pln=0; pln<mem_size[2]; pln++) {
// printf("%c", local_array[row][col][pln]);
// }
// printf("\n");
// }
// printf("-----------------------\n");
// }
// }
MPI_File_close(&fh);
// Construct the plane data types
MPI_Datatype yzplane;
MPI_Type_vector(local_size[1], local_size[2], local_size[2]+2, MPI_CHAR, &yzplane);
MPI_Type_commit(&yzplane);
MPI_Datatype xzplane;
MPI_Type_vector(local_size[0], local_size[2], ((local_size[2]+2)*local_size[1])+((local_size[2]+2)*2), MPI_CHAR, &xzplane);
MPI_Type_commit(&xzplane);
// this type will also copy the corner x columns, can't skip blocks intermittently
// since we aren't worrying about the corner data, it's ok
MPI_Datatype xyplane;
MPI_Type_vector((local_size[0]*local_size[1])+((local_size[0]*2)-2), 1, local_size[2]+2, MPI_CHAR, &xyplane);
MPI_Type_commit(&xyplane);
MPI_Barrier(comm);
// start the iteration loop
int iterations;
int kCounter = k;
for (iterations = 0; iterations < j; iterations++) {
// send updated planes
// Neighbors Array:
// 0 1 2 3 4 5
// row- col- col+ row+ plane- plane+
// Note: corners are not handled
// send top yzplane
if (rank != neighbors[0]) MPI_Send(&local_array[1][1][1], 1, yzplane, neighbors[0], 0, comm);
// recv bottom yzplane
if (rank != neighbors[3]) MPI_Recv(&local_array[local_size[0]+1][1][1], 1, yzplane, neighbors[3], 0, comm, &status);
// send bottom yzplane
if (rank != neighbors[3]) MPI_Send(&local_array[local_size[0]][1][1], 1, yzplane, neighbors[3], 0, comm);
// recv top yzplane
if (rank != neighbors[0]) MPI_Recv(&local_array[0][1][1], 1, yzplane, neighbors[0], 0, comm, &status);
// send left xzplane
if (rank != neighbors[1]) MPI_Send(&local_array[1][1][1], 1, xzplane, neighbors[1], 0, comm);
// recv right xzplane
if (rank != neighbors[2]) MPI_Recv(&local_array[1][local_size[1]+1][1], 1, xzplane, neighbors[2], 0, comm, &status);
// send right xzplane
if (rank != neighbors[2]) MPI_Send(&local_array[1][local_size[1]][1], 1, xzplane, neighbors[2], 0, comm);
// recv left xzplane
if (rank != neighbors[1]) MPI_Recv(&local_array[1][0][1], 1, xzplane, neighbors[1], 0, comm, &status);
// send front xyplane
if (rank != neighbors[4]) MPI_Send(&local_array[1][1][1], 1, xyplane, neighbors[4], 0, comm);
// recv back xyplane
if (rank != neighbors[5]) MPI_Recv(&local_array[1][1][local_size[2]+1], 1, xyplane, neighbors[5], 0, comm, &status);
// send back xyplane
if (rank != neighbors[5]) MPI_Send(&local_array[1][1][local_size[2]], 1, xyplane, neighbors[5], 0, comm);
// recv front xyplane
if (rank != neighbors[4]) MPI_Recv(&local_array[1][1][0], 1, xyplane, neighbors[4], 0, comm, &status);
// if (rank==0) {
// for(row=0; row<mem_size[0]; row++) {
// for(col=0; col<mem_size[1]; col++) {
// for(pln=0; pln<mem_size[2]; pln++) {
// printf("%c", local_array[row][col][pln]);
// }
// printf("\n");
// }
// printf("-----------------------\n");
// }
// }
// run the game of life
// gameOfLife(local_array, local_array_copy, local_size[0], local_size[1], l, rank);
// swap the arrays
// temp1 = local_array;
// local_array = local_array_copy;
// local_array_copy = temp1;
//
// temp2 = local_array_pointer;
// local_array_pointer = local_array_copy_pointer;
// local_array_copy_pointer = temp2;
// check to see if this iteration needs a checkpoint
kCounter--;
if (kCounter == 0) {
kCounter = k;
// checkpoint code
MPI_File_open(MPI_COMM_WORLD, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
MPI_File_write_all(fh, local_array_pointer, 1, memtype, &status);
MPI_File_close(&fh);
if (rank == 0)
printf("Checkpoint made: Iteration %d\n", iterations+1);
} // end if kCounter == 0
} // end iteration loop
iterations--;
// all done! repeat the checkpoint process
MPI_File_open(MPI_COMM_WORLD, FINAL_RESULTS, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
MPI_File_write_all(fh, local_array_pointer, 1, memtype, &status);
MPI_File_close(&fh);
if (rank == 0)
printf("Final Results made: Iteration %d\n", iterations+1);
time2 = MPI_Wtime();
if (rank == 0)
printf("Elapsed Seconds: %f\n", time2-time1);fflush(stdout);
MPI_Finalize();
return EXIT_SUCCESS;
}
int main(int argc, char **argv)
{
int *writebuf, *readbuf, i, mynod, nprocs, len, err;
char *filename;
int errs=0, toterrs;
MPI_Datatype newtype;
MPI_File fh;
MPI_Status status;
MPI_Info info;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: coll_test -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
writebuf = (int *) malloc(BUFSIZE*sizeof(int));
readbuf = (int *) malloc(BUFSIZE*sizeof(int));
/* test atomicity of contiguous accesses */
/* initialize file to all zeros */
if (!mynod) {
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
MPI_File_close(&fh);
#if VERBOSE
fprintf(stderr, "\ntesting contiguous accesses\n");
#endif
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
/* set atomicity to true */
err = MPI_File_set_atomicity(fh, 1);
if (err != MPI_SUCCESS) {
fprintf(stderr, "Atomic mode not supported on this file system.\n");fflush(stderr);
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Barrier(MPI_COMM_WORLD);
/* process 0 writes and others concurrently read. In atomic mode,
the data read must be either all old values or all new values; nothing
in between. */
if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) { /* the rest must also be 0 */
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 0) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else if (readbuf[0] == 10) { /* the rest must also be 10 */
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 10) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else {
errs++;
fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
/* repeat the same test with a noncontiguous filetype */
MPI_Type_vector(BUFSIZE, 1, 2, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
MPI_Info_create(&info);
/* I am setting these info values for testing purposes only. It is
better to use the default values in practice. */
MPI_Info_set(info, "ind_rd_buffer_size", "1209");
MPI_Info_set(info, "ind_wr_buffer_size", "1107");
if (!mynod) {
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, info, &fh);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
MPI_File_close(&fh);
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous accesses\n");
#endif
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, info, &fh);
MPI_File_set_atomicity(fh, 1);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) {
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 0) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else if (readbuf[0] == 10) {
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 10) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else {
errs++;
fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (mynod == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
MPI_Type_free(&newtype);
MPI_Info_free(&info);
free(writebuf);
free(readbuf);
free(filename);
MPI_Finalize();
return 0;
}
void writetofile(int rank, int a)
{
#if defined(MPI) && defined(FILEIMAGE)
// dump all the tiles to 1 file using MPI-IO
int color = 0;
MPI_Comm tilecomm;
MPI_File fhtout;
if (owneroftile == rank) color = 1;
PRINTDEBUG("%3d: tileowner=%d, color=%d\n", rank, owneroftile, color);
MPI_Comm_split(MPI_COMM_WORLD, color, rank, &tilecomm);
if (color) {
int cartrank;
MPI_Comm cartcomm;
const MPI_Status status;
// init parameters for Cart_create
int gsizes[2] = {viewport.h, viewport.w*ELEMENTSPERPIXEL};
int psizes[2] = {viewport.tiley, viewport.tilex};
int lsizes[2] = {gsizes[0]/psizes[0], (gsizes[1])/psizes[1]};
int dims[2] = {psizes[0], psizes[1]};
int periods[2] = {1,1};
int coords[2];
PRINTDEBUG("%3d: gsizes=%d,%d, psizes=%d,%d, lsizes=%d,%d\n", rank, gsizes[0], gsizes[1], psizes[0], psizes[1],
lsizes[0], lsizes[1]);
// Create cartesian coord
MPI_Cart_create(tilecomm, 2, dims, periods, 0, &cartcomm);
MPI_Comm_rank(cartcomm, &cartrank);
MPI_Cart_coords(cartcomm, cartrank, 2, coords);
// check coords
PRINTDEBUG("%3d: coords=%d,%d\n", cartrank, coords[0], coords[1]);
/* global indices of first element of local array */
int startindex[2];
startindex[0] = coords[0] * lsizes[0];
startindex[1] = coords[1] * lsizes[1];
PRINTDEBUG("%3d: start=%d,%d\n", cartrank, startindex[0], startindex[1]);
int filetype;
filetype=0;
MPI_Type_create_subarray(2, gsizes, lsizes, startindex, MPI_ORDER_C, MPI_FLOAT, &filetype);
MPI_Type_commit(&filetype);
char imagefull[32];
sprintf(imagefull,"imagefull%d.raw", a);
PRINTDEBUG("%s\n", imagefull);
MPI_File_open(cartcomm, imagefull, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fhtout);
MPI_File_set_view(fhtout, 0, MPI_FLOAT, filetype, "native", MPI_INFO_NULL);
int localarraysize = lsizes[0] * lsizes[1];
PRINTDEBUG("%3d: size=%d\n", cartrank, localarraysize);
MPI_File_write(fhtout, tilebuffer, localarraysize, MPI_FLOAT, &status);
int count;
MPI_Get_count(&status, MPI_FLOAT, &count);
PRINTDEBUG("%3d: cnt=%d\n", cartrank, count);
MPI_File_close(&fhtout);
MPI_Comm_free(&cartcomm);
MPI_Type_free(&filetype);
// free(tilebuffer);
}
MPI_Comm_free(&tilecomm);
#endif
}
int run_test (test_param_t *test) {
ADIO_Offset st_offset, end_offset;
MPI_File fh;
int is_contig;
int ind_err = 0, exp_err = 0;
MPI_Datatype filetype;
MPI_Type_struct (test->type_count, test->type_blocklens,
test->type_indices, test->type_oldtypes, &filetype);
MPI_Type_commit (&filetype);
MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
MPI_File_seek (fh, test->offset, MPI_SEEK_SET);
ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE, ADIO_INDIVIDUAL,
test->offset, &st_offset, &end_offset);
ind_err = 0;
if (st_offset != test->correct_st_offset) {
printf ("Individual st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
ind_err = 1;
}
if (end_offset != test->correct_end_offset) {
printf ("Individual st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
ind_err = 1;
}
MPI_File_close (&fh);
if (ind_err)
printf ("Individual Calc FAILED\n");
MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
if (!is_contig)
MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
MPI_File_seek (fh, 0, MPI_SEEK_SET);
ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE,
ADIO_EXPLICIT_OFFSET, test->offset, &st_offset,
&end_offset);
exp_err = 0;
if (st_offset != test->correct_st_offset) {
printf ("Explicit st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
exp_err = 1;
}
if (end_offset != test->correct_end_offset) {
printf ("Explicit st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
exp_err = 1;
}
if (exp_err)
printf ("Explicit Calc FAILED\n");
MPI_File_close (&fh);
if (!is_contig)
MPI_Type_free (&filetype);
return (exp_err || ind_err);
}
int main(int argc, char **argv)
{
int *buf, i, mynod, nprocs, len, b[3];
int errs=0, toterrs;
MPI_Aint d[3];
MPI_File fh;
MPI_Status status;
char *filename;
MPI_Datatype typevec, newtype, t[3];
MPI_Info info;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
if (nprocs != 2) {
fprintf(stderr, "Run this program on two processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes (length first, then string) */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: noncontig -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE*sizeof(int));
MPI_Type_vector(SIZE/2, 1, 2, MPI_INT, &typevec);
/* create a struct type with explicitly set LB and UB; displacements
* of typevec are such that the types for the two processes won't
* overlap.
*/
b[0] = b[1] = b[2] = 1;
d[0] = 0;
d[1] = mynod*sizeof(int);
d[2] = SIZE*sizeof(int);
t[0] = MPI_LB;
t[1] = typevec;
t[2] = MPI_UB;
/* keep the struct, ditch the vector */
MPI_Type_struct(3, b, d, t, &newtype);
MPI_Type_commit(&newtype);
MPI_Type_free(&typevec);
MPI_Info_create(&info);
/* I am setting these info values for testing purposes only. It is
better to use the default values in practice. */
MPI_Info_set(info, "ind_rd_buffer_size", "1209");
MPI_Info_set(info, "ind_wr_buffer_size", "1107");
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using independent I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* set the file view for each process -- now writes go into the non-
* overlapping but interleaved region defined by the struct type up above
*/
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
/* fill our buffer with a pattern and write, using our type again */
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_write(fh, buf, 1, newtype, &status);
MPI_Barrier(MPI_COMM_WORLD);
/* fill the entire buffer with -1's. read back with type.
* note that the result of this read should be that every other value
* in the buffer is still -1, as defined by our type.
*/
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_read_at(fh, 0, buf, 1, newtype, &status);
/* check that all the values read are correct and also that we didn't
* overwrite any of the -1 values that we shouldn't have.
*/
for (i=0; i<SIZE; i++) {
if (!mynod) {
if ((i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
if (!(i%2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if ((i%2) && (buf[i] != i + mynod*SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
if (!(i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using independent I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* in this case we write to either the first half or the second half
* of the file space, so the regions are not interleaved. this is done
* by leaving the file view at its default.
*/
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_write_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
MPI_Barrier(MPI_COMM_WORLD);
/* same as before; fill buffer with -1's and then read; every other
* value should still be -1 after the read
*/
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_read_at(fh, mynod*(SIZE/2)*sizeof(int), buf, 1, newtype, &status);
/* verify that the buffer looks like it should */
for (i=0; i<SIZE; i++) {
if (!mynod) {
if ((i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
if (!(i%2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if ((i%2) && (buf[i] != i + mynod*SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
if (!(i%2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n",
mynod, i, buf[i]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using independent I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
info, &fh);
/* set the file view so that we have interleaved access again */
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
/* this time write a contiguous buffer */
for (i=0; i<SIZE; i++) buf[i] = i + mynod*SIZE;
MPI_File_write(fh, buf, SIZE, MPI_INT, &status);
MPI_Barrier(MPI_COMM_WORLD);
/* fill buffer with -1's; this time they will all be overwritten */
for (i=0; i<SIZE; i++) buf[i] = -1;
MPI_File_read_at(fh, 0, buf, SIZE, MPI_INT, &status);
for (i=0; i<SIZE; i++) {
if (!mynod) {
if (buf[i] != i) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i);
}
}
else {
if (buf[i] != i + mynod*SIZE) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod*SIZE);
}
}
}
MPI_File_close(&fh);
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (mynod == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
MPI_Type_free(&newtype);
MPI_Info_free(&info);
free(buf);
free(filename);
MPI_Finalize();
return 0;
}
/*
* Open a file through the MPIIO interface. Setup file view.
*/
static void *MPIIO_Open(char *testFileName, IOR_param_t * param)
{
int fd_mode = (int)0,
offsetFactor,
tasksPerFile,
transfersPerBlock = param->blockSize / param->transferSize;
struct fileTypeStruct {
int globalSizes[2], localSizes[2], startIndices[2];
} fileTypeStruct;
MPI_File *fd;
MPI_Comm comm;
MPI_Info mpiHints = MPI_INFO_NULL;
fd = (MPI_File *) malloc(sizeof(MPI_File));
if (fd == NULL)
ERR("malloc failed()");
*fd = 0;
/* set IOR file flags to MPIIO flags */
/* -- file open flags -- */
if (param->openFlags & IOR_RDONLY) {
fd_mode |= MPI_MODE_RDONLY;
}
if (param->openFlags & IOR_WRONLY) {
fd_mode |= MPI_MODE_WRONLY;
}
if (param->openFlags & IOR_RDWR) {
fd_mode |= MPI_MODE_RDWR;
}
if (param->openFlags & IOR_APPEND) {
fd_mode |= MPI_MODE_APPEND;
}
if (param->openFlags & IOR_CREAT) {
fd_mode |= MPI_MODE_CREATE;
}
if (param->openFlags & IOR_EXCL) {
fd_mode |= MPI_MODE_EXCL;
}
if (param->openFlags & IOR_TRUNC) {
fprintf(stdout, "File truncation not implemented in MPIIO\n");
}
if (param->openFlags & IOR_DIRECT) {
fprintf(stdout, "O_DIRECT not implemented in MPIIO\n");
}
/*
* MPI_MODE_UNIQUE_OPEN mode optimization eliminates the overhead of file
* locking. Only open a file in this mode when the file will not be con-
* currently opened elsewhere, either inside or outside the MPI environment.
*/
fd_mode |= MPI_MODE_UNIQUE_OPEN;
if (param->filePerProc) {
comm = MPI_COMM_SELF;
} else {
comm = testComm;
}
SetHints(&mpiHints, param->hintsFileName);
/*
* note that with MP_HINTS_FILTERED=no, all key/value pairs will
* be in the info object. The info object that is attached to
* the file during MPI_File_open() will only contain those pairs
* deemed valid by the implementation.
*/
/* show hints passed to file */
if (rank == 0 && param->showHints) {
fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd),
"cannot open file");
/* show hints actually attached to file handle */
if (rank == 0 && param->showHints) {
MPI_CHECK(MPI_File_get_info(*fd, &mpiHints),
"cannot get file info");
fprintf(stdout, "\nhints returned from opened file {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
/* preallocate space for file */
if (param->preallocate && param->open == WRITE) {
MPI_CHECK(MPI_File_preallocate(*fd,
(MPI_Offset) (param->segmentCount
*
param->blockSize *
param->numTasks)),
"cannot preallocate file");
}
/* create file view */
if (param->useFileView) {
/* create contiguous transfer datatype */
MPI_CHECK(MPI_Type_contiguous
(param->transferSize / sizeof(IOR_size_t),
MPI_LONG_LONG_INT, ¶m->transferType),
"cannot create contiguous datatype");
MPI_CHECK(MPI_Type_commit(¶m->transferType),
"cannot commit datatype");
if (param->filePerProc) {
offsetFactor = 0;
tasksPerFile = 1;
} else {
offsetFactor = (rank + rankOffset) % param->numTasks;
tasksPerFile = param->numTasks;
}
/*
* create file type using subarray
*/
fileTypeStruct.globalSizes[0] = 1;
fileTypeStruct.globalSizes[1] =
transfersPerBlock * tasksPerFile;
fileTypeStruct.localSizes[0] = 1;
fileTypeStruct.localSizes[1] = transfersPerBlock;
fileTypeStruct.startIndices[0] = 0;
fileTypeStruct.startIndices[1] =
transfersPerBlock * offsetFactor;
MPI_CHECK(MPI_Type_create_subarray
(2, fileTypeStruct.globalSizes,
fileTypeStruct.localSizes,
fileTypeStruct.startIndices, MPI_ORDER_C,
param->transferType, ¶m->fileType),
"cannot create subarray");
MPI_CHECK(MPI_Type_commit(¶m->fileType),
"cannot commit datatype");
MPI_CHECK(MPI_File_set_view(*fd, (MPI_Offset) 0,
param->transferType,
param->fileType, "native",
(MPI_Info) MPI_INFO_NULL),
"cannot set file view");
}
return ((void *)fd);
}
int main(int argc, char** argv) {
MPI_Init(&argc, &argv);
setup_globals();
/* Parse arguments. */
int SCALE = 16;
int edgefactor = 16; /* nedges / nvertices, i.e., 2*avg. degree */
int energytime = 0;
if (argc >= 2) SCALE = atoi(argv[1]);
if (argc >= 3) edgefactor = atoi(argv[2]);
if (argc >= 4) energytime = atoi(argv[3]);
if (argc <= 1 || argc >= 5 || SCALE == 0 || edgefactor == 0) {
if (rank == 0) {
fprintf(stderr, "Usage: %s SCALE edgefactor energytime\n SCALE = log_2(# vertices) [integer, required]\n edgefactor = (# edges) / (# vertices) = .5 * (average vertex degree) [integer, defaults to 16]\n energytime = time to run the energy benchmark (will print markers) [integer, seconds, defaults to 0]\n(Random number seed and Kronecker initiator are in main.c)\n", argv[0]);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
uint64_t seed1 = 2, seed2 = 3;
const char* filename = getenv("TMPFILE");
/* If filename is NULL, store data in memory */
tuple_graph tg;
tg.nglobaledges = (int64_t)(edgefactor) << SCALE;
int64_t nglobalverts = (int64_t)(1) << SCALE;
tg.data_in_file = (filename != NULL);
if (tg.data_in_file) {
MPI_File_set_errhandler(MPI_FILE_NULL, MPI_ERRORS_ARE_FATAL);
MPI_File_open(MPI_COMM_WORLD, (char*)filename, MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_DELETE_ON_CLOSE | MPI_MODE_UNIQUE_OPEN, MPI_INFO_NULL, &tg.edgefile);
MPI_File_set_size(tg.edgefile, tg.nglobaledges * sizeof(packed_edge));
MPI_File_set_view(tg.edgefile, 0, packed_edge_mpi_type, packed_edge_mpi_type, "native", MPI_INFO_NULL);
MPI_File_set_atomicity(tg.edgefile, 0);
}
/* Make the raw graph edges. */
/* Get roots for BFS runs, plus maximum vertex with non-zero degree (used by
* validator). */
int num_bfs_roots = 64;
int64_t* bfs_roots = (int64_t*)xmalloc(num_bfs_roots * sizeof(int64_t));
int64_t max_used_vertex = 0;
double make_graph_start = MPI_Wtime();
{
/* Spread the two 64-bit numbers into five nonzero values in the correct
* range. */
uint_fast32_t seed[5];
make_mrg_seed(seed1, seed2, seed);
/* As the graph is being generated, also keep a bitmap of vertices with
* incident edges. We keep a grid of processes, each row of which has a
* separate copy of the bitmap (distributed among the processes in the
* row), and then do an allreduce at the end. This scheme is used to avoid
* non-local communication and reading the file separately just to find BFS
* roots. */
MPI_Offset nchunks_in_file = (tg.nglobaledges + FILE_CHUNKSIZE - 1) / FILE_CHUNKSIZE;
int64_t bitmap_size_in_bytes = int64_min(BITMAPSIZE, (nglobalverts + CHAR_BIT - 1) / CHAR_BIT);
if (bitmap_size_in_bytes * size * CHAR_BIT < nglobalverts) {
bitmap_size_in_bytes = (nglobalverts + size * CHAR_BIT - 1) / (size * CHAR_BIT);
}
int ranks_per_row = ((nglobalverts + CHAR_BIT - 1) / CHAR_BIT + bitmap_size_in_bytes - 1) / bitmap_size_in_bytes;
int nrows = size / ranks_per_row;
int my_row = -1, my_col = -1;
unsigned char* restrict has_edge = NULL;
MPI_Comm cart_comm;
{
int dims[2] = {size / ranks_per_row, ranks_per_row};
int periods[2] = {0, 0};
MPI_Cart_create(MPI_COMM_WORLD, 2, dims, periods, 1, &cart_comm);
}
int in_generating_rectangle = 0;
if (cart_comm != MPI_COMM_NULL) {
in_generating_rectangle = 1;
{
int dims[2], periods[2], coords[2];
MPI_Cart_get(cart_comm, 2, dims, periods, coords);
my_row = coords[0];
my_col = coords[1];
}
MPI_Comm this_col;
MPI_Comm_split(cart_comm, my_col, my_row, &this_col);
MPI_Comm_free(&cart_comm);
has_edge = (unsigned char*)xMPI_Alloc_mem(bitmap_size_in_bytes);
memset(has_edge, 0, bitmap_size_in_bytes);
/* Every rank in a given row creates the same vertices (for updating the
* bitmap); only one writes them to the file (or final memory buffer). */
packed_edge* buf = (packed_edge*)xmalloc(FILE_CHUNKSIZE * sizeof(packed_edge));
MPI_Offset block_limit = (nchunks_in_file + nrows - 1) / nrows;
/* fprintf(stderr, "%d: nchunks_in_file = %" PRId64 ", block_limit = %" PRId64 " in grid of %d rows, %d cols\n", rank, (int64_t)nchunks_in_file, (int64_t)block_limit, nrows, ranks_per_row); */
if (tg.data_in_file) {
tg.edgememory_size = 0;
tg.edgememory = NULL;
} else {
int my_pos = my_row + my_col * nrows;
int last_pos = (tg.nglobaledges % ((int64_t)FILE_CHUNKSIZE * nrows * ranks_per_row) != 0) ?
(tg.nglobaledges / FILE_CHUNKSIZE) % (nrows * ranks_per_row) :
-1;
int64_t edges_left = tg.nglobaledges % FILE_CHUNKSIZE;
int64_t nedges = FILE_CHUNKSIZE * (tg.nglobaledges / ((int64_t)FILE_CHUNKSIZE * nrows * ranks_per_row)) +
FILE_CHUNKSIZE * (my_pos < (tg.nglobaledges / FILE_CHUNKSIZE) % (nrows * ranks_per_row)) +
(my_pos == last_pos ? edges_left : 0);
/* fprintf(stderr, "%d: nedges = %" PRId64 " of %" PRId64 "\n", rank, (int64_t)nedges, (int64_t)tg.nglobaledges); */
tg.edgememory_size = nedges;
tg.edgememory = (packed_edge*)xmalloc(nedges * sizeof(packed_edge));
}
MPI_Offset block_idx;
for (block_idx = 0; block_idx < block_limit; ++block_idx) {
/* fprintf(stderr, "%d: On block %d of %d\n", rank, (int)block_idx, (int)block_limit); */
MPI_Offset start_edge_index = int64_min(FILE_CHUNKSIZE * (block_idx * nrows + my_row), tg.nglobaledges);
MPI_Offset edge_count = int64_min(tg.nglobaledges - start_edge_index, FILE_CHUNKSIZE);
packed_edge* actual_buf = (!tg.data_in_file && block_idx % ranks_per_row == my_col) ?
tg.edgememory + FILE_CHUNKSIZE * (block_idx / ranks_per_row) :
buf;
/* fprintf(stderr, "%d: My range is [%" PRId64 ", %" PRId64 ") %swriting into index %" PRId64 "\n", rank, (int64_t)start_edge_index, (int64_t)(start_edge_index + edge_count), (my_col == (block_idx % ranks_per_row)) ? "" : "not ", (int64_t)(FILE_CHUNKSIZE * (block_idx / ranks_per_row))); */
if (!tg.data_in_file && block_idx % ranks_per_row == my_col) {
assert (FILE_CHUNKSIZE * (block_idx / ranks_per_row) + edge_count <= tg.edgememory_size);
}
generate_kronecker_range(seed, SCALE, start_edge_index, start_edge_index + edge_count, actual_buf);
if (tg.data_in_file && my_col == (block_idx % ranks_per_row)) { /* Try to spread writes among ranks */
MPI_File_write_at(tg.edgefile, start_edge_index, actual_buf, edge_count, packed_edge_mpi_type, MPI_STATUS_IGNORE);
}
ptrdiff_t i;
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (i = 0; i < edge_count; ++i) {
int64_t src = get_v0_from_edge(&actual_buf[i]);
int64_t tgt = get_v1_from_edge(&actual_buf[i]);
if (src == tgt) continue;
if (src / bitmap_size_in_bytes / CHAR_BIT == my_col) {
#ifdef _OPENMP
#pragma omp atomic
#endif
has_edge[(src / CHAR_BIT) % bitmap_size_in_bytes] |= (1 << (src % CHAR_BIT));
}
if (tgt / bitmap_size_in_bytes / CHAR_BIT == my_col) {
#ifdef _OPENMP
#pragma omp atomic
#endif
has_edge[(tgt / CHAR_BIT) % bitmap_size_in_bytes] |= (1 << (tgt % CHAR_BIT));
}
}
}
free(buf);
#if 0
/* The allreduce for each root acts like we did this: */
MPI_Allreduce(MPI_IN_PLACE, has_edge, bitmap_size_in_bytes, MPI_UNSIGNED_CHAR, MPI_BOR, this_col);
#endif
MPI_Comm_free(&this_col);
} else {
tg.edgememory = NULL;
tg.edgememory_size = 0;
}
MPI_Allreduce(&tg.edgememory_size, &tg.max_edgememory_size, 1, MPI_INT64_T, MPI_MAX, MPI_COMM_WORLD);
/* Find roots and max used vertex */
{
uint64_t counter = 0;
int bfs_root_idx;
for (bfs_root_idx = 0; bfs_root_idx < num_bfs_roots; ++bfs_root_idx) {
int64_t root;
while (1) {
double d[2];
make_random_numbers(2, seed1, seed2, counter, d);
root = (int64_t)((d[0] + d[1]) * nglobalverts) % nglobalverts;
counter += 2;
if (counter > 2 * nglobalverts) break;
int is_duplicate = 0;
int i;
for (i = 0; i < bfs_root_idx; ++i) {
if (root == bfs_roots[i]) {
is_duplicate = 1;
break;
}
}
if (is_duplicate) continue; /* Everyone takes the same path here */
int root_ok = 0;
if (in_generating_rectangle && (root / CHAR_BIT / bitmap_size_in_bytes) == my_col) {
root_ok = (has_edge[(root / CHAR_BIT) % bitmap_size_in_bytes] & (1 << (root % CHAR_BIT))) != 0;
}
MPI_Allreduce(MPI_IN_PLACE, &root_ok, 1, MPI_INT, MPI_LOR, MPI_COMM_WORLD);
if (root_ok) break;
}
bfs_roots[bfs_root_idx] = root;
}
num_bfs_roots = bfs_root_idx;
/* Find maximum non-zero-degree vertex. */
{
int64_t i;
max_used_vertex = 0;
if (in_generating_rectangle) {
for (i = bitmap_size_in_bytes * CHAR_BIT; i > 0; --i) {
if (i > nglobalverts) continue;
if (has_edge[(i - 1) / CHAR_BIT] & (1 << ((i - 1) % CHAR_BIT))) {
max_used_vertex = (i - 1) + my_col * CHAR_BIT * bitmap_size_in_bytes;
break;
}
}
}
MPI_Allreduce(MPI_IN_PLACE, &max_used_vertex, 1, MPI_INT64_T, MPI_MAX, MPI_COMM_WORLD);
}
}
if (in_generating_rectangle) {
MPI_Free_mem(has_edge);
}
if (tg.data_in_file) {
MPI_File_sync(tg.edgefile);
}
}
double make_graph_stop = MPI_Wtime();
double make_graph_time = make_graph_stop - make_graph_start;
if (rank == 0) { /* Not an official part of the results */
fprintf(stderr, "graph_generation: %f s\n", make_graph_time);
}
/* Make user's graph data structure. */
double data_struct_start = MPI_Wtime();
make_graph_data_structure(&tg);
double data_struct_stop = MPI_Wtime();
double data_struct_time = data_struct_stop - data_struct_start;
if (rank == 0) { /* Not an official part of the results */
fprintf(stderr, "construction_time: %f s\n", data_struct_time);
}
/* Number of edges visited in each BFS; a double so get_statistics can be
* used directly. */
double* edge_counts = (double*)xmalloc(num_bfs_roots * sizeof(double));
/* Run BFS. */
int validation_passed = 1;
double* bfs_times = (double*)xmalloc(num_bfs_roots * sizeof(double));
double* validate_times = (double*)xmalloc(num_bfs_roots * sizeof(double));
uint64_t nlocalverts = get_nlocalverts_for_pred();
int64_t* pred = (int64_t*)xMPI_Alloc_mem(nlocalverts * sizeof(int64_t));
/* energy measurement loop -- the following C block enables running
* the benchmark for the specified time to conduct reasonable power
* measurements for low-resolution devices. The code will print a
* banner to stderr that enables to calibrate the power measurement.
* The loop is run until the specified time expires. Make sure the
* input is large enough to avoid caching!
* (c) Torsten Hoefler, ETH Zurich, direct questions to [email protected] */
{
double time_left=(double)energytime; /* energy measurement time left */
int iters = 0;
if(energytime > 0 && rank == 0) {
time_t timestamp;
time(×tamp);
fprintf(stderr, "Energy Benchmark Loop Starting on %s", ctime(×tamp));
}
while(time_left > 0) {
int bfs_root_idx;
time_left += MPI_Wtime();
for (bfs_root_idx = 0; bfs_root_idx < num_bfs_roots; ++bfs_root_idx) {
int64_t root = bfs_roots[bfs_root_idx];
/* Clear the pred array. */
memset(pred, 0, nlocalverts * sizeof(int64_t));
run_bfs(root, &pred[0]);
}
time_left -= MPI_Wtime();
MPI_Bcast(&time_left, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
iters++;
}
if(energytime > 0 && rank == 0) {
time_t timestamp;
time(×tamp);
fprintf(stderr, "Energy Benchmark Loop Finishing (%i iterations, %.6f per iteration) on %s", iters, (energytime-time_left)/num_bfs_roots/iters, ctime(×tamp));
}
}
/* end of modified code for power measurement */
int bfs_root_idx;
for (bfs_root_idx = 0; bfs_root_idx < num_bfs_roots; ++bfs_root_idx) {
int64_t root = bfs_roots[bfs_root_idx];
if (rank == 0) fprintf(stderr, "Running BFS %d\n", bfs_root_idx);
/* Clear the pred array. */
memset(pred, 0, nlocalverts * sizeof(int64_t));
/* Do the actual BFS. */
double bfs_start = MPI_Wtime();
run_bfs(root, &pred[0]);
double bfs_stop = MPI_Wtime();
bfs_times[bfs_root_idx] = bfs_stop - bfs_start;
if (rank == 0) fprintf(stderr, "Time for BFS %d is %f\n", bfs_root_idx, bfs_times[bfs_root_idx]);
/* Validate result. */
if (rank == 0) fprintf(stderr, "Validating BFS %d\n", bfs_root_idx);
double validate_start = MPI_Wtime();
int64_t edge_visit_count;
int validation_passed_one = validate_bfs_result(&tg, max_used_vertex + 1, nlocalverts, root, pred, &edge_visit_count);
double validate_stop = MPI_Wtime();
validate_times[bfs_root_idx] = validate_stop - validate_start;
if (rank == 0) fprintf(stderr, "Validate time for BFS %d is %f\n", bfs_root_idx, validate_times[bfs_root_idx]);
edge_counts[bfs_root_idx] = (double)edge_visit_count;
if (rank == 0) fprintf(stderr, "TEPS for BFS %d is %g\n", bfs_root_idx, edge_visit_count / bfs_times[bfs_root_idx]);
if (!validation_passed_one) {
validation_passed = 0;
if (rank == 0) fprintf(stderr, "Validation failed for this BFS root; skipping rest.\n");
break;
}
}
MPI_Free_mem(pred);
free(bfs_roots);
free_graph_data_structure();
if (tg.data_in_file) {
MPI_File_close(&tg.edgefile);
} else {
free(tg.edgememory); tg.edgememory = NULL;
}
/* Print results. */
if (rank == 0) {
if (!validation_passed) {
fprintf(stdout, "No results printed for invalid run.\n");
} else {
int i;
fprintf(stdout, "SCALE: %d\n", SCALE);
fprintf(stdout, "edgefactor: %d\n", edgefactor);
fprintf(stdout, "NBFS: %d\n", num_bfs_roots);
fprintf(stdout, "graph_generation: %g\n", make_graph_time);
fprintf(stdout, "num_mpi_processes: %d\n", size);
fprintf(stdout, "construction_time: %g\n", data_struct_time);
double stats[s_LAST];
get_statistics(bfs_times, num_bfs_roots, stats);
fprintf(stdout, "min_time: %g\n", stats[s_minimum]);
fprintf(stdout, "firstquartile_time: %g\n", stats[s_firstquartile]);
fprintf(stdout, "median_time: %g\n", stats[s_median]);
fprintf(stdout, "thirdquartile_time: %g\n", stats[s_thirdquartile]);
fprintf(stdout, "max_time: %g\n", stats[s_maximum]);
fprintf(stdout, "mean_time: %g\n", stats[s_mean]);
fprintf(stdout, "stddev_time: %g\n", stats[s_std]);
get_statistics(edge_counts, num_bfs_roots, stats);
fprintf(stdout, "min_nedge: %.11g\n", stats[s_minimum]);
fprintf(stdout, "firstquartile_nedge: %.11g\n", stats[s_firstquartile]);
fprintf(stdout, "median_nedge: %.11g\n", stats[s_median]);
fprintf(stdout, "thirdquartile_nedge: %.11g\n", stats[s_thirdquartile]);
fprintf(stdout, "max_nedge: %.11g\n", stats[s_maximum]);
fprintf(stdout, "mean_nedge: %.11g\n", stats[s_mean]);
fprintf(stdout, "stddev_nedge: %.11g\n", stats[s_std]);
double* secs_per_edge = (double*)xmalloc(num_bfs_roots * sizeof(double));
for (i = 0; i < num_bfs_roots; ++i) secs_per_edge[i] = bfs_times[i] / edge_counts[i];
get_statistics(secs_per_edge, num_bfs_roots, stats);
fprintf(stdout, "min_TEPS: %g\n", 1. / stats[s_maximum]);
fprintf(stdout, "firstquartile_TEPS: %g\n", 1. / stats[s_thirdquartile]);
fprintf(stdout, "median_TEPS: %g\n", 1. / stats[s_median]);
fprintf(stdout, "thirdquartile_TEPS: %g\n", 1. / stats[s_firstquartile]);
fprintf(stdout, "max_TEPS: %g\n", 1. / stats[s_minimum]);
fprintf(stdout, "harmonic_mean_TEPS: %g\n", 1. / stats[s_mean]);
/* Formula from:
* Title: The Standard Errors of the Geometric and Harmonic Means and
* Their Application to Index Numbers
* Author(s): Nilan Norris
* Source: The Annals of Mathematical Statistics, Vol. 11, No. 4 (Dec., 1940), pp. 445-448
* Publisher(s): Institute of Mathematical Statistics
* Stable URL: http://www.jstor.org/stable/2235723
* (same source as in specification). */
fprintf(stdout, "harmonic_stddev_TEPS: %g\n", stats[s_std] / (stats[s_mean] * stats[s_mean] * sqrt(num_bfs_roots - 1)));
free(secs_per_edge); secs_per_edge = NULL;
free(edge_counts); edge_counts = NULL;
get_statistics(validate_times, num_bfs_roots, stats);
fprintf(stdout, "min_validate: %g\n", stats[s_minimum]);
fprintf(stdout, "firstquartile_validate: %g\n", stats[s_firstquartile]);
fprintf(stdout, "median_validate: %g\n", stats[s_median]);
fprintf(stdout, "thirdquartile_validate: %g\n", stats[s_thirdquartile]);
fprintf(stdout, "max_validate: %g\n", stats[s_maximum]);
fprintf(stdout, "mean_validate: %g\n", stats[s_mean]);
fprintf(stdout, "stddev_validate: %g\n", stats[s_std]);
#if 0
for (i = 0; i < num_bfs_roots; ++i) {
fprintf(stdout, "Run %3d: %g s, validation %g s\n", i + 1, bfs_times[i], validate_times[i]);
}
#endif
}
}
free(bfs_times);
free(validate_times);
cleanup_globals();
MPI_Finalize();
return 0;
}
static int test_mpio_derived_dtype(char *filename) {
MPI_File fh;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
int i;
int nerrors = 0; /* number of errors */
MPI_Datatype etype,filetype;
MPI_Datatype adv_filetype,bas_filetype[2];
MPI_Datatype etypenew, filetypenew;
MPI_Offset disp;
MPI_Status Status;
MPI_Aint adv_disp[2];
MPI_Aint offsets[1];
int blocklens[1],adv_blocklens[2];
int count,outcount;
int retcode;
int mpi_rank,mpi_size;
char buf[3],outbuf[3] = {0};
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
retcode = 0;
for(i=0;i<3;i++)
buf[i] = i+1;
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
disp = 0;
etype = MPI_BYTE;
count = 1;
blocklens[0] = 1;
offsets[0] = 0;
if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
count = 1;
blocklens[0]=1;
offsets[0] = 1;
if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetypenew))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetypenew))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
outcount = 2;
adv_blocklens[0] = 1;
adv_blocklens[1] = 1;
adv_disp[0] = 0;
adv_disp[1] = 1;
bas_filetype[0] = filetype;
bas_filetype[1] = filetypenew;
if((mpi_err= MPI_Type_struct(outcount,adv_blocklens,adv_disp,bas_filetype,&adv_filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_struct failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&adv_filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_set_view(fh,disp,etype,adv_filetype,"native",MPI_INFO_NULL))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_write(fh,buf,3,MPI_BYTE,&Status))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_write failed (%s)\n", mpi_err_str);
return 1;
;
}
if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_open(MPI_COMM_WORLD,filename,MPI_MODE_RDONLY,MPI_INFO_NULL,&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_set_view(fh,0,MPI_BYTE,MPI_BYTE,"native",MPI_INFO_NULL))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_read(fh,outbuf,3,MPI_BYTE,&Status))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_read failed (%s)\n", mpi_err_str);
return 1;
}
if(outbuf[2]==2) {
retcode = 0;
}
else {
/* if(mpi_rank == 0) {
printf("complicated derived datatype is NOT working at this platform\n");
printf("go back to hdf5/config and find the corresponding\n");
printf("configure-specific file and change ?????\n");
}
*/
retcode = -1;
}
if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed (%s)\n", mpi_err_str);
return 1;
}
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
#ifdef H5_MPI_COMPLEX_DERIVED_DATATYPE_WORKS
if(retcode == -1) {
if(mpi_rank == 0) {
printf("Complicated derived datatype is NOT working at this platform\n");
printf("Go back to hdf5/config and find the corresponding\n");
printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n");
printf("hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype-works='no'}\n");
printf(" at the end of the file.\n");
printf(" Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
#else
if(retcode == 0) {
if(mpi_rank == 0) {
printf(" This is NOT an error, What it really says is\n");
printf("Complicated derived datatype is WORKING at this platform\n");
printf(" Go back to hdf5/config and find the corresponding \n");
printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n");
printf("hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype-works='no'}\n");
printf(" at the end of the file.\n");
printf("Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
if(retcode == -1) retcode = 0;
#endif
return retcode;
}
int
main (int argc, char **argv)
{
MPI_Request request;
MPI_File fh;
MPI_Datatype ftype;
MPI_Offset offset;
MPI_Status status;
int rank, wsize, fsize, i;
char file_name[128];
int buf[BUF_SIZE * TEST_OPS];
int count;
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &wsize);
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
strcpy (file_name, argv[0]);
strcat (file_name, ".tmp");
MPI_File_open (MPI_COMM_WORLD, file_name, MPI_MODE_RDWR | MPI_MODE_CREATE,
MPI_INFO_NULL, &fh);
fsize = wsize * BUF_SIZE * TEST_OPS;
MPI_File_preallocate (fh, fsize);
memset (buf, 0, BUF_SIZE * TEST_OPS);
offset = 0;
count = BLOCK_SIZE;
for (i = 0; i < TEST_OPS; i++)
{
offset = i * BLOCK_SIZE + (rank * BLOCK_SIZE * TEST_OPS);
MPI_File_seek (fh, offset, MPI_SEEK_SET);
MPI_File_write (fh, buf, count, MPI_INT, &status);
MPI_File_seek (fh, offset, MPI_SEEK_SET);
MPI_File_read (fh, buf, count, MPI_INT, &status);
}
for (i = 0; i < TEST_OPS; i++)
{
offset = i * BLOCK_SIZE + (rank * BLOCK_SIZE * TEST_OPS);
MPI_File_write_at (fh, offset, buf, count, MPI_INT, &status);
MPI_File_read_at (fh, offset, buf, count, MPI_INT, &status);
}
MPI_Type_vector (fsize / BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE * wsize,
MPI_INT, &ftype);
MPI_Type_commit (&ftype);
offset = rank * BLOCK_SIZE * TEST_OPS;
count = BLOCK_SIZE * TEST_OPS;
MPI_File_set_view (fh, offset, MPI_INT, ftype, "native", MPI_INFO_NULL);
MPI_File_write_all (fh, buf, count, MPI_INT, &status);
MPI_File_read_all (fh, buf, count, MPI_INT, &status);
MPI_File_close (&fh);
MPI_Finalize ();
}
static int
test_mpio_special_collective(char *filename)
{
int mpi_size, mpi_rank;
MPI_File fh;
MPI_Datatype etype,buftype,filetype;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
char writedata[2];
char *buf;
int i;
int count,bufcount;
int blocklens[2];
MPI_Aint offsets[2];
MPI_Offset mpi_off;
MPI_Status mpi_stat;
int retcode;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
retcode = 0;
/* create MPI data type */
etype = MPI_BYTE;
if(mpi_rank == 0 || mpi_rank == 1) {
count = DIMSIZE;
bufcount = 1;
}
else {
count = 0;
bufcount = 0;
}
blocklens[0] = count;
offsets[0] = mpi_rank*count;
blocklens[1] = count;
offsets[1] = (mpi_size+mpi_rank)*count;
if(count !=0) {
if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&buftype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&buftype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
}
else {
filetype = MPI_BYTE;
buftype = MPI_BYTE;
}
/* Open a file */
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE ,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
/* each process writes some data */
for (i=0; i < 2*DIMSIZE; i++)
writedata[i] = mpi_rank*DIMSIZE + i;
mpi_off = 0;
if((mpi_err = MPI_File_set_view(fh, mpi_off, MPI_BYTE, filetype, "native", MPI_INFO_NULL))
!= MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
buf = writedata;
if ((mpi_err = MPI_File_write_at_all(fh, mpi_off, buf, bufcount, buftype,
&mpi_stat))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, bufcount, mpi_err_str);
return 1;
};
if ((mpi_err = MPI_File_close(&fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed. \n");
return 1;
};
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
#ifdef H5_MPI_SPECIAL_COLLECTIVE_IO_WORKS
if(retcode != 0) {
if(mpi_rank == 0) {
printf("special collective IO is NOT working at this platform\n");
printf("Go back to hdf5/config and find the corresponding\n");
printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n");
printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n");
printf(" at the end of the file.\n");
printf(" Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
#else
if(retcode == 0) {
if(mpi_rank == 0) {
printf(" This is NOT an error, What it really says is\n");
printf("special collective IO is WORKING at this platform\n");
printf(" Go back to hdf5/config and find the corresponding \n");
printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n");
printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n");
printf(" at the end of the file.\n");
printf("Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
#endif
return retcode;
}
int main(int argc, char **argv)
{
int *buf, i, mynod, nprocs, len, b[3];
int errs = 0, toterrs;
MPI_Aint d[3];
MPI_File fh;
MPI_Status status;
char *filename;
MPI_Datatype typevec, newtype, t[3];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
if (nprocs != 2) {
fprintf(stderr, "Run this program on two processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: noncontig_coll -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len + 1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len + 1);
MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE * sizeof(int));
MPI_Type_vector(SIZE / 2, 1, 2, MPI_INT, &typevec);
b[0] = b[1] = b[2] = 1;
d[0] = 0;
d[1] = mynod * sizeof(int);
d[2] = SIZE * sizeof(int);
t[0] = MPI_LB;
t[1] = typevec;
t[2] = MPI_UB;
MPI_Type_struct(3, b, d, t, &newtype);
MPI_Type_commit(&newtype);
MPI_Type_free(&typevec);
if (!mynod) {
#if VERBOSE
fprintf(stderr,
"\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL));
for (i = 0; i < SIZE; i++)
buf[i] = i + mynod * SIZE;
MPI_CHECK(MPI_File_write_all(fh, buf, 1, newtype, &status));
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < SIZE; i++)
buf[i] = -1;
MPI_CHECK(MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status));
for (i = 0; i < SIZE; i++) {
if (!mynod) {
if ((i % 2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]);
}
if (!(i % 2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i);
}
} else {
if ((i % 2) && (buf[i] != i + mynod * SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod * SIZE);
}
if (!(i % 2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]);
}
}
}
MPI_CHECK(MPI_File_close(&fh));
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr,
"\ntesting noncontiguous in memory, contiguous in file using collective I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
for (i = 0; i < SIZE; i++)
buf[i] = i + mynod * SIZE;
MPI_CHECK(MPI_File_write_at_all(fh, mynod * (SIZE / 2) * sizeof(int),
buf, 1, newtype, &status));
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < SIZE; i++)
buf[i] = -1;
MPI_CHECK(MPI_File_read_at_all(fh, mynod * (SIZE / 2) * sizeof(int), buf, 1, newtype, &status));
for (i = 0; i < SIZE; i++) {
if (!mynod) {
if ((i % 2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]);
}
if (!(i % 2) && (buf[i] != i)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i);
}
} else {
if ((i % 2) && (buf[i] != i + mynod * SIZE)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod * SIZE);
}
if (!(i % 2) && (buf[i] != -1)) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]);
}
}
}
MPI_CHECK(MPI_File_close(&fh));
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
#if VERBOSE
fprintf(stderr,
"\ntesting contiguous in memory, noncontiguous in file using collective I/O\n");
#endif
MPI_File_delete(filename, MPI_INFO_NULL);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL));
for (i = 0; i < SIZE; i++)
buf[i] = i + mynod * SIZE;
MPI_CHECK(MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status));
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < SIZE; i++)
buf[i] = -1;
MPI_CHECK(MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status));
for (i = 0; i < SIZE; i++) {
if (!mynod) {
if (buf[i] != i) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i);
}
} else {
if (buf[i] != i + mynod * SIZE) {
errs++;
fprintf(stderr, "Process %d: buf %d is %d, should be %d\n",
mynod, i, buf[i], i + mynod * SIZE);
}
}
}
MPI_CHECK(MPI_File_close(&fh));
MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (mynod == 0) {
if (toterrs > 0) {
fprintf(stderr, "Found %d errors\n", toterrs);
} else {
fprintf(stdout, " No Errors\n");
}
}
MPI_Type_free(&newtype);
free(buf);
free(filename);
MPI_Finalize();
return 0;
}
/**
* \brief Measures the time to write once to a file.
*
* Only one process is active. It writes once to a file.
*
* Remark:<br>
* With the <tt>O_DIRECT</tt> flag set, cache effects are minimized, because I/O
* is done directly to/from user space buffers. The operation system's page
* cache is bypassed. Under Linux 2.6 alignment to 512-byte boundaries is
* required for buffer and file offset. Thus the following parameters should be
* set in a SKaMPI input file:
* - <tt>set_send_buffert_alignment (512)</tt>
* - <tt>set_recv_buffert_alignment (512)</tt>
* - <tt>switch_buffer_cycling_off ()</tt><br>
*
* <tt>O_DIRECT</tt> is only relevant if the POSIX-API is used for I/O.
*
* For more information please refer to the <tt>open ()</tt> man pages.
*
* \param[in] size size of memory buffer, i.e. number of <tt>MPI_BYTE</tt>s
* \param[in] api POSIX-API or MPI-API for I/O accesses
* \param[in] create_flag write into existing file (FALSE) or create it (TRUE)
* \param[in] directio_flag open file with <tt>O_DIRECT</tt> flag to minimize
* cache effects
*
* \return measured time
*/
double measure_MPI_IO_write_file_once (int size, char *api, int create_flag, int directio_flag){
double start_time = 1.0, end_time = 0.0;
int open_flags;
char *error_string;
if (get_measurement_rank () == 0){
if (strcmp (api, POSIX_API) == 0){
if (directio_flag != 0)
open_flags = O_WRONLY | O_DIRECT;
else
open_flags = O_WRONLY;
errno = 0;
if (create_flag == 0){ /* open existing file */
if ((io_fd = open (io_filename, open_flags)) < 0){
error_string = strerror (errno);
error_with_abort (errno,
"\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed."
"\nCannot open local file (write only mode)."
"\nError: %s\n",
size, api, create_flag, directio_flag, error_string);
}
}
else { /* open nonexisting file and create it */
if ((io_fd = open (io_filename, open_flags|O_CREAT, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0){
error_string = strerror (errno);
error_with_abort (errno,
"\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed."
"\nCannot open local file (write only mode)."
"\nError: %s\n",
size, api, create_flag, directio_flag, error_string);
}
}
start_time = start_synchronization ();
write (io_fd, get_send_buffer (), size);
fsync (io_fd);
end_time = MPI_Wtime ();
close (io_fd);
}
else{ /* if strcmp (api, POSIX_API) != 0 */
if (create_flag == 0){
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &io_fh);
}
else{ /* if create_flag != 0*/
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &io_fh);
}
MPI_File_set_view (io_fh, (MPI_Offset)0,
MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
start_time = start_synchronization ();
MPI_File_write (io_fh, get_send_buffer (), size, MPI_BYTE, MPI_STATUS_IGNORE);
MPI_File_sync (io_fh);
end_time = MPI_Wtime ();
MPI_File_close (&io_fh);
}
}
else if (get_measurement_rank () != 0) {
start_synchronization ();
}
stop_synchronization ();
if (get_measurement_rank () == 0)
return end_time - start_time;
else
return -1.0;
}
static int
test_mpio_special_collective(char *filename)
{
int mpi_size, mpi_rank;
MPI_File fh;
MPI_Datatype etype,buftype,filetype;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
char writedata[2*DIMSIZE];
char filerep[7] = "native";
int i;
int count,bufcount;
int blocklens[2];
MPI_Aint offsets[2];
MPI_Offset mpi_off = 0;
MPI_Status mpi_stat;
int retcode = 0;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
/* create MPI data type */
etype = MPI_BYTE;
if(mpi_rank == 0 || mpi_rank == 1) {
count = DIMSIZE;
bufcount = 1;
} /* end if */
else {
count = 0;
bufcount = 0;
} /* end else */
blocklens[0] = count;
offsets[0] = mpi_rank*count;
blocklens[1] = count;
offsets[1] = (mpi_size+mpi_rank)*count;
if(count !=0) {
if((mpi_err = MPI_Type_hindexed(2,
blocklens,
offsets,
etype,
&filetype)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
} /* end if */
if((mpi_err = MPI_Type_commit(&filetype)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
} /* end if */
if((mpi_err = MPI_Type_hindexed(2,
blocklens,
offsets,
etype,
&buftype)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
} /* end if */
if((mpi_err = MPI_Type_commit(&buftype)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
} /* end if */
} /* end if */
else {
filetype = MPI_BYTE;
buftype = MPI_BYTE;
} /* end else */
/* Open a file */
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD,
filename,
MPI_MODE_RDWR | MPI_MODE_CREATE,
MPI_INFO_NULL,
&fh)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
} /* end if */
/* each process writes some data */
for (i=0; i < 2*DIMSIZE; i++)
writedata[i] = (char)(mpi_rank*DIMSIZE + i);
/* Set the file view */
if((mpi_err = MPI_File_set_view(fh,
mpi_off,
MPI_BYTE,
filetype,
filerep,
MPI_INFO_NULL)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
} /* end if */
/* Collectively write into the file */
if ((mpi_err = MPI_File_write_at_all(fh,
mpi_off,
writedata,
bufcount,
buftype,
&mpi_stat)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, bufcount, mpi_err_str);
return 1;
} /* end if */
/* Close the file */
if ((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed. \n");
return 1;
} /* end if */
/* Perform a barrier */
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
if(retcode != 0) {
if(mpi_rank == 0) {
printf("special collective IO is NOT working at this platform\n");
printf(" Please report to [email protected] about this problem.\n");
} /* end if */
retcode = 1;
} /* end if */
return retcode;
} /* test_mpio_special_collective */
int main(int argc, char *argv[])
{
int iarrayOfSizes[2], iarrayOfSubsizes[2], iarrayOfStarts[2], ilocal_size;
int nproc[2], periods[2], icoord[2];
int m, n, i, j, wsize, wrank, crank, ndims, lrows, lcols, grow, gcol, err;
MPI_Datatype filetype;
MPI_File fh;
MPI_Comm cartcomm;
MPI_Info info0, info3;
double t, topen, twrite, tclose, wrate;
double *local_array;
char nstripesStr[12], stripeUnitStr[12];
int nstripes = -1;
int stripeUnit = -1;
MPI_Offset headerSize = 0;
MPI_Init(0,0);
MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
/* Get global array size */
m = n = 128; /* Set default size */
/* ioda [ n ] [ m ] [ nstripes ] [ stripeunit ] [ headersize ] */
if (argc > 0) {
if (argc > 1) m = atoi(argv[1]);
if (argc > 2) n = atoi(argv[2]);
if (argc > 3) nstripes = atoi(argv[3]);
if (argc > 4) stripeUnit = atoi(argv[4]);
if (argc > 5) headerSize = atoi(argv[5]);
if (argc > 6) {
if (wrank == 0)
fprintf(stderr,"Unrecognized argument %s\n", argv[6]);
MPI_Abort(MPI_COMM_WORLD,1);
}
}
if (wrank == 0) printf("Matrix is [%d,%d]; file dir = %s\n", m, n, MYSCRATCHDIR );
/* The default number of stripes = totalsize/1M */
if (nstripes < 0) {
nstripes = n * m * sizeof(double) / (1024*1024);
if (nstripes < 1) nstripes = 1;
}
if (wrank == 0) printf("nstripes = %d, stripeUnit = %d, header size = %d\n",
nstripes, stripeUnit, (int)headerSize);
/* Use topology routines to get decomposition and coordinates */
MPI_Comm_size(MPI_COMM_WORLD, &wsize);
nproc[0] = 0; nproc[1] = 0;
ndims = 2;
MPI_Dims_create(wsize, ndims, nproc);
periods[0] = 0; periods[1] = 0;
MPI_Cart_create(MPI_COMM_WORLD, ndims, nproc, periods, 1, &cartcomm);
MPI_Comm_rank(cartcomm, &crank);
MPI_Cart_coords(cartcomm, crank, ndims, icoord);
iarrayOfSizes[0] = m;
iarrayOfSizes[1] = n;
iarrayOfSubsizes[0] = m/nproc[0];
iarrayOfSubsizes[1] = n/nproc[1];
iarrayOfStarts[0] = icoord[0] * iarrayOfSubsizes[0];
iarrayOfStarts[1] = icoord[1] * iarrayOfSubsizes[1];
/* Initialize my block of the data */
ilocal_size = iarrayOfSubsizes[0] * iarrayOfSubsizes[1];
lrows = iarrayOfSubsizes[0];
lcols = iarrayOfSubsizes[1];
local_array = (double *)malloc(lrows*lcols*sizeof(double));
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (i=0; i<lrows; i++) {
for (j=0; j<lcols; j++) {
local_array[j*lrows+i] = (grow+i) + (gcol+j)*m;
}
}
/* Fortran order simply means the data is stored by columns */
MPI_Type_create_subarray(ndims, iarrayOfSizes, iarrayOfSubsizes,
iarrayOfStarts, MPI_ORDER_FORTRAN, MPI_DOUBLE,
&filetype);
MPI_Type_commit(&filetype);
info0 = MPI_INFO_NULL;
info3 = MPI_INFO_NULL;
if (nstripes > 0 || stripeUnit > 0) {
MPI_Info_create(&info0);
if (nstripes > 0) {
snprintf(nstripesStr, sizeof(nstripesStr), "%d", nstripes);
MPI_Info_set(info0, "striping_factor", nstripesStr);
MPI_Info_set(info0, "cb_nodes", nstripesStr);
}
if (stripeUnit > 0) {
snprintf(stripeUnitStr, sizeof(stripeUnitStr), "%d", stripeUnit);
MPI_Info_set(info0, "striping_unit", stripeUnitStr);
}
MPI_Info_dup(info0, &info3);
MPI_Info_set(info3, "romio_no_indep_rw", "true");
/* Other hints to consider:
direct_io=true
The default cb_buffer_size is 16777216 , but is overridden by the
striping unit, which is smaller by default.
*/
}
/* level - 3 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-3.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info3, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-3.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_File_set_view(fh, headerSize, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_write_all(fh, local_array, ilocal_size, MPI_DOUBLE,
MPI_STATUS_IGNORE);
twrite = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "collective write");
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-3.out");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
/* level - 0 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-0.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info0, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-0.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (j=0; j<lcols; j++) {
MPI_Offset offset = headerSize +
((MPI_Offset)(grow) + (MPI_Offset)(gcol+j)*m) * sizeof(double);
err = MPI_File_write_at(fh, offset, local_array+j*lrows, lrows, MPI_DOUBLE,
MPI_STATUS_IGNORE);
if (err != MPI_SUCCESS) myAbort(err, "write at");
}
twrite = MPI_Wtime() - t;
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-0");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
if (info0 != MPI_INFO_NULL) {
MPI_Info_free(&info0);
MPI_Info_free(&info3);
}
free(local_array);
MPI_Finalize();
return 0;
}