void IOserver::start() { //starting IO server (Sinc line) MPI_Status status,status2; MPI_Request send_statut_request; int flag=1; int control; int count=0; int fileID; int fileCounter; int newFilenameLength; int len; bool getingDataFlag; int getingData[IO_ClientSize_]; serverOn_flag=true; while(serverOn_flag) { //cout<<count<<endl; ostreamFile_flag=false; if(IO_Rank_==0) { //cout<<count<<endl; //count++; //send non-blocking for status server free. (Sync line) serverReady_flag=true; MPI_Isend(&serverReady_flag,1,MPI::BOOL,1,SERVER_STATE_TAG,syncLineComm_,&send_statut_request); //test if an ostream need to be open or if the server need to be stoped (Sync line) blocking rec MPI_Recv(&control,1,MPI::INT,1,SERVER_CONTROL_TAG,syncLineComm_,&status); } MPI_Bcast(&control,1,MPI::INT,0,IO_Comm_); //MPI_Bcast(&ostreamFile_flag,1,MPI::BOOL,0,IO_Comm_); if(control==CONTROL_STOP) { //if(IO_Rank_==0)cout<<"stop server"<<endl; serverOn_flag=false; } if(control==CONTROL_OPEN_OSTREAM) { //if(IO_Rank_==0)cout<<"ostream"<<endl; ostreamFile_flag=true; serverReady_flag=false; if(IO_Rank_==0)MPI_Isend(&serverReady_flag,1,MPI::BOOL,1,SERVER_STATE_TAG,syncLineComm_,&send_statut_request); fileCounter=0; while(ostreamFile_flag) { //cout<<"rank: "<<IO_Rank_<<", ostream open"<<endl; //test if there is a new file (Sync line) if(IO_Rank_==0)MPI_Recv(&control,1,MPI::INT,1,IO_FILE_CONTROL_TAG,syncLineComm_,&status); MPI_Bcast(&control,1,MPI::INT,0,IO_Comm_); if(control==CONTROL_CLOSE_OSTREAM) { //if(IO_Rank_==0)cout<<"stop ostream"<<endl; ostreamFile_flag=false; } if(control==CONTROL_CREATE_FILE) { //create new file ID if(fileCounter>= MAX_FILE_NUMBER) { if(IO_Rank_==0)cout<<"Already to much file"<<endl; fileID=FILE_FAIL; } else { fileID=fileCounter; //if(IO_Rank_==0)cout<<"IO_server creating new file : "<<fileID<<endl; } if(IO_Rank_==0)MPI_Ssend(&fileID,1,MPI::INT,1,IO_FILE_CONTROL_FILEID_TAG,syncLineComm_); if(fileID != FILE_FAIL) { if(IO_Rank_==0) { MPI_Probe(1,IO_FILE_CONTROL_FILENAME_TAG,syncLineComm_,&status); MPI_Get_count(&status,MPI::CHAR,&newFilenameLength); } MPI_Bcast(&newFilenameLength,1,MPI::INT,0,IO_Comm_); char * filename; filename = (char*)malloc(newFilenameLength*sizeof(char)); if(IO_Rank_==0)MPI_Recv(filename,newFilenameLength,MPI::CHAR,1,IO_FILE_CONTROL_FILENAME_TAG,syncLineComm_,&status2); MPI_Bcast(filename,newFilenameLength,MPI::CHAR,0,IO_Comm_); files[fileID].filename=filename; free(filename); files[fileID].type=FILETYPE_UNSTRUCTURED; // no structured file implemented!!!! //cout<< files[fileID].filename <<endl; if(fileID==0)files[fileID].data=dataBuffer; else files[fileID].data=&(files[fileID-1].data[files[fileID-1].size]); files[fileID].size=0; fileCounter++; if(files[fileID].type==FILETYPE_UNSTRUCTURED) { for(int i=0;i<IO_ClientSize_;i++)getingData[i]=1; getingDataFlag=true; while(getingDataFlag) { MPI_Iprobe(MPI_ANY_SOURCE,fileID, masterClientComm_,&flag,&status); if(flag==true) { char * send; send=&(files[fileID].data[files[fileID].size]); int size; MPI_Get_count(&status,MPI::CHAR,&size); MPI_Recv(send,size,MPI::CHAR,status.MPI_SOURCE,fileID,masterClientComm_,&status2); files[fileID].size+=size; } else { MPI_Iprobe(MPI_ANY_SOURCE,IO_FILE_CONTROL_CLOSE_TAG,masterClientComm_ ,&flag,&status); if(flag==true) { //cout<<"file closed"<<endl; int tempFileID; int total=0; int source = status.MPI_SOURCE; //cout<<source<<endl; MPI_Recv(&tempFileID,1,MPI::INT,source,IO_FILE_CONTROL_CLOSE_TAG,masterClientComm_,&status2); //cout<<source<<endl; getingData[source-1]=0; for(int i=0;i<IO_ClientSize_;i++)total+=getingData[i]; if(total==0)getingDataFlag=false; //cout<<total<<endl; } } } //cout<<"file closed"<<endl; MPI_Barrier(IO_Comm_); } MPI_Barrier(IO_Comm_); } } }//close stream and write if(fileCounter!=0) { for(int i=0;i < fileCounter;i++) { //cout<<"writing file: "<< i<<endl; MPI_File ofile; long file_Offset=0; long temp_long=0; string str_fname; str_fname = files[i].filename + int2string(IO_Node_,999)+".dat"; char * fname = &(str_fname[0]); for(int k=0;k<(IO_NodeSize_-1); k++) { if(IO_NodeRank_==k) { temp_long = file_Offset + files[i].size; MPI_Send(&temp_long,1,MPI_LONG, k+1 , 0, IO_NodeComm_ ); } if(IO_NodeRank_==k+1) { MPI_Recv( &file_Offset, 1, MPI_LONG, k, 0, IO_NodeComm_, &status); } } if(files[i].size!=0) { MPI_File_open(IO_NodeComm_,fname,MPI_MODE_WRONLY | MPI_MODE_CREATE,MPI_INFO_NULL,&ofile); MPI_File_set_view(ofile,file_Offset,MPI_CHAR,MPI_CHAR,(char*)"native",MPI_INFO_NULL); MPI_File_write_all(ofile,files[i].data,files[i].size,MPI_CHAR,&status); MPI_File_close(&ofile); } } } fileCounter=0; MPI_Barrier(IO_Comm_); } } }
int main (int argc, char *argv[]) { int proc_num, my_rank, len; int i, j; double start_time, elapsed_time, all_time; double all_time_max, all_time_avg, all_time_min; struct timespec ts; MPI_Status status; MPI_File fh; MPI_Datatype contig_type; MPI_Init(&argc, &argv); // get the number of procs and rank in the comm MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); if(argc != 5) { printf("Wrong argument number!\n"); printf("Use %s filename request_size repeat_times\n", argv[0]); return 0; } int req_size = atoi(argv[2]); int repeat_time = atoi(argv[3]); double ht_read_time= atof(argv[4]); ht_read_time *= 2.0; //if(my_rank == 0) // printf("Sleep time: %lf\n",ht_read_time); ts.tv_sec = (int)ht_read_time; ts.tv_nsec = (ht_read_time - ts.tv_sec) * 1000000000; MPI_Offset stride = proc_num * req_size; MPI_Offset tmp_pos = my_rank * req_size; char *read_data = (char*)malloc(req_size); MPI_Type_contiguous( req_size, MPI_CHAR, &contig_type); MPI_Type_commit(&contig_type); start_time = MPI_Wtime(); //MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDWR, MPI_INFO_NULL, &fh); if(fh==NULL){ printf("File not exist\n"); return -1; } for(i = 0; i < repeat_time; i++) { // MPI_Barrier(MPI_COMM_WORLD); MPI_File_read_at( fh, tmp_pos, read_data, 1, contig_type, &status ); tmp_pos += stride; nanosleep(&ts, NULL); } MPI_File_close(&fh); elapsed_time = MPI_Wtime() - start_time; MPI_Reduce(&elapsed_time, &all_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); MPI_Reduce(&elapsed_time, &all_time_min, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); MPI_Reduce(&elapsed_time, &all_time_avg, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); all_time_avg /= proc_num; MPI_Barrier(MPI_COMM_WORLD); double data_in_mb = (proc_num*(double)req_size*repeat_time)/(1024.0*1024.0); if(my_rank == 0) printf("Total time: %lf Min time: %lf Avg time: %lf Total data: %dM Agg Bandwidth: %lf\n", all_time, all_time_min, all_time_avg, (int)data_in_mb, data_in_mb/all_time); // printf("%d: %lf\n",my_rank, elapsed_time); free(read_data); MPI_Type_free(&contig_type); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { int *buf, i, rank, nprocs, len, sum, global_sum; char *filename; MPI_File fh; MPI_Status status; MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); /* process 0 takes the file name as a command-line argument and broadcasts it to other processes */ if (!rank) { i = 1; while ((i < argc) && strcmp("-fname", *argv)) { i++; argv++; } if (i >= argc) { printf("\n*# Usage: shared_fp <mpiparameter> -- -fname filename\n\n"); MPI_Abort(MPI_COMM_WORLD, 1); } argv++; len = strlen(*argv); filename = (char *) malloc(len+10); strcpy(filename, *argv); MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD); } else { MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); filename = (char *) malloc(len+10); MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD); } buf = (int *) malloc(COUNT * sizeof(int)); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); for (i=0; i<COUNT; i++) buf[i] = COUNT*rank + i; MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_write_shared(fh, buf, COUNT, MPI_INT, &status); for (i=0; i<COUNT; i++) buf[i] = 0; MPI_Barrier(MPI_COMM_WORLD); MPI_File_seek_shared(fh, 0, MPI_SEEK_SET); MPI_File_read_shared(fh, buf, COUNT, MPI_INT, &status); MPI_File_close(&fh); sum = 0; for (i=0; i<COUNT; i++) sum += buf[i]; MPI_Allreduce(&sum, &global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (global_sum != (((COUNT*nprocs - 1)*(COUNT*nprocs))/2)) printf("Error: sum %d, global_sum %d, %d\n", sum, global_sum,(((COUNT*nprocs - 1)*(COUNT*nprocs))/2)); free(buf); free(filename); if (!rank) printf("Done\n"); MPI_Finalize(); return 0; }
static int test_mpio_derived_dtype(char *filename) { MPI_File fh; char mpi_err_str[MPI_MAX_ERROR_STRING]; int mpi_err_strlen; int mpi_err; int i; int nerrors = 0; /* number of errors */ MPI_Datatype etype,filetype; MPI_Datatype adv_filetype,bas_filetype[2]; MPI_Datatype etypenew, filetypenew; MPI_Offset disp; MPI_Status Status; MPI_Aint adv_disp[2]; MPI_Aint offsets[1]; int blocklens[1],adv_blocklens[2]; int count,outcount; int retcode; int mpi_rank,mpi_size; char buf[3],outbuf[3] = {0}; MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); retcode = 0; for(i=0;i<3;i++) buf[i] = i+1; if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_open failed (%s)\n", mpi_err_str); return 1; } disp = 0; etype = MPI_BYTE; count = 1; blocklens[0] = 1; offsets[0] = 0; if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetype)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } count = 1; blocklens[0]=1; offsets[0] = 1; if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetypenew)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&filetypenew))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } outcount = 2; adv_blocklens[0] = 1; adv_blocklens[1] = 1; adv_disp[0] = 0; adv_disp[1] = 1; bas_filetype[0] = filetype; bas_filetype[1] = filetypenew; if((mpi_err= MPI_Type_struct(outcount,adv_blocklens,adv_disp,bas_filetype,&adv_filetype)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_struct failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&adv_filetype))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } if((mpi_err = MPI_File_set_view(fh,disp,etype,adv_filetype,"native",MPI_INFO_NULL))!= MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_set_view failed (%s)\n", mpi_err_str); return 1; } if((mpi_err = MPI_File_write(fh,buf,3,MPI_BYTE,&Status))!= MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_write failed (%s)\n", mpi_err_str); return 1; ; } if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_close failed (%s)\n", mpi_err_str); return 1; } if((mpi_err = MPI_File_open(MPI_COMM_WORLD,filename,MPI_MODE_RDONLY,MPI_INFO_NULL,&fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_open failed (%s)\n", mpi_err_str); return 1; } if((mpi_err = MPI_File_set_view(fh,0,MPI_BYTE,MPI_BYTE,"native",MPI_INFO_NULL))!= MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_set_view failed (%s)\n", mpi_err_str); return 1; } if((mpi_err = MPI_File_read(fh,outbuf,3,MPI_BYTE,&Status))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_read failed (%s)\n", mpi_err_str); return 1; } if(outbuf[2]==2) { retcode = 0; } else { /* if(mpi_rank == 0) { printf("complicated derived datatype is NOT working at this platform\n"); printf("go back to hdf5/config and find the corresponding\n"); printf("configure-specific file and change ?????\n"); } */ retcode = -1; } if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_close failed (%s)\n", mpi_err_str); return 1; } mpi_err = MPI_Barrier(MPI_COMM_WORLD); #ifdef H5_MPI_COMPLEX_DERIVED_DATATYPE_WORKS if(retcode == -1) { if(mpi_rank == 0) { printf("Complicated derived datatype is NOT working at this platform\n"); printf("Go back to hdf5/config and find the corresponding\n"); printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n"); printf("hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype-works='no'}\n"); printf(" at the end of the file.\n"); printf(" Please report to [email protected] about this problem.\n"); } retcode = 1; } #else if(retcode == 0) { if(mpi_rank == 0) { printf(" This is NOT an error, What it really says is\n"); printf("Complicated derived datatype is WORKING at this platform\n"); printf(" Go back to hdf5/config and find the corresponding \n"); printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n"); printf("hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype-works='no'}\n"); printf(" at the end of the file.\n"); printf("Please report to [email protected] about this problem.\n"); } retcode = 1; } if(retcode == -1) retcode = 0; #endif return retcode; }
int main(int argc,char **argv) { MPI_File file; long long mapxsize,mapysize; long long myxmin,myxmax,myymin,myymax; int processes_in_x_dim,processes_in_y_dim; int my_proc_id_in_x_dim,my_proc_id_in_y_dim; long long boxxsize,boxysize; // sizes of a map fragment handled by each process int myrank,proccount; MPI_Offset filesize; long long x,y; // counters to go through a map fragment double max_similarity,my_similarity,my_temp_similarity; double cell_val; int provided_thread_support; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE, &provided_thread_support); // first read the file name from command line if (argc<7) { printf("\nSyntax: 2Dmapsearch-MPIIO <map_filename> mapxsize mapysize processes_in_x_dim processes_in_y_dim pmem_path\n"); MPI_Finalize(); exit(-1); } mapxsize=atol(argv[2]); mapysize=atol(argv[3]); processes_in_x_dim=atoi(argv[4]); processes_in_y_dim=atoi(argv[5]); if (mapxsize*mapysize<=0) { printf("\nWrong map size given.\n"); MPI_Finalize(); exit(-1); } // find out my rank and the number of processes MPI_Comm_rank(MPI_COMM_WORLD,&myrank); MPI_Comm_size(MPI_COMM_WORLD,&proccount); // now check if the number of processes matches the specified processes in dims if (proccount!=(processes_in_x_dim*processes_in_y_dim)) { printf("\nThe number of processes started does not match processes_in_x_dim*processes_in_y_dim.\n"); MPI_Finalize(); exit(-1); } MPI_Info info; MPI_Info_create(&info); MPI_Info_set(info,"pmem_path",argv[6]); MPI_Info_set(info,"pmem_io_mode","0"); MPI_File_open(MPI_COMM_WORLD,argv[1],MPI_MODE_RDWR,info,&file) ; // now check the size of the file vs the given map size MPI_File_get_size(file,&filesize); if (filesize<mapxsize*mapysize) { printf("\nFile too small for the specified map size.\n"); MPI_File_close(&file); MPI_Finalize(); exit(-1); } // now each process should determine its bounding box for the map // length of each box will be (mapxsize/processes_in_x_dim) and similarly for the y dimension boxxsize=(mapxsize/processes_in_x_dim); boxysize=(mapysize/processes_in_y_dim); my_proc_id_in_x_dim=myrank%processes_in_x_dim; my_proc_id_in_y_dim=myrank/processes_in_x_dim; myxmin=my_proc_id_in_x_dim*boxxsize; myymin=my_proc_id_in_y_dim*boxysize; myxmax=myxmin+boxxsize; myymax=myymin+boxysize; // now each process should scan its fragment // if a certain element is detected then the application scans its immediate surroundings for elements of some other types my_similarity=0; for(x=myxmin;x<myxmax;x++) for(y=myymin;y<myymax;y++) { cell_val=get_xy_cell(x,y,file,mapxsize,mapysize); if ((cell_val>=CELL_VAL_LOW_THRESHOLD) && (cell_val<=CELL_VAL_HIGH_THRESHOLD)) my_temp_similarity=eval_surrounding(x,y,file,mapxsize,mapysize); if (my_temp_similarity>=my_similarity) my_similarity=my_temp_similarity; } // now all processes should select the highest similarity MPI_Reduce(&my_similarity,&max_similarity,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD); if (!myrank) { printf("\nThe final similarity is %f\n",max_similarity); } MPI_File_close(&file); MPI_Finalize(); }
void parallel_readwrite(char *file_name, void *dump_buffer, int type_of_file, int is_write, long long offset) { #if MPI && DO_PARALLEL_WRITE MPI_File fh; MPI_Status status; MPI_Datatype mpi_elementary_type, mpi_file_type; int file_open_error, file_write_error ; int error_string_length; char error_string[BUFSIZ]; MPI_Offset file_size; int count; void *mpi_buffer; size_t mpi_buffer_size; int mode; MPI_Offset mpi_offset; MPI_Barrier(MPI_COMM_WORLD); if (is_write) { mode = MPI_MODE_CREATE | MPI_MODE_WRONLY | MPI_MODE_APPEND; } else { mode = MPI_MODE_RDONLY; } file_open_error = MPI_File_open(MPI_COMM_WORLD, file_name, mode, MPI_INFO_NULL, &fh); if (file_open_error != MPI_SUCCESS) { MPI_Error_string(file_open_error, error_string, &error_string_length); fprintf(stderr, "parallel_readwrite(): error opening file: %3d: %s\n", mpi_rank, error_string); MPI_Abort(MPI_COMM_WORLD, file_open_error); /* It is still OK to abort, because we have failed to open the file. */ } else { // if (i_am_the_master) // chmod(file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (offset < 0L) { if(is_write) { MPI_File_get_position(fh, &mpi_offset); offset = mpi_offset; } else { offset = 0L; } } MPI_Barrier(MPI_COMM_WORLD); //differentiate data type and buffers involved based on file type if( DUMP_FILE == type_of_file ) { mpi_elementary_type = MPI_DUMP_TYPE; mpi_file_type = dump_file_type; mpi_buffer = (void*)dump_buffer; mpi_buffer_size = dump_buffer_size; } else if( GDUMP_FILE == type_of_file){ mpi_elementary_type = MPI_GDUMP_TYPE; mpi_file_type = gdump_file_type; mpi_buffer = (void*)gdump_buffer; mpi_buffer_size = gdump_buffer_size; } else if( GDUMP2_FILE == type_of_file){ mpi_elementary_type = MPI_GDUMP2_TYPE; mpi_file_type = gdump2_file_type; mpi_buffer = (void*)gdump2_buffer; mpi_buffer_size = gdump2_buffer_size; } else if( RDUMP_FILE == type_of_file){ mpi_elementary_type = MPI_RDUMP_TYPE; mpi_file_type = rdump_file_type; mpi_buffer = (void*)rdump_buffer; mpi_buffer_size = rdump_buffer_size; } else if( FDUMP_FILE == type_of_file){ mpi_elementary_type = MPI_FDUMP_TYPE; mpi_file_type = fdump_file_type; mpi_buffer = (void*)fdump_buffer; mpi_buffer_size = fdump_buffer_size; } else { if(i_am_the_master) fprintf(stderr, "Unknown file type %d\n", type_of_file); MPI_File_close(&fh); MPI_Finalize(); exit(2); } MPI_File_set_view(fh, offset, mpi_elementary_type, mpi_file_type, "native", MPI_INFO_NULL); if (is_write) { file_write_error = MPI_File_write_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type, &status); } else { file_write_error = MPI_File_read_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type, &status); } if (file_write_error != MPI_SUCCESS) { MPI_Error_string(file_write_error, error_string, &error_string_length); fprintf(stderr, "parallel_readwrite(): error %s file: %3d: %s\n", (is_write)?("writing"):("reading"), mpi_rank, error_string); MPI_File_close(&fh); //if (i_am_the_master) MPI_File_delete(file_name, MPI_INFO_NULL); MPI_Finalize(); exit(1); } // MPI_Get_count(&status, MPI_FLOAT, &count); // MPI_File_get_size(fh, &file_size); // if(1) { // printf("%3d: wrote %d floats, expected to write %lld floats\n", mpi_rank, count, (long long int)dump_buffer_size); // printf("%3d: file size is %lld bytes, header-related offset is %lld\n", mpi_rank, file_size, offset); // } MPI_File_close(&fh); } #endif }
static int test_mpio_overlap_writes(char *filename) { int mpi_size, mpi_rank; MPI_Comm comm; MPI_Info info = MPI_INFO_NULL; int color, mrc; MPI_File fh; int i; int vrfyerrs, nerrs; unsigned char buf[4093]; /* use some prime number for size */ int bufsize = sizeof(buf); MPI_Offset stride; MPI_Offset mpi_off; MPI_Status mpi_stat; if (VERBOSE_MED) printf("MPIO independent overlapping writes test on file %s\n", filename); nerrs = 0; /* set up MPI parameters */ MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); /* Need at least 2 processes */ if (mpi_size < 2) { if (MAINPROCESS) printf("Need at least 2 processes to run MPIO test.\n"); printf(" -SKIP- \n"); return 0; } /* splits processes 0 to n-2 into one comm. and the last one into another */ color = ((mpi_rank < (mpi_size - 1)) ? 0 : 1); mrc = MPI_Comm_split (MPI_COMM_WORLD, color, mpi_rank, &comm); VRFY((mrc==MPI_SUCCESS), "Comm_split succeeded"); if (color==0){ /* First n-1 processes (color==0) open a file and write it */ mrc = MPI_File_open(comm, filename, MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh); VRFY((mrc==MPI_SUCCESS), ""); stride = 1; mpi_off = mpi_rank*stride; while (mpi_off < MPIO_TEST_WRITE_SIZE){ /* make sure the write does not exceed the TEST_WRITE_SIZE */ if (mpi_off+stride > MPIO_TEST_WRITE_SIZE) stride = MPIO_TEST_WRITE_SIZE - mpi_off; /* set data to some trivial pattern for easy verification */ for (i=0; i<stride; i++) buf[i] = (unsigned char)(mpi_off+i); mrc = MPI_File_write_at(fh, mpi_off, buf, (int)stride, MPI_BYTE, &mpi_stat); VRFY((mrc==MPI_SUCCESS), ""); /* move the offset pointer to last byte written by all processes */ mpi_off += (mpi_size - 1 - mpi_rank) * stride; /* Increase chunk size without exceeding buffer size. */ /* Then move the starting offset for next write. */ stride *= 2; if (stride > bufsize) stride = bufsize; mpi_off += mpi_rank*stride; } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); mrc = MPI_Comm_free(&comm); VRFY((mrc==MPI_SUCCESS), "MPI_Comm_free"); /* sync with the other waiting processes */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync after writes"); }else{ /* last process waits till writes are done, * then opens file to verify data. */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync after writes"); mrc = MPI_File_open(comm, filename, MPI_MODE_RDONLY, info, &fh); VRFY((mrc==MPI_SUCCESS), ""); stride = bufsize; for (mpi_off=0; mpi_off < MPIO_TEST_WRITE_SIZE; mpi_off += bufsize){ /* make sure it does not read beyond end of data */ if (mpi_off+stride > MPIO_TEST_WRITE_SIZE) stride = MPIO_TEST_WRITE_SIZE - mpi_off; mrc = MPI_File_read_at(fh, mpi_off, buf, (int)stride, MPI_BYTE, &mpi_stat); VRFY((mrc==MPI_SUCCESS), ""); vrfyerrs=0; for (i=0; i<stride; i++){ unsigned char expected; expected = (unsigned char)(mpi_off+i); if ((expected != buf[i]) && (vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED)) { printf("proc %d: found data error at [%ld], expect %u, got %u\n", mpi_rank, (long)(mpi_off+i), expected, buf[i]); } } if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED) printf("proc %d: [more errors ...]\n", mpi_rank); nerrs += vrfyerrs; } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); mrc = MPI_Comm_free(&comm); VRFY((mrc==MPI_SUCCESS), "MPI_Comm_free"); } /* * one more sync to ensure all processes have done reading * before ending this test. */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync before leaving test"); return (nerrs); }
int MPI_File_fclose(MPI_FFile * handle) { MPI_File_fflush(handle); MPI_File_close(&(handle->fh)); return 0; }
int main(int argc, char **argv) { int *writebuf, *readbuf, i, mynod, nprocs, len, err; char *filename; int errs=0, toterrs; MPI_Datatype newtype; MPI_File fh; MPI_Status status; MPI_Info info; MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &mynod); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); /* process 0 takes the file name as a command-line argument and broadcasts it to other processes */ if (!mynod) { i = 1; while ((i < argc) && strcmp("-fname", *argv)) { i++; argv++; } if (i >= argc) { fprintf(stderr, "\n*# Usage: coll_test -fname filename\n\n"); MPI_Abort(MPI_COMM_WORLD, 1); } argv++; len = strlen(*argv); filename = (char *) malloc(len+1); strcpy(filename, *argv); MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD); } else { MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); filename = (char *) malloc(len+1); MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD); } writebuf = (int *) malloc(BUFSIZE*sizeof(int)); readbuf = (int *) malloc(BUFSIZE*sizeof(int)); /* test atomicity of contiguous accesses */ /* initialize file to all zeros */ if (!mynod) { MPI_File_delete(filename, MPI_INFO_NULL); MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); for (i=0; i<BUFSIZE; i++) writebuf[i] = 0; MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status); MPI_File_close(&fh); #if VERBOSE fprintf(stderr, "\ntesting contiguous accesses\n"); #endif } MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<BUFSIZE; i++) writebuf[i] = 10; for (i=0; i<BUFSIZE; i++) readbuf[i] = 20; MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); /* set atomicity to true */ err = MPI_File_set_atomicity(fh, 1); if (err != MPI_SUCCESS) { fprintf(stderr, "Atomic mode not supported on this file system.\n");fflush(stderr); MPI_Abort(MPI_COMM_WORLD, 1); } MPI_Barrier(MPI_COMM_WORLD); /* process 0 writes and others concurrently read. In atomic mode, the data read must be either all old values or all new values; nothing in between. */ if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status); else { err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status); if (err == MPI_SUCCESS) { if (readbuf[0] == 0) { /* the rest must also be 0 */ for (i=1; i<BUFSIZE; i++) if (readbuf[i] != 0) { errs++; fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]); MPI_Abort(MPI_COMM_WORLD, 1); } } else if (readbuf[0] == 10) { /* the rest must also be 10 */ for (i=1; i<BUFSIZE; i++) if (readbuf[i] != 10) { errs++; fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]); MPI_Abort(MPI_COMM_WORLD, 1); } } else { errs++; fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]); } } } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); /* repeat the same test with a noncontiguous filetype */ MPI_Type_vector(BUFSIZE, 1, 2, MPI_INT, &newtype); MPI_Type_commit(&newtype); MPI_Info_create(&info); /* I am setting these info values for testing purposes only. It is better to use the default values in practice. */ MPI_Info_set(info, "ind_rd_buffer_size", "1209"); MPI_Info_set(info, "ind_wr_buffer_size", "1107"); if (!mynod) { MPI_File_delete(filename, MPI_INFO_NULL); MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); for (i=0; i<BUFSIZE; i++) writebuf[i] = 0; MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info); MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status); MPI_File_close(&fh); #if VERBOSE fprintf(stderr, "\ntesting noncontiguous accesses\n"); #endif } MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<BUFSIZE; i++) writebuf[i] = 10; for (i=0; i<BUFSIZE; i++) readbuf[i] = 20; MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); MPI_File_set_atomicity(fh, 1); MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status); else { err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status); if (err == MPI_SUCCESS) { if (readbuf[0] == 0) { for (i=1; i<BUFSIZE; i++) if (readbuf[i] != 0) { errs++; fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]); MPI_Abort(MPI_COMM_WORLD, 1); } } else if (readbuf[0] == 10) { for (i=1; i<BUFSIZE; i++) if (readbuf[i] != 10) { errs++; fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]); MPI_Abort(MPI_COMM_WORLD, 1); } } else { errs++; fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]); } } } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); if (mynod == 0) { if( toterrs > 0) { fprintf( stderr, "Found %d errors\n", toterrs ); } else { fprintf( stdout, " No Errors\n" ); } } MPI_Type_free(&newtype); MPI_Info_free(&info); free(writebuf); free(readbuf); free(filename); MPI_Finalize(); return 0; }
void read_unordered_shared( const char * filename, void * local_ptr, size_t size ) { MPI_Status status; MPI_File infile; MPI_Datatype datatype; MPI_Info info; char * local_char_ptr = static_cast< char * >( local_ptr ); // Stupid MPI uses a signed integer for the count of data elements // to write. On all the machines we use, this means the max count // is 2^31-1. To work around this, we create a datatype large // enough that we never need a count value larger than 2^30. // move this many gigabyte chunks const size_t gigabyte = 1L << 30; size_t round_count = size / gigabyte; // if there will be any bytes left over, move those too if( size & (gigabyte - 1) ) round_count++; MPI_CHECK( MPI_Info_create( &info ) ); if( FLAGS_optimize_for_lustre ) { std::map< const std::string, const std::string > info_map = {{ // disable independent file operations, since we know this // routine is the only one touching the file { "romio_no_indep_rw", "true" } // // disable collective io on lustre for "small" files <= 1GB // , { "romio_lustre_ds_in_coll", "1073741825" ) ); // set collective buffering block size to something reasonable , { "cb_buffer_size", "33554432" } // // disable collective buffering for writing // , { "romio_cb_write", "disable" } // // disable collective buffering for writing // , { "romio_cb_read", "disable" } // disable data sieving for writing , { "romio_ds_write", "disable" } // disable data sieving for writing , { "romio_ds_read", "disable" } // enable direct IO , { "direct_read", "true" } , { "direct_write", "true" } // ??? // , { "romio_lustre_co_ratio", "1" } // maybe // , { "access_style", "read_once" } }}; set_mpi_info( info, info_map ); } // open file for reading int mode = MPI_MODE_RDONLY; MPI_CHECK( MPI_File_open( global_communicator.grappa_comm, const_cast< char * >( filename ), mode, info, &infile ) ); // make sure we will read exactly the whole file int64_t total_size = 0; MPI_CHECK( MPI_Reduce( &size, &total_size, 1, MPI_INT64_T, MPI_SUM, 0, global_communicator.grappa_comm ) ); if( 0 == Grappa::mycore() ) { MPI_Offset file_size = 0; MPI_CHECK( MPI_File_get_size( infile, &file_size ) ); CHECK_EQ( file_size, total_size ) << "Sizes don't line up to read the enitre file?"; } // // dump file info for debugging // if( Grappa::mycore == 0 ) { // dump_mpi_file_info( infile ); // } // compute number of rounds required to read entire file MPI_CHECK( MPI_Allreduce( MPI_IN_PLACE, &round_count, 1, MPI_INT64_T, MPI_MAX, global_communicator.grappa_comm ) ); // read file in gigabyte-sized rounds for( int i = 0; i < round_count; ++i ) { if( size > gigabyte ) { MPI_CHECK( MPI_File_read_shared( infile, local_char_ptr, gigabyte, MPI_BYTE, &status ) ); size -= gigabyte; local_char_ptr += gigabyte; } else { MPI_CHECK( MPI_File_read_shared( infile, local_char_ptr, size, MPI_BYTE, &status ) ); } } MPI_CHECK( MPI_Info_free( &info ) ); MPI_CHECK( MPI_File_close( &infile ) ); }
void fileparse(unsigned int num_rows) { int world_size, world_rank; MPI_Comm_size(MPI_COMM_WORLD, &world_size); MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); char search_phrase[] = "AdeptProject"; size_t sp_len = strlen(search_phrase); unsigned int desired_line_len = 81; char line[desired_line_len]; srand(time(NULL)); // Set seed int i = 0; int r = 0; int m = 0; int mismatch = 0; int r_count = 0; int m_count = 0; struct timespec start, end; /* p_num_rows is the number of rows across all processes */ unsigned int p_num_rows; p_num_rows = (unsigned int) (num_rows * world_size); /* Generate (on the fly) the test file for the run */ /* Make this single threaded for ease */ if (world_rank == 0) { FILE* fp; fp = fopen("testfile", "w+"); for (i = 0; i < p_num_rows; i++) { r = create_line(search_phrase, sp_len, line, desired_line_len); m = seek_match(search_phrase, sp_len, line, desired_line_len); if (r != m) { mismatch++; } if (r == 0) { r_count++; } if (m == 0) { m_count++; } fprintf(fp, "%s\n", line); } fsync(fileno(fp)); fclose(fp); } m = 0; MPI_Info info; MPI_Info_create(&info); MPI_File fh; MPI_Status status; /* For holding the data from the file before parsing */ char *lb = (char*) malloc(sizeof (char)*num_rows * (desired_line_len + 1)); char *lbp = NULL; MPI_Barrier(MPI_COMM_WORLD); if (world_rank == 0) { clock_gettime(CLOCK, &start); } m_count = 0; /* This part should use MPI-IO */ MPI_File_open(MPI_COMM_WORLD, "testfile", MPI_MODE_RDWR | MPI_MODE_CREATE, info, &fh); MPI_File_read_at(fh, world_rank * num_rows * (desired_line_len + 1), lb, num_rows * (desired_line_len + 1), MPI_CHAR, &status); for (i = 0; i < num_rows; i++) { lbp = &lb[i * (desired_line_len + 1)]; m = seek_match(search_phrase, sp_len, lbp, desired_line_len); if (m == 0) { m_count++; } } MPI_Barrier(MPI_COMM_WORLD); MPI_File_close(&fh); if (world_rank == 0) { clock_gettime(CLOCK, &end); elapsed_time_hr(start, end, "Fileparse"); } MPI_Barrier(MPI_COMM_WORLD); unlink("testfile"); // Use this to ensure the generated file is removed from the system upon finish }
void write_unordered_shared( const char * filename, void * local_ptr, size_t size ) { MPI_Status status; MPI_File outfile; MPI_Datatype datatype; MPI_Info info; char * local_char_ptr = static_cast< char * >( local_ptr ); // Stupid MPI uses a signed integer for the count of data elements // to write. On all the machines we use, this means the max count // is 2^31-1. To work around this, we create a datatype large // enough that we never need a count value larger than 2^30. // move this many gigabyte chunks const size_t gigabyte = 1L << 30; size_t round_count = size / gigabyte; // if there will be any bytes left over, move those too if( size & (gigabyte - 1) ) round_count++; MPI_CHECK( MPI_Info_create( &info ) ); if( FLAGS_optimize_for_lustre ) { std::map< const std::string, const std::string > info_map = {{ // disable independent file operations, since we know this // routine is the only one touching the file { "romio_no_indep_rw", "true" } // it's important to set lustre striping properly for performance. // we're supposed to be able to do this through MPI ROMIO, but mpi installations are not always configured to allow this. // if not, then before saving a file, run a command like this to create it: // lfs setstripe -c -1 -s 32m <filename> // below are what we'd like to do. // we want 32MB lustre blocks, but this probably doesn't do anything , { "striping_unit", "33554432" } // we want to use all lustre OSTs, but this probably doesn't do anything , { "striping_factor", "-1" } // // disable collective io on lustre for "small" files <= 1GB // , { "romio_lustre_ds_in_coll", "1073741825" ) ); // set collective buffering block size to something reasonable , { "cb_buffer_size", "33554432" } // // disable collective buffering for writing // , { "romio_cb_write", "disable" } // // disable collective buffering for writing // , { "romio_cb_read", "disable" } // disable data sieving for writing , { "romio_ds_write", "disable" } // disable data sieving for writing , { "romio_ds_read", "disable" } // enable direct IO , { "direct_read", "true" } , { "direct_write", "true" } // ??? // , { "romio_lustre_co_ratio", "1" } // maybe // , { "access_style", "read_once" } }}; set_mpi_info( info, info_map ); } // open file for writing int mode = MPI_MODE_CREATE | MPI_MODE_WRONLY; MPI_CHECK( MPI_File_open( global_communicator.grappa_comm, const_cast< char * >( filename ), mode, info, &outfile ) ); // drop any data previously in file MPI_CHECK( MPI_File_set_size( outfile, 0 ) ); // dump file info for debugging // if( Grappa::mycore == 0 ) { // dump_mpi_file_info( outfile ); // } // compute number of rounds required to read entire file MPI_CHECK( MPI_Allreduce( MPI_IN_PLACE, &round_count, 1, MPI_INT64_T, MPI_MAX, global_communicator.grappa_comm ) ); // write file in gigabyte-sized rounds for( int i = 0; i < round_count; ++i ) { if( size > gigabyte ) { MPI_CHECK( MPI_File_write_shared( outfile, local_char_ptr, gigabyte, MPI_BYTE, &status ) ); size -= gigabyte; local_char_ptr += gigabyte; } else { MPI_CHECK( MPI_File_write_shared( outfile, local_char_ptr, size, MPI_BYTE, &status ) ); } } MPI_CHECK( MPI_Info_free( &info ) ); MPI_CHECK( MPI_File_close( &outfile ) ); }
void writetofile(int rank, int a) { #if defined(MPI) && defined(FILEIMAGE) // dump all the tiles to 1 file using MPI-IO int color = 0; MPI_Comm tilecomm; MPI_File fhtout; if (owneroftile == rank) color = 1; PRINTDEBUG("%3d: tileowner=%d, color=%d\n", rank, owneroftile, color); MPI_Comm_split(MPI_COMM_WORLD, color, rank, &tilecomm); if (color) { int cartrank; MPI_Comm cartcomm; const MPI_Status status; // init parameters for Cart_create int gsizes[2] = {viewport.h, viewport.w*ELEMENTSPERPIXEL}; int psizes[2] = {viewport.tiley, viewport.tilex}; int lsizes[2] = {gsizes[0]/psizes[0], (gsizes[1])/psizes[1]}; int dims[2] = {psizes[0], psizes[1]}; int periods[2] = {1,1}; int coords[2]; PRINTDEBUG("%3d: gsizes=%d,%d, psizes=%d,%d, lsizes=%d,%d\n", rank, gsizes[0], gsizes[1], psizes[0], psizes[1], lsizes[0], lsizes[1]); // Create cartesian coord MPI_Cart_create(tilecomm, 2, dims, periods, 0, &cartcomm); MPI_Comm_rank(cartcomm, &cartrank); MPI_Cart_coords(cartcomm, cartrank, 2, coords); // check coords PRINTDEBUG("%3d: coords=%d,%d\n", cartrank, coords[0], coords[1]); /* global indices of first element of local array */ int startindex[2]; startindex[0] = coords[0] * lsizes[0]; startindex[1] = coords[1] * lsizes[1]; PRINTDEBUG("%3d: start=%d,%d\n", cartrank, startindex[0], startindex[1]); int filetype; filetype=0; MPI_Type_create_subarray(2, gsizes, lsizes, startindex, MPI_ORDER_C, MPI_FLOAT, &filetype); MPI_Type_commit(&filetype); char imagefull[32]; sprintf(imagefull,"imagefull%d.raw", a); PRINTDEBUG("%s\n", imagefull); MPI_File_open(cartcomm, imagefull, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fhtout); MPI_File_set_view(fhtout, 0, MPI_FLOAT, filetype, "native", MPI_INFO_NULL); int localarraysize = lsizes[0] * lsizes[1]; PRINTDEBUG("%3d: size=%d\n", cartrank, localarraysize); MPI_File_write(fhtout, tilebuffer, localarraysize, MPI_FLOAT, &status); int count; MPI_Get_count(&status, MPI_FLOAT, &count); PRINTDEBUG("%3d: cnt=%d\n", cartrank, count); MPI_File_close(&fhtout); MPI_Comm_free(&cartcomm); MPI_Type_free(&filetype); // free(tilebuffer); } MPI_Comm_free(&tilecomm); #endif }
int run_test (test_param_t *test) { ADIO_Offset st_offset, end_offset; MPI_File fh; int is_contig; int ind_err = 0, exp_err = 0; MPI_Datatype filetype; MPI_Type_struct (test->type_count, test->type_blocklens, test->type_indices, test->type_oldtypes, &filetype); MPI_Type_commit (&filetype); MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL); MPI_File_seek (fh, test->offset, MPI_SEEK_SET); ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE, ADIO_INDIVIDUAL, test->offset, &st_offset, &end_offset); ind_err = 0; if (st_offset != test->correct_st_offset) { printf ("Individual st_offset = %lld end_offset = %lld\n", st_offset, end_offset); ind_err = 1; } if (end_offset != test->correct_end_offset) { printf ("Individual st_offset = %lld end_offset = %lld\n", st_offset, end_offset); ind_err = 1; } MPI_File_close (&fh); if (ind_err) printf ("Individual Calc FAILED\n"); MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR, MPI_INFO_NULL, &fh); if (!is_contig) MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL); MPI_File_seek (fh, 0, MPI_SEEK_SET); ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE, ADIO_EXPLICIT_OFFSET, test->offset, &st_offset, &end_offset); exp_err = 0; if (st_offset != test->correct_st_offset) { printf ("Explicit st_offset = %lld end_offset = %lld\n", st_offset, end_offset); exp_err = 1; } if (end_offset != test->correct_end_offset) { printf ("Explicit st_offset = %lld end_offset = %lld\n", st_offset, end_offset); exp_err = 1; } if (exp_err) printf ("Explicit Calc FAILED\n"); MPI_File_close (&fh); if (!is_contig) MPI_Type_free (&filetype); return (exp_err || ind_err); }
int main(int argc, char **argv) { int rank, size; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if(argc != 4 && rank == 0) { fprintf(stderr, "Usage: %s str1_file str2_file out_file\n", argv[0]); MPI_Abort(MPI_COMM_WORLD, 1); } if(size == 1) { fprintf(stderr, "At least 2 processes expected\n"); MPI_Abort(MPI_COMM_WORLD, 1); } MPI_Barrier(MPI_COMM_WORLD); char *alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; int alphabet_len = strlen(alphabet); #ifdef DEBUG_TIME double start_t, end_t; start_t = end_t = 0; // suppress warnings about uninitialized variables if(rank == 0) start_t = MPI_Wtime(); #endif // open files with str1 and str2 and read it's contents char *filename_str1 = argv[1]; char *filename_str2 = argv[2]; char *filename_out = argv[3]; MPI_File F_str1, F_str2; MPI_File_open(MPI_COMM_WORLD, filename_str1, MPI_MODE_RDONLY, MPI_INFO_NULL, &F_str1); MPI_File_open(MPI_COMM_WORLD, filename_str2, MPI_MODE_RDONLY, MPI_INFO_NULL, &F_str2); // get lengths of str1 and str2 MPI_Offset str1_len, str2_len; MPI_File_get_size(F_str1, &str1_len); MPI_File_get_size(F_str2, &str2_len); char *str1 = (char*)malloc(sizeof(char) * (str1_len + 1)); char *str2 = (char*)malloc(sizeof(char) * (str2_len + 1)); // now read str1 and str2 from files MPI_File_read_at(F_str1, (MPI_Offset)0, str1, str1_len, MPI_CHAR, MPI_STATUS_IGNORE); str1[str1_len] = '\0'; MPI_File_read_at(F_str2, (MPI_Offset)0, str2, str2_len, MPI_CHAR, MPI_STATUS_IGNORE); str2[str2_len] = '\0'; MPI_File_close(&F_str1); MPI_File_close(&F_str2); #ifdef DEBUG_TIME if(rank == 0) { end_t = MPI_Wtime(); printf("%f seconds for reading from files\n", end_t - start_t); } #endif int i; int start_index, end_index; int chunk_size = str2_len / (size - 1); if(rank > 0) { // every rank>0 processes part of str2 // then rank=0 reduces results start_index = (rank - 1) * chunk_size; end_index = rank * chunk_size - 1; if(rank == size - 1) // last rank processes everything, what is left end_index = str2_len - 1; int *appearances = (int*)malloc(sizeof(int) * alphabet_len); // appearances[0] corresponds to 'A' // appearances[25] - 'Z' memset(appearances, 0, sizeof(int) * alphabet_len); int letter; for(i = start_index; i <= end_index; i++) { letter = str2[i] - 'A'; appearances[letter] += 1; } MPI_Gatherv(appearances, alphabet_len, MPI_INT, NULL, NULL, NULL, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(appearances, alphabet_len, MPI_INT, 0, MPI_COMM_WORLD); // now every rank>0 contains _global_ count of how many times every letter from alphabet appears in str2 int *ds[alphabet_len], ds_length[alphabet_len]; // ds=decreasing sequence for(i = 0; i < alphabet_len; i++) { ds[i] = (int*)malloc(sizeof(int) * appearances[i]); // ds[0] corresponds to letter 'A' // if str2='bacdeafa', then ds[0]={8, 6, 2} memset(ds[i], 0, sizeof(int) * appearances[i]); ds_length[i] = 0; } // every rank>0 processes it's part of str2 // but now rank=1 processes _last_ part of str2 // for example, if str2='abcdefghj', size=4 (including rank=0) // then rank=1 processes 'ghj', rank=2 - 'def', rank=3 - 'abc' start_index = (size - (rank + 1)) * chunk_size; end_index = (size - rank) * chunk_size - 1; if(rank == 1) { end_index = str2_len - 1; } for(i = end_index; i >= start_index; i--) { letter = str2[i] - 'A'; ds[letter][ds_length[letter]] = i; ds_length[letter] += 1; } // decreasing sequence is done, send results to rank=0 MPI_Gather(ds_length, alphabet_len, MPI_INT, NULL, 0, MPI_INT, 0, MPI_COMM_WORLD); // now send all ds's for(i = 0; i < alphabet_len; i++) { MPI_Gatherv(ds[i], ds_length[i], MPI_INT, NULL, NULL, NULL, MPI_INT, 0, MPI_COMM_WORLD); } // cleanup for(i = 0; i < alphabet_len; i++) { free(ds[i]); } free(appearances); } if(rank == 0) { #ifdef DEBUG_TIME start_t = MPI_Wtime(); #endif int *appearances = (int*)malloc(sizeof(int) * ((size - 1) * alphabet_len + 1)); // '+ 1' to receive value from rank=0 // but it is unused int *recvcounts = (int*)malloc(sizeof(int) * size); int *displs = (int*)malloc(sizeof(int) * size); recvcounts[0] = 1; displs[0] = 0; for(i = 1; i < size; i++) { start_index = (i - 1) * chunk_size; //end_index = i * chunk_size - 1; //if(i == size - 1) // end_index = str2_len - 1; recvcounts[i] = alphabet_len; displs[i] = alphabet_len * (i - 1) + 1; } MPI_Gatherv(MPI_IN_PLACE, 1, MPI_INT, appearances, recvcounts, displs, MPI_INT, 0, MPI_COMM_WORLD); // reduce data from all processes int k; int letter_displacement; for(i = 1; i <= alphabet_len; i++) { letter_displacement = i - 1; for(k = 2; k < size; k++) { appearances[i] += appearances[displs[k] + letter_displacement]; } } #ifdef DEBUG_TIME end_t = MPI_Wtime(); printf("%f seconds for calculation of how many times every letter appears in str2\n", end_t - start_t); start_t = MPI_Wtime(); #endif // now appearances[1] through appearances[26] contain _global_ count of how many times every letter appears in str2 // broadcast it to all processes MPI_Bcast(appearances + 1, alphabet_len, MPI_INT, 0, MPI_COMM_WORLD); // receive decreasing sequences' lengths int ds_lengths[alphabet_len * size]; // rank=0 sends 26 MPI_INTs, so ds_lengths[0] through ds_lengths[25] are not significant MPI_Gather(MPI_IN_PLACE, alphabet_len, MPI_INT, ds_lengths, alphabet_len, MPI_INT, 0, MPI_COMM_WORLD); int *ds[alphabet_len]; recvcounts[0] = 1; displs[0] = 0; for(i = 0; i < alphabet_len; i++) { // only appearances[1] through appearances[26] contain _global_ count ds[i] = (int*)malloc(sizeof(int) * (appearances[i + 1] + 1)); // '+1' to receive value from rank=0 for(k = 1; k < size; k++) { recvcounts[k] = ds_lengths[alphabet_len * k + i]; //k=1 - rank=1. ds_length[26] - 'A', ds_lengths[27] - 'B' //k=2 - rank=2, ds_length[52] - 'A', ds_lengths[53] - 'B' if(k == 1) { displs[k] = 1; } else { displs[k] = ds_lengths[alphabet_len * (k - 1) + i] + displs[k - 1]; } } MPI_Gatherv(MPI_IN_PLACE, 1, MPI_INT, ds[i], recvcounts, displs, MPI_INT, 0, MPI_COMM_WORLD); } #ifdef DEBUG_TIME end_t = MPI_Wtime(); printf("%f seconds for creation of decreasing sequences\n", end_t - start_t); start_t = MPI_Wtime(); #endif int letter_appearances; int dec_sequences_count = 1; int **dec_sequences = (int**)malloc(sizeof(int**) * 1); dec_sequences[0] = (int*)malloc(sizeof(int) * appearances[str1[0] - 'A' + 1]); int *dec_sequences_lengths = (int*)malloc(sizeof(int) * 1); dec_sequences_lengths[0] = appearances[str1[0] - 'A' + 1]; for(i = 0; i < dec_sequences_lengths[0]; i++) { dec_sequences[0][i] = ds[(int)(str1[0] - 'A')][i + 1]; } int seq; int letter; for(i = 1; i < str1_len; i++) { letter = str1[i] - 'A'; letter_appearances = appearances[letter + 1]; // '+1' because appearances[0] is not valid for(k = 1; k <= letter_appearances; k++) { seq = lower_bound(dec_sequences, dec_sequences_lengths, dec_sequences_count, ds[letter][k]); if(seq >= 0) { dec_sequences_lengths[seq] += 1; dec_sequences[seq] = (int*)realloc(dec_sequences[seq], sizeof(int) * dec_sequences_lengths[seq]); // it is bad, i know dec_sequences[seq][dec_sequences_lengths[seq] - 1] = ds[letter][k]; // but would not rewrite this, because it will still be slower } else { // than lcs_sequential dec_sequences_count += 1; dec_sequences_lengths = (int*)realloc(dec_sequences_lengths, sizeof(int) * dec_sequences_count); dec_sequences_lengths[dec_sequences_count - 1] = 1; dec_sequences = (int**)realloc(dec_sequences, sizeof(int**) * dec_sequences_count); dec_sequences[dec_sequences_count - 1] = (int*)malloc(sizeof(int) * 1); dec_sequences[dec_sequences_count - 1][0] = ds[letter][k]; } } } #ifdef DEBUG_TIME end_t = MPI_Wtime(); printf("%f seconds for LCS creation\n", end_t - start_t); #endif FILE *F_out = fopen(filename_out, "w"); if(!F_out) { fprintf(stderr, "Unable to open file '%s' for writing\nOutput to console...\n", filename_out); F_out = stdout; } fprintf(F_out, "lcs length = %d\n", dec_sequences_count); fprintf(F_out, "lcs sequence\n"); for(i = 0; i < dec_sequences_count; i++) { fprintf(F_out, "%c", str2[dec_sequences[i][dec_sequences_lengths[i] - 1]]); } fprintf(F_out, "\n"); fclose(F_out); // cleanup free(dec_sequences_lengths); for(i = 0; i < dec_sequences_count; i++) { free(dec_sequences[i]); } free(dec_sequences); for(i = 0; i < alphabet_len; i++) { free(ds[i]); } free(appearances); free(recvcounts); free(displs); } free(str1); free(str2); MPI_Finalize(); return 0; }
int main(int argc, char *argv[]) { char *filename; int i, len, nprocs, amode, err, nerrs = 0; int blen[2], disp[2]; MPI_Datatype etype, filetype; MPI_File fh; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (nprocs != 1) { fprintf(stderr, "Run this program on 1 process\n"); MPI_Abort(MPI_COMM_WORLD, 1); } i = 1; while ((i < argc) && strcmp("-fname", *argv)) { i++; argv++; } if (i >= argc) { len = 8; filename = (char *) malloc(len + 10); strcpy(filename, "testfile"); } else { argv++; len = (int) strlen(*argv); filename = (char *) malloc(len + 1); strcpy(filename, *argv); } MPI_File_delete(filename, MPI_INFO_NULL); amode = MPI_MODE_RDWR | MPI_MODE_CREATE; err = MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh); CHECK_ERROR(err, nerrs); /* create etype with negative disp */ disp[0] = -2; disp[1] = 2; blen[0] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&etype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&etype); CHECK_ERROR(err, nerrs); /* create etype with decreasing disp */ disp[0] = 3; blen[0] = 1; disp[1] = 0; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&etype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&etype); CHECK_ERROR(err, nerrs); /* create etype with overlaps */ disp[0] = 0; blen[0] = 3; disp[1] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&etype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&etype); CHECK_ERROR(err, nerrs); /* create filetype with negative disp */ disp[0] = -2; disp[1] = 2; blen[0] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&filetype); CHECK_ERROR(err, nerrs); /* create filetype with decreasing disp */ disp[0] = 3; blen[0] = 1; disp[1] = 0; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&filetype); CHECK_ERROR(err, nerrs); /* create filetype with overlaps */ disp[0] = 0; blen[0] = 3; disp[1] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_close(&fh); CHECK_ERROR(err, nerrs); /* open the file for read only */ amode = MPI_MODE_RDONLY; err = MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh); CHECK_ERROR(err, nerrs); /* create etype with negative disp */ disp[0] = -2; disp[1] = 2; blen[0] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&etype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&etype); CHECK_ERROR(err, nerrs); /* create etype with decreasing disp */ disp[0] = 3; blen[0] = 1; disp[1] = 0; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&etype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&etype); CHECK_ERROR(err, nerrs); /* create etype with overlaps (should be OK for read-only) */ disp[0] = 0; blen[0] = 3; disp[1] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&etype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL); CHECK_ERROR(err, nerrs); err = MPI_Type_free(&etype); CHECK_ERROR(err, nerrs); /* create filetype with negative disp */ disp[0] = -2; disp[1] = 2; blen[0] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&filetype); CHECK_ERROR(err, nerrs); /* create filetype with decreasing disp */ disp[0] = 3; blen[0] = 1; disp[1] = 0; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs); err = MPI_Type_free(&filetype); CHECK_ERROR(err, nerrs); /* create filetype with overlaps (should be OK for read-only) */ disp[0] = 0; blen[0] = 3; disp[1] = 1; blen[1] = 1; err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype); CHECK_ERROR(err, nerrs); err = MPI_Type_commit(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_ERROR(err, nerrs); err = MPI_Type_free(&filetype); CHECK_ERROR(err, nerrs); err = MPI_File_close(&fh); CHECK_ERROR(err, nerrs); if (nerrs == 0) printf(" No Errors\n"); MPI_Finalize(); return (nerrs > 0); }
int main(int argc, char *argv[]) { int i, n, nlocal; int numprocs, myrank; MPI_File f; char* filename = "input/8"; MPI_Status status; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &numprocs); MPI_Comm_rank(MPI_COMM_WORLD, &myrank); if(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &f) != MPI_SUCCESS) { fprintf(stderr, "Cannot open file %s\n", filename); MPI_Abort(MPI_COMM_WORLD, FILE_NOT_FOUND); MPI_Finalize(); return 1; } MPI_File_seek(f, 0, MPI_SEEK_SET); MPI_File_read(f, &n, 1, MPI_INT, &status); nlocal = n/numprocs; if(myrank == numprocs - 1) nlocal = nlocal + n % numprocs; int *a = (int *)malloc(nlocal * n * sizeof(int)); MPI_File_seek(f, (myrank * nlocal * n + 1) * sizeof(int), MPI_SEEK_SET); MPI_File_read(f, &a[0], nlocal * n, MPI_INT, &status); MPI_File_close(&f); // int j; // if(myrank == 3) { // for(i = 0; i < nlocal; i++) { // for(j = 0; j < n; j++) { // printf("%d ", a[i * n +j]); // } // printf("\n"); // } // } double start = MPI_Wtime(); floyd_all_pairs_sp_1d(n, nlocal, a); double stop = MPI_Wtime(); printf("[%d] Completed in %1.3f seconds\n", myrank, stop-start); // if(myrank == 3) { // for(i = 0; i < nlocal; i++) { // for(j = 0; j < n; j++) { // printf("%d ", a[i * n +j]); // } // printf("\n"); // } // } if(MPI_File_open(MPI_COMM_WORLD, "output/8", MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &f) != MPI_SUCCESS) { printf("Cannot open file %s\n", "out"); MPI_Abort(MPI_COMM_WORLD, FILE_NOT_FOUND); MPI_Finalize(); return 1; } if(myrank == 0) { MPI_File_seek(f, 0, MPI_SEEK_SET); MPI_File_write(f, &n, 1, MPI_INT, &status); } for(i = 0; i < nlocal; i++) { MPI_File_seek(f, (myrank * nlocal * n + 1) * sizeof(int), MPI_SEEK_SET); MPI_File_write(f, &a[0], nlocal * n, MPI_INT, &status); } MPI_File_close(&f); free(a); MPI_Finalize(); return 0; }
static void close() { if(mpiFile != MPI_FILE_NULL) MPI_File_close(&mpiFile); }
/* * Verify that MPI_Offset exceeding 2**31 can be computed correctly. * Print any failure as information only, not as an error so that this * won't abort the remaining test or other separated tests. * * Test if MPIO can write file from under 2GB to over 2GB and then * from under 4GB to over 4GB. * Each process writes 1MB in round robin fashion. * Then reads the file back in by reverse order, that is process 0 * reads the data of process n-1 and vice versa. */ static int test_mpio_gb_file(char *filename) { int mpi_size, mpi_rank; MPI_Info info = MPI_INFO_NULL; int mrc; MPI_File fh; int i, j, n; int vrfyerrs; int writerrs; /* write errors */ int nerrs; int ntimes; /* how many times */ char *buf = NULL; char expected; MPI_Offset size; MPI_Offset mpi_off; MPI_Offset mpi_off_old; MPI_Status mpi_stat; struct stat stat_buf; int is_signed, sizeof_mpi_offset; nerrs = 0; /* set up MPI parameters */ MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); if (VERBOSE_MED) printf("MPI_Offset range test\n"); /* figure out the signness and sizeof MPI_Offset */ mpi_off = 0; is_signed = ((MPI_Offset)(mpi_off - 1)) < 0; sizeof_mpi_offset = (int)(sizeof(MPI_Offset)); /* * Verify the sizeof MPI_Offset and correctness of handling multiple GB * sizes. */ if (MAINPROCESS){ /* only process 0 needs to check it*/ printf("MPI_Offset is %s %d bytes integeral type\n", is_signed ? "signed" : "unsigned", (int)sizeof(MPI_Offset)); if (sizeof_mpi_offset <= 4 && is_signed){ printf("Skipped 2GB range test " "because MPI_Offset cannot support it\n"); }else { /* verify correctness of assigning 2GB sizes */ mpi_off = 2 * 1024 * (MPI_Offset)MB; INFO((mpi_off>0), "2GB OFFSET assignment no overflow"); INFO((mpi_off-1)==TWO_GB_LESS1, "2GB OFFSET assignment succeed"); /* verify correctness of increasing from below 2 GB to above 2GB */ mpi_off = TWO_GB_LESS1; for (i=0; i < 3; i++){ mpi_off_old = mpi_off; mpi_off = mpi_off + 1; /* no overflow */ INFO((mpi_off>0), "2GB OFFSET increment no overflow"); /* correct inc. */ INFO((mpi_off-1)==mpi_off_old, "2GB OFFSET increment succeed"); } } if (sizeof_mpi_offset <= 4){ printf("Skipped 4GB range test " "because MPI_Offset cannot support it\n"); }else { /* verify correctness of assigning 4GB sizes */ mpi_off = 4 * 1024 * (MPI_Offset)MB; INFO((mpi_off>0), "4GB OFFSET assignment no overflow"); INFO((mpi_off-1)==FOUR_GB_LESS1, "4GB OFFSET assignment succeed"); /* verify correctness of increasing from below 4 GB to above 4 GB */ mpi_off = FOUR_GB_LESS1; for (i=0; i < 3; i++){ mpi_off_old = mpi_off; mpi_off = mpi_off + 1; /* no overflow */ INFO((mpi_off>0), "4GB OFFSET increment no overflow"); /* correct inc. */ INFO((mpi_off-1)==mpi_off_old, "4GB OFFSET increment succeed"); } } } /* * Verify if we can write to a file of multiple GB sizes. */ if (VERBOSE_MED) printf("MPIO GB file test %s\n", filename); if (sizeof_mpi_offset <= 4){ printf("Skipped GB file range test " "because MPI_Offset cannot support it\n"); }else{ buf = malloc(MB); VRFY((buf!=NULL), "malloc succeed"); /* open a new file. Remove it first in case it exists. */ /* Must delete because MPI_File_open does not have a Truncate mode. */ /* Don't care if it has error. */ MPI_File_delete(filename, MPI_INFO_NULL); MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */ mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_OPEN"); printf("MPIO GB file write test %s\n", filename); /* instead of writing every bytes of the file, we will just write * some data around the 2 and 4 GB boundaries. That should cover * potential integer overflow and filesystem size limits. */ writerrs = 0; for (n=2; n <= 4; n+=2){ ntimes = GB/MB*n/mpi_size + 1; for (i=ntimes-2; i <= ntimes; i++){ mpi_off = (i*mpi_size + mpi_rank)*(MPI_Offset)MB; if (VERBOSE_MED) HDfprintf(stdout,"proc %d: write to mpi_off=%016llx, %lld\n", mpi_rank, mpi_off, mpi_off); /* set data to some trivial pattern for easy verification */ for (j=0; j<MB; j++) *(buf+j) = i*mpi_size + mpi_rank; if (VERBOSE_MED) HDfprintf(stdout,"proc %d: writing %d bytes at offset %lld\n", mpi_rank, MB, mpi_off); mrc = MPI_File_write_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat); INFO((mrc==MPI_SUCCESS), "GB size file write"); if (mrc!=MPI_SUCCESS) writerrs++; } } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync after writes"); /* * Verify if we can read the multiple GB file just created. */ /* open it again to verify the data written */ /* but only if there was no write errors */ printf("MPIO GB file read test %s\n", filename); if (errors_sum(writerrs)>0){ printf("proc %d: Skip read test due to previous write errors\n", mpi_rank); goto finish; } mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh); VRFY((mrc==MPI_SUCCESS), ""); /* Only read back parts of the file that have been written. */ for (n=2; n <= 4; n+=2){ ntimes = GB/MB*n/mpi_size + 1; for (i=ntimes-2; i <= ntimes; i++){ mpi_off = (i*mpi_size + (mpi_size - mpi_rank - 1))*(MPI_Offset)MB; if (VERBOSE_MED) HDfprintf(stdout,"proc %d: read from mpi_off=%016llx, %lld\n", mpi_rank, mpi_off, mpi_off); mrc = MPI_File_read_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat); INFO((mrc==MPI_SUCCESS), "GB size file read"); expected = i*mpi_size + (mpi_size - mpi_rank - 1); vrfyerrs=0; for (j=0; j<MB; j++){ if ((*(buf+j) != expected) && (vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED)){ printf("proc %d: found data error at [%ld+%d], expect %d, got %d\n", mpi_rank, (long)mpi_off, j, expected, *(buf+j)); } } if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED) printf("proc %d: [more errors ...]\n", mpi_rank); nerrs += vrfyerrs; } } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); /* * one more sync to ensure all processes have done reading * before ending this test. */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync before leaving test"); /* * Check if MPI_File_get_size works correctly. Some systems (only SGI Altix * Propack 4 so far) return wrong file size. It can be avoided by reconfiguring * with "--disable-mpi-size". */ #ifdef H5_HAVE_MPI_GET_SIZE printf("Test if MPI_File_get_size works correctly with %s\n", filename); mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh); VRFY((mrc==MPI_SUCCESS), ""); if (MAINPROCESS){ /* only process 0 needs to check it*/ mrc = MPI_File_get_size(fh, &size); VRFY((mrc==MPI_SUCCESS), ""); mrc=stat(filename, &stat_buf); VRFY((mrc==0), ""); /* Hopefully this casting is safe */ if(size != (MPI_Offset)(stat_buf.st_size)) { printf("Warning: MPI_File_get_size doesn't return correct file size. To avoid using it in the library, reconfigure and rebuild the library with --disable-mpi-size.\n"); } } /* close file and free the communicator */ mrc = MPI_File_close(&fh); VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE"); /* * one more sync to ensure all processes have done reading * before ending this test. */ mrc = MPI_Barrier(MPI_COMM_WORLD); VRFY((mrc==MPI_SUCCESS), "Sync before leaving test"); #else printf("Skipped testing MPI_File_get_size because it's disabled\n"); #endif } finish: if (buf) HDfree(buf); return (nerrs); }
int main(int argc, char *argv[] ) { double time1, time2; time1 = MPI_Wtime(); int rank, processors; int j; // number of iterations int k; // number of iterations to perform before creating a checkpoint int l; // number of random samples per grid point int checkpoint_resume = 0; // 1 = resume from last checkpoint int c; // used to hold a character int i=0, row = 0, col = 0, pln = 0; // array iterators char ***local_array; char **local_array_2nd; char *local_array_pointer; char ***local_array_copy; char **local_array_copy_2nd; char *local_array_copy_pointer; char ***temp, *temp_pointer; int file_open_error; int command_line_incomplete = 0; int grid_size[3] = {0,0,0}; int proc_size[3] = {0,0,0}; int local_size[3] = {0,0,0}; int remainder_size[3] = {0,0,0}; int coords[3] = {0,0,0}; int start_indices[3] = {0,0,0}; int periods[3] = {0,0,0}; int mem_size[3] = {0,0,0}; MPI_Status status; MPI_Datatype filetype, memtype; MPI_File fh; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &processors); MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Interpret the command line arguments -------------------------------- if (rank == 0) { if (argc < 6 || argc > 8) { fputs("usage: x y z j k l r\n", stderr); fputs("where: x,y,z = x, y and z dimensions\n", stderr); fputs(" j = how many times the game of life is played\n", stderr); fputs(" k = checkpoint every k iterations\n", stderr); fputs(" l = number of random samples per grid point\n", stderr); fputs(" r = resume from the last checkpoint\n", stderr); fputs(INITIAL, stderr); fputs(" must be present.\n", stderr); fputs(CHECKPOINT, stderr); fputs(" must be present if resuming from the last checkpoint.\n", stderr); exit(EXIT_FAILURE); } } j = (int) strtol(argv[4], NULL, 10); k = (int) strtol(argv[5], NULL, 10); l = (int) strtol(argv[6], NULL, 10); if ( argc == 7 ) if ( argv[6][0] == 'r' ) checkpoint_resume = 1; if (rank == 0) printf("%d iterations \ncheckpoint every %d iterations \n%d samples per grid point \ncheckpoint resume = %d\n", j,k,l,checkpoint_resume); grid_size[0] = (int) strtol(argv[1], NULL, 10); grid_size[1] = (int) strtol(argv[2], NULL, 10); grid_size[2] = (int) strtol(argv[3], NULL, 10); if (rank==0) printf("grid_size: %d, %d, %d\n", grid_size[0], grid_size[1], grid_size[2]); MPI_Dims_create(processors, 3, proc_size); if (rank==0) printf("proc_size: %d, %d, %d\n", proc_size[0], proc_size[1], proc_size[2]); local_size[0] = grid_size[0] / proc_size[0]; local_size[1] = grid_size[1] / proc_size[1]; local_size[2] = grid_size[2] / proc_size[2]; if (rank==0) printf("local_size: %d, %d, %d\n", local_size[0], local_size[1], local_size[2]); remainder_size[0] = grid_size[0] % proc_size[0]; remainder_size[1] = grid_size[1] % proc_size[1]; remainder_size[2] = grid_size[2] % proc_size[2]; if (rank==0) printf("remainder_size: %d, %d, %d\n", remainder_size[0], remainder_size[1], remainder_size[2]); if (remainder_size[0] != 0 || remainder_size[1] != 0 || remainder_size[2] != 0) { fputs("remainder size != 0, check your dimensions", stderr); MPI_Finalize(); exit(EXIT_FAILURE); } MPI_Comm comm; MPI_Cart_create(MPI_COMM_WORLD, 3, proc_size, periods, 0, &comm); MPI_Comm_rank(comm, &rank); MPI_Cart_coords(comm, rank, 3, coords); start_indices[0] = coords[0] * local_size[0]; start_indices[1] = coords[1] * local_size[1]; start_indices[2] = coords[2] * local_size[2]; /* printf("A coords R%d: (%d, %d, %d) (%d, %d, %d)\n", rank, coords[0], coords[1], coords[2], start_indices[0], start_indices[1], start_indices[2]);*/ fflush(stdout); // create the file type --------------------------------------------------- MPI_Type_create_subarray(3, grid_size, local_size, start_indices, MPI_ORDER_C, MPI_CHAR, &filetype); MPI_Type_commit(&filetype); // create a local memory type with ghost rows ----------------------------- mem_size[0] = local_size[0] + 2; mem_size[1] = local_size[1] + 2; mem_size[2] = local_size[2] + 2; start_indices[0] = start_indices[1] = start_indices[2] = 1; MPI_Type_create_subarray(3, mem_size, local_size, start_indices, MPI_ORDER_C, MPI_CHAR, &memtype); MPI_Type_commit(&memtype); // find my neighbors ------------------------------------------------------ int nxminus, nxplus, nyminus, nyplus, nzminus, nzplus, tag = 333, *neighbors; // Neighbors Array: row- col- col+ row+ plane- plane+ neighbors = (int *) malloc(6 * sizeof(int)); for(i=0; i<6; i++) neighbors[i] = rank; MPI_Cart_shift(comm, 0, 1, &nxminus, &nxplus); MPI_Cart_shift(comm, 1, 1, &nyminus, &nyplus); MPI_Cart_shift(comm, 2, 1, &nzminus, &nzplus); // printf(" %d sending south to %d receiving from %d \n",rank,nxplus,nxminus); // fflush(stdout); MPI_Sendrecv(&rank, 1, MPI_INT, nxplus, tag, &(neighbors[0]), 1, MPI_INT, nxminus, tag, comm, &status); // printf(" %d sending North to %d receiving from %d \n",rank,nxminus,nxplus); // fflush(stdout); MPI_Sendrecv(&rank, 1, MPI_INT, nxminus, tag, &(neighbors[3]), 1, MPI_INT, nxplus, tag, comm, &status); // printf(" %d sending East to %d receiving from %d \n",rank,nyplus,nyminus); // fflush(stdout); MPI_Sendrecv(&rank, 1, MPI_INT, nyplus, tag, &neighbors[1], 1, MPI_INT, nyminus, tag, comm, &status); // printf(" %d sending West to %d receiving from %d \n",rank,nyminus,nyplus); // fflush(stdout); MPI_Sendrecv(&rank, 1, MPI_INT, nyminus, tag, &neighbors[2], 1, MPI_INT, nyplus, tag, comm, &status); // printf(" %d sending backwards to %d receiving from %d \n",rank,nzplus,nzminus); // fflush(stdout); MPI_Sendrecv(&rank, 1, MPI_INT, nzplus, tag, &(neighbors[4]), 1, MPI_INT, nzminus, tag, comm, &status); // printf(" %d sending forward to %d receiving from %d \n",rank,nzminus,nzplus); // fflush(stdout); MPI_Sendrecv(&rank, 1, MPI_INT, nzminus, tag, &(neighbors[5]), 1, MPI_INT, nzplus, tag, comm, &status); /* printf("neighboors R%d : (row-) %d (col-) %d (col+) %d (row+) %d (plane-) %d (plane+) %d\n",rank,neighbors[0],neighbors[1],neighbors[2],neighbors[3],neighbors[4],neighbors[5]);*/ fflush(stdout); //init_sprng(1,time(0),SPRNG_DEFAULT); srand((unsigned int)time(NULL)); // Open the initial condition (checkpoint or not) ---------------------- if ( checkpoint_resume ) { file_open_error = MPI_File_open(MPI_COMM_WORLD, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_set_view(fh,0, MPI_CHAR, filetype, "native", MPI_INFO_NULL); } else { file_open_error = MPI_File_open(MPI_COMM_WORLD, INITIAL, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_set_view(fh,0, MPI_CHAR, filetype, "native", MPI_INFO_NULL); } if (file_open_error != MPI_SUCCESS) { if (checkpoint_resume) fputs(CHECKPOINT, stderr); else fputs(INITIAL, stderr); fputs(" could not be opened.\n", stderr); exit(EXIT_FAILURE); } // Allocate and Populate the local array ---------------------------------- local_array_copy_pointer = (char *) malloc(mem_size[0] * mem_size[1] * mem_size[2] * sizeof(char)); local_array_copy_2nd = (char **) malloc(mem_size[0] * mem_size[1] * sizeof(char*)); local_array_copy = (char ***) malloc(mem_size[0] * sizeof(char*)); for(i = 0; i < mem_size[0] * mem_size[1]; i++) local_array_copy_2nd[i] = &local_array_copy_pointer[i * mem_size[2]]; for(i = 0; i < mem_size[0]; i++) local_array_copy[i] = &local_array_copy_2nd[i * mem_size[1]]; local_array_pointer = (char *) malloc(mem_size[0] * mem_size[1] * mem_size[2] * sizeof(char)); local_array_2nd = (char **) malloc(mem_size[0] * mem_size[1] * sizeof(char*)); local_array = (char ***) malloc(mem_size[0] * sizeof(char*)); for(i = 0; i < mem_size[0] * mem_size[1]; i++) local_array_2nd[i] = &local_array_pointer[i * mem_size[2]]; for(i = 0; i < mem_size[0]; i++) local_array[i] = &local_array_2nd[i * mem_size[1]]; // if (rank==0) printf("Malloc complete\n"); for(row=0; row<mem_size[0]; row++) { for(col=0; col<mem_size[1]; col++) { for(pln=0; pln<mem_size[2]; pln++) { local_array[row][col][pln] = local_array_copy[row][col][pln] = '0'; } } } // if (rank==0) printf("Setup complete\n"); MPI_File_read_all(fh, local_array_pointer, 1, memtype, &status); if (rank==0) printf("File Read\n"); // if (rank==0) { // for(row=0; row<mem_size[0]; row++) { // for(col=0; col<mem_size[1]; col++) { // for(pln=0; pln<mem_size[2]; pln++) { // printf("%c", local_array[row][col][pln]); // } // printf("\n"); // } // printf("-----------------------\n"); // } // } MPI_File_close(&fh); // Construct the plane data types MPI_Datatype yzplane; MPI_Type_vector(local_size[1], local_size[2], local_size[2]+2, MPI_CHAR, &yzplane); MPI_Type_commit(&yzplane); MPI_Datatype xzplane; MPI_Type_vector(local_size[0], local_size[2], ((local_size[2]+2)*local_size[1])+((local_size[2]+2)*2), MPI_CHAR, &xzplane); MPI_Type_commit(&xzplane); // this type will also copy the corner x columns, can't skip blocks intermittently // since we aren't worrying about the corner data, it's ok MPI_Datatype xyplane; MPI_Type_vector((local_size[0]*local_size[1])+((local_size[0]*2)-2), 1, local_size[2]+2, MPI_CHAR, &xyplane); MPI_Type_commit(&xyplane); MPI_Barrier(comm); // start the iteration loop int iterations; int kCounter = k; for (iterations = 0; iterations < j; iterations++) { // send updated planes // Neighbors Array: // 0 1 2 3 4 5 // row- col- col+ row+ plane- plane+ // Note: corners are not handled // send top yzplane if (rank != neighbors[0]) MPI_Send(&local_array[1][1][1], 1, yzplane, neighbors[0], 0, comm); // recv bottom yzplane if (rank != neighbors[3]) MPI_Recv(&local_array[local_size[0]+1][1][1], 1, yzplane, neighbors[3], 0, comm, &status); // send bottom yzplane if (rank != neighbors[3]) MPI_Send(&local_array[local_size[0]][1][1], 1, yzplane, neighbors[3], 0, comm); // recv top yzplane if (rank != neighbors[0]) MPI_Recv(&local_array[0][1][1], 1, yzplane, neighbors[0], 0, comm, &status); // send left xzplane if (rank != neighbors[1]) MPI_Send(&local_array[1][1][1], 1, xzplane, neighbors[1], 0, comm); // recv right xzplane if (rank != neighbors[2]) MPI_Recv(&local_array[1][local_size[1]+1][1], 1, xzplane, neighbors[2], 0, comm, &status); // send right xzplane if (rank != neighbors[2]) MPI_Send(&local_array[1][local_size[1]][1], 1, xzplane, neighbors[2], 0, comm); // recv left xzplane if (rank != neighbors[1]) MPI_Recv(&local_array[1][0][1], 1, xzplane, neighbors[1], 0, comm, &status); // send front xyplane if (rank != neighbors[4]) MPI_Send(&local_array[1][1][1], 1, xyplane, neighbors[4], 0, comm); // recv back xyplane if (rank != neighbors[5]) MPI_Recv(&local_array[1][1][local_size[2]+1], 1, xyplane, neighbors[5], 0, comm, &status); // send back xyplane if (rank != neighbors[5]) MPI_Send(&local_array[1][1][local_size[2]], 1, xyplane, neighbors[5], 0, comm); // recv front xyplane if (rank != neighbors[4]) MPI_Recv(&local_array[1][1][0], 1, xyplane, neighbors[4], 0, comm, &status); // if (rank==0) { // for(row=0; row<mem_size[0]; row++) { // for(col=0; col<mem_size[1]; col++) { // for(pln=0; pln<mem_size[2]; pln++) { // printf("%c", local_array[row][col][pln]); // } // printf("\n"); // } // printf("-----------------------\n"); // } // } // run the game of life // gameOfLife(local_array, local_array_copy, local_size[0], local_size[1], l, rank); // swap the arrays // temp1 = local_array; // local_array = local_array_copy; // local_array_copy = temp1; // // temp2 = local_array_pointer; // local_array_pointer = local_array_copy_pointer; // local_array_copy_pointer = temp2; // check to see if this iteration needs a checkpoint kCounter--; if (kCounter == 0) { kCounter = k; // checkpoint code MPI_File_open(MPI_COMM_WORLD, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); MPI_File_set_view(fh, 0, MPI_CHAR, filetype, "native", MPI_INFO_NULL); MPI_File_write_all(fh, local_array_pointer, 1, memtype, &status); MPI_File_close(&fh); if (rank == 0) printf("Checkpoint made: Iteration %d\n", iterations+1); } // end if kCounter == 0 } // end iteration loop iterations--; // all done! repeat the checkpoint process MPI_File_open(MPI_COMM_WORLD, FINAL_RESULTS, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); MPI_File_set_view(fh, 0, MPI_CHAR, filetype, "native", MPI_INFO_NULL); MPI_File_write_all(fh, local_array_pointer, 1, memtype, &status); MPI_File_close(&fh); if (rank == 0) printf("Final Results made: Iteration %d\n", iterations+1); time2 = MPI_Wtime(); if (rank == 0) printf("Elapsed Seconds: %f\n", time2-time1);fflush(stdout); MPI_Finalize(); return EXIT_SUCCESS; }
static int test_mpio_1wMr(char *filename, int special_request) { char hostname[128]; int mpi_size, mpi_rank; MPI_File fh; char mpi_err_str[MPI_MAX_ERROR_STRING]; int mpi_err_strlen; int mpi_err; unsigned char writedata[DIMSIZE], readdata[DIMSIZE]; unsigned char expect_val; int i, irank; int nerrs = 0; /* number of errors */ int atomicity; MPI_Offset mpi_off; MPI_Status mpi_stat; MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); if (MAINPROCESS && VERBOSE_MED){ printf("Testing one process writes, all processes read.\n"); printf("Using %d processes accessing file %s\n", mpi_size, filename); printf(" (Filename can be specified via program argument)\n"); } /* show the hostname so that we can tell where the processes are running */ if (VERBOSE_DEF){ if (gethostname(hostname, 128) < 0){ PRINTID; printf("gethostname failed\n"); return 1; } PRINTID; printf("hostname=%s\n", hostname); } /* Delete any old file in order to start anew. */ /* Must delete because MPI_File_open does not have a Truncate mode. */ /* Don't care if it has error. */ MPI_File_delete(filename, MPI_INFO_NULL); MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */ if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDWR | MPI_MODE_CREATE , MPI_INFO_NULL, &fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_open failed (%s)\n", mpi_err_str); return 1; } if (special_request & USEATOM){ /* ================================================== * Set atomcity to true (1). A POSIX compliant filesystem * should not need this. * ==================================================*/ if ((mpi_err = MPI_File_get_atomicity(fh, &atomicity)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_get_atomicity failed (%s)\n", mpi_err_str); } if (VERBOSE_HI) printf("Initial atomicity = %d\n", atomicity); if ((mpi_err = MPI_File_set_atomicity(fh, 1)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_set_atomicity failed (%s)\n", mpi_err_str); } if ((mpi_err = MPI_File_get_atomicity(fh, &atomicity)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_get_atomicity failed (%s)\n", mpi_err_str); } if (VERBOSE_HI) printf("After set_atomicity atomicity = %d\n", atomicity); } /* This barrier is not necessary but do it anyway. */ MPI_Barrier(MPI_COMM_WORLD); if (VERBOSE_HI){ PRINTID; printf("between MPI_Barrier and MPI_File_write_at\n"); } /* ================================================== * Each process calculates what to write but * only process irank(0) writes. * ==================================================*/ irank=0; for (i=0; i < DIMSIZE; i++) writedata[i] = irank*DIMSIZE + i; mpi_off = irank*DIMSIZE; /* Only one process writes */ if (mpi_rank==irank){ if (VERBOSE_HI){ PRINTID; printf("wrote %d bytes at %ld\n", DIMSIZE, (long)mpi_off); } if ((mpi_err = MPI_File_write_at(fh, mpi_off, writedata, DIMSIZE, MPI_BYTE, &mpi_stat)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n", (long) mpi_off, DIMSIZE, mpi_err_str); return 1; }; }; /* Bcast the return code and */ /* make sure all writing are done before reading. */ MPI_Bcast(&mpi_err, 1, MPI_INT, irank, MPI_COMM_WORLD); if (VERBOSE_HI){ PRINTID; printf("MPI_Bcast: mpi_err = %d\n", mpi_err); } if (special_request & USEFSYNC){ /* ================================================== * Do a file sync. A POSIX compliant filesystem * should not need this. * ==================================================*/ if (VERBOSE_HI) printf("Apply MPI_File_sync\n"); /* call file_sync to force the write out */ if ((mpi_err = MPI_File_sync(fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_sync failed (%s)\n", mpi_err_str); } MPI_Barrier(MPI_COMM_WORLD); /* call file_sync to force the write out */ if ((mpi_err = MPI_File_sync(fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_sync failed (%s)\n", mpi_err_str); } } /* This barrier is not necessary because the Bcase or File_sync above */ /* should take care of it. Do it anyway. */ MPI_Barrier(MPI_COMM_WORLD); if (VERBOSE_HI){ PRINTID; printf("after MPI_Barrier\n"); } /* ================================================== * Each process reads what process 0 wrote and verify. * ==================================================*/ irank=0; mpi_off = irank*DIMSIZE; if ((mpi_err = MPI_File_read_at(fh, mpi_off, readdata, DIMSIZE, MPI_BYTE, &mpi_stat)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); PRINTID; printf("MPI_File_read_at offset(%ld), bytes (%d), failed (%s)\n", (long) mpi_off, DIMSIZE, mpi_err_str); return 1; }; for (i=0; i < DIMSIZE; i++){ expect_val = irank*DIMSIZE + i; if (readdata[i] != expect_val){ PRINTID; printf("read data[%d:%d] got %02x, expect %02x\n", irank, i, readdata[i], expect_val); nerrs++; } } MPI_File_close(&fh); if (VERBOSE_HI){ PRINTID; printf("%d data errors detected\n", nerrs); } mpi_err = MPI_Barrier(MPI_COMM_WORLD); return nerrs; }
int main(int argc, char **argv) { MPI_File fp; LemonWriter *w; LemonReader *r; LemonRecordHeader *h; double *data; double tick, tock; double *timesRead; double *timesWrite; double stdRead = 0.0; double stdWrite = 0.0; int mpisize; int rank; char const *type; int ldsize; unsigned long long int fsize; int *hashMatch, *hashMatchAll; double const rscale = 1.0 / RAND_MAX; int ME_flag=1, MB_flag=1, status=0; int latDist[] = {0, 0, 0, 0}; int periods[] = {1, 1, 1, 1}; int locSizes[4]; int latSizes[4]; int localVol = 1; int latVol = localVol; MPI_Comm cartesian; int i, j; md5_state_t state; md5_byte_t before[16]; md5_byte_t after[16]; int L; int iters; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &mpisize); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc != 3) { usage(rank, argv); MPI_Finalize(); return 1; } L = atoi(argv[1]); if (L <= 0) usage(rank, argv); iters = atoi(argv[2]); if (iters <= 0) usage(rank, argv); timesWrite = (double*)calloc(iters, sizeof(double)); if (timesWrite == (double*)NULL) { fprintf(stderr, "ERROR: Could not allocate memory.\n"); return 1; } timesRead = (double*)calloc(iters, sizeof(double)); if (timesRead == (double*)NULL) { fprintf(stderr, "ERROR: Could not allocate memory.\n"); return 1; } hashMatch = (int*)calloc(iters, sizeof(int)); if (hashMatch == (int*)NULL) { fprintf(stderr, "ERROR: Could not allocate memory.\n"); return 1; } hashMatchAll = (int*)calloc(iters, sizeof(int)); if (hashMatchAll == (int*)NULL) { fprintf(stderr, "ERROR: Could not allocate memory.\n"); return 1; } /* Construct a Cartesian topology, adjust lattice sizes where needed */ MPI_Dims_create(mpisize, 4, latDist); for (i = 0; i < 4; ++i) { int div = (i == 3 ? (2 * L) : L) / latDist[i]; locSizes[i] = div ? div : 1; localVol *= locSizes[i]; latSizes[i] = locSizes[i] * latDist[i]; } latVol = mpisize * localVol; ldsize = localVol * 72 * sizeof(double); fsize = (unsigned long long int)latVol * 72 * sizeof(double); MPI_Cart_create(MPI_COMM_WORLD, 4, latDist, periods, 1, &cartesian); MPI_Comm_rank(cartesian, &rank); if (rank == 0) { fprintf(stdout, "Benchmark on a block of data %s in size,\n", humanForm(fsize)); fprintf(stdout, "representing a %u x %u x %u x %u lattice", latSizes[0], latSizes[1], latSizes[2], latSizes[3]); if (mpisize == 1) fprintf(stdout, ".\n\n"); else { fprintf(stdout, ",\ndistributed over %u MPI processes\n", mpisize); fprintf(stdout, "for a local %u x %u x %u x %u lattice.\n\n", locSizes[0], locSizes[1], locSizes[2], locSizes[3]); } } /* Allocate a block of memory for dummy data to write */ data = (double*)malloc(ldsize); if (data == (double*)NULL) { fprintf(stderr, "ERROR: Could not allocate memory.\n"); return 1; } srand(time(NULL) + rank); /* Start of test */ for (i = 0; i < iters; ++i) { if (rank == 0) fprintf(stdout, "Measurement %d of %d.\n", i + 1, iters); /* Create a block of dummy data to write out Fill with some random numbers to make sure we don't get coincidental matches here */ for (j = 0; j < (localVol * 72); ++j) data[j] = rscale * (double)rand(); /* Calculate a hash of the data, to check integrity against */ md5_init(&state); md5_append(&state, (md5_byte_t const *)data, ldsize); md5_finish(&state, before); /* Note that the following is the only (?) way to truncate the file with MPI */ MPI_File_open(cartesian, "benchmark.test", MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fp); MPI_File_set_size(fp, 0); w = lemonCreateWriter(&fp, cartesian); h = lemonCreateHeader(MB_flag, ME_flag, "benchmark", latVol); status = lemonWriteRecordHeader(h, w); lemonDestroyHeader(h); MPI_Barrier(cartesian); tick = MPI_Wtime(); lemonWriteLatticeParallel(w, data, 72 * sizeof(double), latSizes); tock = MPI_Wtime(); MPI_Barrier(cartesian); timesWrite[i] = tock - tick; if (rank == 0) fprintf(stdout, "Time spent writing was %4.2g s.\n", timesWrite[i]); lemonWriterCloseRecord(w); lemonDestroyWriter(w); MPI_File_close(&fp); /* Clear data to avoid an utterly failed read giving md5 hash matches from the old data */ memset(data, 0, ldsize); /* Start of reading test */ MPI_File_open(cartesian, "benchmark.test", MPI_MODE_RDONLY | MPI_MODE_DELETE_ON_CLOSE, MPI_INFO_NULL, &fp); r = lemonCreateReader(&fp, cartesian); if (lemonReaderNextRecord(r)) fprintf(stderr, "Node %d reports: next record failed.\n", rank); type = lemonReaderType(r); if (strncmp(type, "benchmark", 13)) fprintf(stderr, "Node %d reports: wrong type read.\n", rank); MPI_Barrier(cartesian); tick = MPI_Wtime(); lemonReadLatticeParallel(r, data, 72 * sizeof(double), latSizes); tock = MPI_Wtime(); timesRead[i] = tock - tick; MPI_Barrier(cartesian); if (rank == 0) fprintf(stdout, "Time spent reading was %4.2g s.\n", timesRead[i]); lemonDestroyReader(r); MPI_File_close(&fp); md5_init(&state); md5_append(&state, (md5_byte_t const *)data, ldsize); md5_finish(&state, after); hashMatch[i] = strncmp((char const *)before, (char const *)after, 16) != 0 ? 1 : 0; MPI_Reduce(hashMatch + i, hashMatchAll + i, 1, MPI_INT, MPI_SUM, 0, cartesian); if (rank == 0) { if (hashMatchAll[i] == 0) fprintf(stdout, "All nodes report that MD5 hash matches.\n\n"); else fprintf(stdout, "WARNING: MD5 hash failure detected!\n\n"); } } /* Aggregate the data */ hashMatch[0] = 0; stdWrite = timesWrite[0] * timesWrite[0]; stdRead = timesRead[0] * timesRead[0]; for (i = 1; i < iters; ++i) { hashMatchAll[0] += hashMatchAll[i]; timesWrite[0] += timesWrite[i]; stdWrite += timesWrite[i] * timesWrite[i]; timesRead[0] += timesRead[i]; stdRead += timesRead[i] * timesRead[i]; } stdWrite /= iters; stdRead /= iters; timesWrite[0] /= iters; timesRead[0] /= iters; stdWrite -= timesWrite[0] * timesWrite[0]; stdRead -= timesRead[0] * timesRead[0]; if (rank == 0) { fprintf(stdout, "Average time spent writing was %4.2e s, ", timesWrite[0]); fprintf(stdout, "with a standard deviation of %4.2e s.\n", sqrt(stdWrite)); fprintf(stdout, "Average time spent reading was %4.2e s, ", timesRead[0]); fprintf(stdout, "with a standard deviation of %4.2e s.\n\n", sqrt(stdRead)); stdWrite *= (double)fsize / (timesWrite[0] * timesWrite[0]); stdRead *= (double)fsize / (timesRead[0] * timesRead[0]); fprintf(stdout, "Average writing speed was %s/s\n", humanForm((unsigned long long int)(fsize / timesWrite[0]))); fprintf(stdout, "Average reading speed was %s/s\n", humanForm((unsigned long long int)(fsize / timesRead[0]))); if (hashMatchAll[0] == 0) fprintf(stdout, "All data hashed correctly.\n"); else fprintf(stdout, "WARNING: %d hash mismatches detected!.\n", hashMatchAll[0]); } MPI_Finalize(); free(data); free(timesWrite); free(timesRead); free(hashMatch); free(hashMatchAll); return(0); }
static int test_mpio_special_collective(char *filename) { int mpi_size, mpi_rank; MPI_File fh; MPI_Datatype etype,buftype,filetype; char mpi_err_str[MPI_MAX_ERROR_STRING]; int mpi_err_strlen; int mpi_err; char writedata[2]; char *buf; int i; int count,bufcount; int blocklens[2]; MPI_Aint offsets[2]; MPI_Offset mpi_off; MPI_Status mpi_stat; int retcode; MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); retcode = 0; /* create MPI data type */ etype = MPI_BYTE; if(mpi_rank == 0 || mpi_rank == 1) { count = DIMSIZE; bufcount = 1; } else { count = 0; bufcount = 0; } blocklens[0] = count; offsets[0] = mpi_rank*count; blocklens[1] = count; offsets[1] = (mpi_size+mpi_rank)*count; if(count !=0) { if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&filetype)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&buftype)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str); return 1; } if((mpi_err=MPI_Type_commit(&buftype))!=MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_Type_commit failed (%s)\n", mpi_err_str); return 1; } } else { filetype = MPI_BYTE; buftype = MPI_BYTE; } /* Open a file */ if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDWR | MPI_MODE_CREATE , MPI_INFO_NULL, &fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_open failed (%s)\n", mpi_err_str); return 1; } /* each process writes some data */ for (i=0; i < 2*DIMSIZE; i++) writedata[i] = mpi_rank*DIMSIZE + i; mpi_off = 0; if((mpi_err = MPI_File_set_view(fh, mpi_off, MPI_BYTE, filetype, "native", MPI_INFO_NULL)) != MPI_SUCCESS) { MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_set_view failed (%s)\n", mpi_err_str); return 1; } buf = writedata; if ((mpi_err = MPI_File_write_at_all(fh, mpi_off, buf, bufcount, buftype, &mpi_stat)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n", (long) mpi_off, bufcount, mpi_err_str); return 1; }; if ((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){ MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen); printf("MPI_File_close failed. \n"); return 1; }; mpi_err = MPI_Barrier(MPI_COMM_WORLD); #ifdef H5_MPI_SPECIAL_COLLECTIVE_IO_WORKS if(retcode != 0) { if(mpi_rank == 0) { printf("special collective IO is NOT working at this platform\n"); printf("Go back to hdf5/config and find the corresponding\n"); printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n"); printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n"); printf(" at the end of the file.\n"); printf(" Please report to [email protected] about this problem.\n"); } retcode = 1; } #else if(retcode == 0) { if(mpi_rank == 0) { printf(" This is NOT an error, What it really says is\n"); printf("special collective IO is WORKING at this platform\n"); printf(" Go back to hdf5/config and find the corresponding \n"); printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n"); printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n"); printf(" at the end of the file.\n"); printf("Please report to [email protected] about this problem.\n"); } retcode = 1; } #endif return retcode; }
/* *************************** * * Main computational kernel * * *************************** */ int correlationKernel(int rank, int size, double* dataMatrixX, double* dataMatrixY, int columns, int rows, char *out_filename, int distance_flag) { int local_check = 0, global_check = 0; int i = 0, j, taskNo; int err, count = 0; unsigned long long fair_chunk = 0, coeff_count = 0; unsigned int init_and_cleanup_loop_iter=0; unsigned long long cor_cur_size = 0; double start_time, end_time; // Variables needed by the Indexed Datatype MPI_Datatype coeff_index_dt; MPI_File fh; int *blocklens, *indices; MPI_Status stat; MPI_Comm comm = MPI_COMM_WORLD; // Master processor keeps track of tasks if (rank == 0) { // Make sure everything will work fine even if there are // less genes than available workers (there are size-1 workers // master does not count) if ( (size-1) > rows ) init_and_cleanup_loop_iter = rows+1; else init_and_cleanup_loop_iter = size; // Start timer start_time = MPI_Wtime(); // Send out initial tasks (remember you have size-1 workers, master does not count) for (i=1; i<init_and_cleanup_loop_iter; i++) { taskNo = i-1; err = MPI_Send(&taskNo, 1, MPI_INT, i, 0, comm); } // Terminate any processes that were not working due to the fact // that the number of rows where less than the actual available workers for(i=init_and_cleanup_loop_iter; i < size; i++) { PROF(rank, "\nPROF_idle : Worker %d terminated due to insufficient work load", i); err = -1; err = MPI_Send(&err, 1, MPI_INT, i, 0, comm); } // Wait for workers to finish their work assignment and ask for more for (i=init_and_cleanup_loop_iter-1; i<rows; i++) { err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat); // Check taskNo to make sure everything is ok. Negative means there is problem // thus terminate gracefully all remaining working workers if ( taskNo < 0 ) { // Reduce by one because one worker is already terminated init_and_cleanup_loop_iter--; // Break and cleanup break; } // The sending processor is ready to work: // It's ID is in stat.MPI_SOURCE // Send it the current task (i) err = MPI_Send(&i, 1, MPI_INT, stat.MPI_SOURCE, 0, comm); } // Clean up processors for (i=1; i<init_and_cleanup_loop_iter; i++) { // All tasks complete - shutdown workers err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat); // If process failed then it will not be waiting to receive anything // We have to ignore the send because it will deadlock if ( taskNo < 0 ) continue; err = -1; err = MPI_Send(&err, 1, MPI_INT, stat.MPI_SOURCE, 0, comm); } // Master is *always* OK local_check = 0; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Check failed, abort if ( global_check != 0 ) { return -1; } // Stop timer end_time = MPI_Wtime(); PROF(rank, "\nPROF_comp (workers=%d) : Time taken by correlation coefficients computations : %g\n", size-1, end_time - start_time); // Start timer start_time = MPI_Wtime(); // Master process must call MPI_File_set_view as well, it's a collective call // Open the file handler MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); // Create the file view MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL); // Write data to disk MPI_File_write_all(fh, &cor[0], 0, MPI_DOUBLE, &stat); // Stop timer end_time = MPI_Wtime(); PROF(rank, "\nPROF_write (workers=%d) : Time taken for global write-file : %g\n", size-1, end_time - start_time); } else { // Compute how many workers will share the work load // Two scenarios exist: // (1) more OR equal number of workers and rows exist // (2) more rows than workers if ( (size-1) > rows ) { // For this scenario each worker will get exaclty one work asssignment. // There is not going to be any other work so it only compute "rows" number // of coefficients fair_chunk = rows; cor_cur_size = fair_chunk; } else { // For this scenario we are going to allocate space equal to a fair // distribution of work assignments *plus* an extra amount of space to // cover any load imbalancing. This amount is expressed as a percentage // of the fair work distribution (see on top, 20% for now) // Plus 1 to round it up or just add some extra space, both are fine fair_chunk = (rows / (size-1)) + 1; DEBUG("fair_chunk %d \n", fair_chunk); // We can use "j" as temporary variable. // Plus 1 to avoid getting 0 from the multiplication. j = (fair_chunk * MEM_PERC) + 1; cor_cur_size = (fair_chunk + j) * rows; DEBUG("cor_cur_size %lld \n", cor_cur_size); } // Allocate memory DEBUG("cor_cur_size %lld \n", cor_cur_size); long long double_size = sizeof(double); DEBUG("malloc size %lld \n", (double_size * cor_cur_size)); cor = (double *)malloc(double_size * cor_cur_size); blocklens = (int *)malloc(sizeof(int) * rows); indices = (int *)malloc(sizeof(int) * rows); mean_value_vectorX = (double *)malloc(sizeof(double) * rows); Sxx_vector = (double *)malloc(sizeof(double) * rows); mean_value_vectorY = (double *)malloc(sizeof(double) * rows); Syy_vector = (double *)malloc(sizeof(double) * rows); // Check that all memory is successfully allocated if ( ( cor == NULL ) || ( blocklens == NULL ) || ( indices == NULL ) || ( mean_value_vectorX == NULL ) || ( Sxx_vector == NULL ) || ( mean_value_vectorY == NULL ) || ( Syy_vector == NULL ) ) { ERR("**ERROR** : Memory allocation failed on worker process %d. Aborting.\n", rank); // Free allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); // Let the master process know its aborting in order to terminate // the rest of the working workers // We have to receive a work assignment first and then terminate // otherwise the master will deadlock trying to give work to this worker err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat); taskNo = -1; err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); // This worker failed local_check = 1; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); return -1; } // Compute necessary parameters for Pearson method // (this will transform the values of the input array to more meaningful data // and save us from a lot of redundant computations) compute_parameters(dataMatrixX, dataMatrixY, rows, columns); // Main loop for workers. They get work from master, compute coefficients, // save them to their *local* vector and ask for more work for(;;) { // Get work err = 0; err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat); // If received task is -1, function is terminated if ( taskNo == -1 ) break; // Check if there is enough memory to store the new coefficients, if not reallocate // the current memory and expand it by MEM_PERC of the approximated size if ( cor_cur_size < (coeff_count + rows) ) { PROF(0, "\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank); DEBUG("\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank); // Use j as temporary again. Add two (or any other value) to avoid 0. // (two is just a random value, you can put any value really...) j = (fair_chunk * MEM_PERC) + 2; cor_cur_size += (j * rows); // Reallocate and check cor = (double *)realloc(cor, sizeof(double) * cor_cur_size); if ( cor == NULL ) { ERR("**ERROR** : Memory re-allocation failed on worker process %d. Aborting.\n", rank); // Let the master process know its aborting in order to terminate // the rest of the working workers taskNo = -1; err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); // This worker failed local_check = 1; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Free all allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); return -1; } } // Compute the correlation coefficients if(dataMatrixY != NULL) { for (j=0; j < rows; j++) { cor[coeff_count] = pearson_XY(dataMatrixX, dataMatrixY, j, taskNo, columns); coeff_count++; } } else { for (j=0; j < rows; j++) { // Set main diagonal to 1 if ( j == taskNo ) { cor[coeff_count] = 1.0; coeff_count++; continue; } cor[coeff_count] = pearson(dataMatrixX, taskNo, j, columns); coeff_count++; } } // The value of blocklens[] represents the number of coefficients on each // row of the corellation array blocklens[count] = rows; // The value of indices[] represents the offset of each row in the data file indices[count] = (taskNo * rows); count++; // Give the master the taskID err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); } // There are two possibilities // (a) everything went well and all workers finished ok // (b) some processes finished ok but one or more of the remaining working workers failed // To make sure all is well an all-reduce will be performed to sync all workers and guarantee success // before moving on to write the output file // This worker is OK local_check = 0; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Check failed if ( global_check != 0 ) { // Free all allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); return -1; } PROF(0, "\nPROF_stats (thread %3d) : Fair chunk of work : %d \t\t Allocated : %d \t\t Computed : %d\n", rank, fair_chunk, cor_cur_size, coeff_count); // If the distance_flag is set, then transform all correlation coefficients to distances if ( distance_flag == 1 ) { for(j=0; j < coeff_count; j++) { cor[j] = 1 - cor[j]; } } // Create and commit the Indexed datatype *ONLY* if there are data available if ( coeff_count != 0 ) { MPI_Type_indexed(count, blocklens, indices, MPI_DOUBLE, &coeff_index_dt); MPI_Type_commit(&coeff_index_dt); } // Open the file handler MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); // Create the file view if ( coeff_count != 0 ) { MPI_File_set_view(fh, 0, MPI_DOUBLE, coeff_index_dt, "native", MPI_INFO_NULL); } else { MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL); } // Write data to disk // TODO coeff_count cannot be greater than max int (for use in the MPI_File_write_all call). // A better fix should be possible, for now throw error. DEBUG("\ncoeff_count is %lld\n", coeff_count); DEBUG("\INT_MAX is %d\n", INT_MAX); if(coeff_count>INT_MAX) { ERR("**ERROR** : Could not run as the chunks of data are too large. Try running again with more MPI processes.\n"); // Free allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); // Let the master process know its aborting in order to terminate // the rest of the working workers // We have to receive a work assignment first and then terminate // otherwise the master will deadlock trying to give work to this worker err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat); taskNo = -1; err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); // This worker failed local_check = 1; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); return -1; } DEBUG("\nWriting %d to disk\n", coeff_count); MPI_File_write_all(fh, &cor[0], coeff_count, MPI_DOUBLE, &stat); if (coeff_count != 0 ) MPI_Type_free(&coeff_index_dt); // Free all allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); } DEBUG("\nAbout to write to disk %d\n", rank); MPI_File_sync( fh ) ; // Causes all previous writes to be transferred to the storage device DEBUG("\nWritten to disk %d\n",rank); // MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine. DEBUG("\nAfter barrier \n", rank); // Close file handler MPI_File_close(&fh); DEBUG("\nAfter file closed /n"); // MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine. DEBUG("\nAbout to return from kernel /n"); return 0; }
void readGraph_singleFile_MPI(graph_t *G, char *filename) { uint8_t align; int rank, size; int offset,offset_row ,offset_col,offset_weight; edge_id_t my_edges[2]; int local_n=0; int local_m=0; int k; uint32_t TotVertices; MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); G->rank = rank; G->nproc = size; G->filename[0] = '\0'; sprintf(G->filename, "%s", filename); int lgsize; for (lgsize = 0; lgsize < size; ++lgsize) { if ((1 << lgsize) == size) break; } MPI_File fh; MPI_Status status; MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh); MPI_File_read(fh, &G->n, 1, MPI_UINT32_T, &status); offset = sizeof(vertex_id_t); TotVertices = G->n; MPI_File_read_at(fh, offset, &G->m, 1, MPI_UINT64_T, &status); offset += sizeof(edge_id_t); MPI_File_read_at(fh, offset, &G->directed, 1, MPI_C_BOOL, &status); offset += sizeof(bool); MPI_File_read_at(fh, offset, &align, 1, MPI_UINT8_T, &status); offset += sizeof(uint8_t); offset_row = offset; for( uint32_t i = 0; i < G->n; i++ ) { if( rank == VERTEX_OWNER(i,TotVertices,size) ) { MPI_File_read_at(fh,offset_row + (i)*sizeof(edge_id_t),&my_edges[0], 2, MPI_UINT64_T, &status); local_n++; local_m += my_edges[1] - my_edges[0]; } } G->local_n = local_n; G->local_m = local_m; offset_col = offset_row + (G->n+1) * sizeof(edge_id_t); offset_weight = offset_col + G->m * sizeof(vertex_id_t); G->rowsIndices = (edge_id_t *)malloc((G->local_n + 1) * sizeof(edge_id_t) ); G->endV = (vertex_id_t *)malloc((G->local_m)*sizeof(vertex_id_t)); G->weights = (weight_t *)malloc((G->local_m)*sizeof(weight_t)); G->rowsIndices[0] = 0; k = 1; for( uint32_t i = 0; i < G->n; i++ ) { if( rank == VERTEX_OWNER(i,TotVertices,size) ) { MPI_File_read_at(fh,offset_row + (i)*sizeof(edge_id_t),&my_edges[0], 2, MPI_UINT64_T, &status); G->rowsIndices[k] = G->rowsIndices[k-1] + my_edges[1] - my_edges[0]; MPI_File_read_at(fh,offset_col + my_edges[0] * sizeof(vertex_id_t), &G->endV[G->rowsIndices[k-1]], G->rowsIndices[k]-G->rowsIndices[k-1], MPI_UINT32_T, &status); MPI_File_read_at(fh,offset_weight + my_edges[0] * sizeof(weight_t), &G->weights[G->rowsIndices[k-1]], G->rowsIndices[k]-G->rowsIndices[k-1], MPI_DOUBLE, &status); k++; } } MPI_File_close(&fh); }
int main(int argc, char** argv) { int my_rank, p; int i, dest; mpz_t currentPrime; unsigned long int product; sscanf(argv[1], "%lu", &product); int secondFactor = 0; int bcastStatus; int equals; /** GMP library variables **/ mpz_t nextPrimeNumber; mpz_t testFactor; mpz_init(nextPrimeNumber); mpz_init_set_str (nextPrimeNumber, argv[1], 10); mpz_init(testFactor); mpz_init_set_ui(currentPrime, 2); mpz_nextprime(nextPrimeNumber, nextPrimeNumber); mpz_t testProduct; mpz_init(testProduct); /** MPI Initialization **/ MPI_Request finalValue; MPI_File out; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &p); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Status status; /** Get Ready to receive a factor if another process finds one */ MPI_Irecv(&secondFactor, 1, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &finalValue); /** Prepare initial offset for each process **/ for (i=0 ; i < my_rank ; i++) { mpz_nextprime(currentPrime, currentPrime); } /** Start Timing **/ double start = MPI_Wtime(), diff; while (!secondFactor) { /** Check if another process has found the factors **/ MPI_Test (&finalValue, &bcastStatus, &status); if(bcastStatus) { /** Somebody else has found the factors, we are done **/ MPI_Wait(&finalValue, &status); break; } /** Skip P primes before checking again **/ for (i=0 ; i < p ; i++) { mpz_nextprime(currentPrime, currentPrime); } /** Brute force check if the current working prime is a factor of the input number **/ for (mpz_set_ui(testFactor , 2) ; mpz_get_ui(testFactor) <= mpz_get_ui(currentPrime); mpz_nextprime(testFactor, testFactor)) { /** Check if another process has found the factors **/ MPI_Test (&finalValue, &bcastStatus, &status); if(bcastStatus) { MPI_Wait(&finalValue, &status); break; } mpz_mul_ui(testProduct, currentPrime, mpz_get_ui(testFactor)); equals = mpz_cmp_ui(testProduct, product); if (equals == 0){ /** We've found the factor, find the second number, secnd it to the other processes **/ secondFactor = mpz_get_ui(testFactor); printf("done by process %d, factors are %lu and %d \n", my_rank, mpz_get_ui(currentPrime), secondFactor); fflush(stdout); for (dest = 0 ; dest < p ; dest++) { if (dest != my_rank) { MPI_Send(&secondFactor, 1, MPI_UNSIGNED_LONG, dest, 0, MPI_COMM_WORLD); } } } } } diff = MPI_Wtime() - start; /** End Timing **/ /** Prepare file contents **/ char fileName[200], fileContents[200]; sprintf(fileName, "time_%lu", product); sprintf(fileContents, "%d\t%f\n", my_rank, diff); /** Write File **/ MPI_File_open( MPI_COMM_WORLD, fileName, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &out ); MPI_File_seek(out, my_rank*strlen ( fileContents ) , MPI_SEEK_SET); MPI_File_write_all(out , &fileContents, strlen ( fileContents ), MPI_CHAR, &status ); MPI_File_close(&out); /** Fin **/ MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return(0); }
/** * \brief Measures the time to write once to a file. * * Only one process is active. It writes once to a file. * * Remark:<br> * With the <tt>O_DIRECT</tt> flag set, cache effects are minimized, because I/O * is done directly to/from user space buffers. The operation system's page * cache is bypassed. Under Linux 2.6 alignment to 512-byte boundaries is * required for buffer and file offset. Thus the following parameters should be * set in a SKaMPI input file: * - <tt>set_send_buffert_alignment (512)</tt> * - <tt>set_recv_buffert_alignment (512)</tt> * - <tt>switch_buffer_cycling_off ()</tt><br> * * <tt>O_DIRECT</tt> is only relevant if the POSIX-API is used for I/O. * * For more information please refer to the <tt>open ()</tt> man pages. * * \param[in] size size of memory buffer, i.e. number of <tt>MPI_BYTE</tt>s * \param[in] api POSIX-API or MPI-API for I/O accesses * \param[in] create_flag write into existing file (FALSE) or create it (TRUE) * \param[in] directio_flag open file with <tt>O_DIRECT</tt> flag to minimize * cache effects * * \return measured time */ double measure_MPI_IO_write_file_once (int size, char *api, int create_flag, int directio_flag){ double start_time = 1.0, end_time = 0.0; int open_flags; char *error_string; if (get_measurement_rank () == 0){ if (strcmp (api, POSIX_API) == 0){ if (directio_flag != 0) open_flags = O_WRONLY | O_DIRECT; else open_flags = O_WRONLY; errno = 0; if (create_flag == 0){ /* open existing file */ if ((io_fd = open (io_filename, open_flags)) < 0){ error_string = strerror (errno); error_with_abort (errno, "\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed." "\nCannot open local file (write only mode)." "\nError: %s\n", size, api, create_flag, directio_flag, error_string); } } else { /* open nonexisting file and create it */ if ((io_fd = open (io_filename, open_flags|O_CREAT, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0){ error_string = strerror (errno); error_with_abort (errno, "\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed." "\nCannot open local file (write only mode)." "\nError: %s\n", size, api, create_flag, directio_flag, error_string); } } start_time = start_synchronization (); write (io_fd, get_send_buffer (), size); fsync (io_fd); end_time = MPI_Wtime (); close (io_fd); } else{ /* if strcmp (api, POSIX_API) != 0 */ if (create_flag == 0){ MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &io_fh); } else{ /* if create_flag != 0*/ MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &io_fh); } MPI_File_set_view (io_fh, (MPI_Offset)0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL); start_time = start_synchronization (); MPI_File_write (io_fh, get_send_buffer (), size, MPI_BYTE, MPI_STATUS_IGNORE); MPI_File_sync (io_fh); end_time = MPI_Wtime (); MPI_File_close (&io_fh); } } else if (get_measurement_rank () != 0) { start_synchronization (); } stop_synchronization (); if (get_measurement_rank () == 0) return end_time - start_time; else return -1.0; }
int SeqSummaryCommand::execute(){ try{ if (abort == true) { if (calledHelp) { return 0; } return 2; } //set current fasta to fastafile m->setFastaFile(fastafile); map<string, string> variables; variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(fastafile)); string summaryFile = getOutputFileName("summary",variables); int numSeqs = 0; vector<int> startPosition; vector<int> endPosition; vector<int> seqLength; vector<int> ambigBases; vector<int> longHomoPolymer; if (namefile != "") { nameMap = m->readNames(namefile); } else if (countfile != "") { CountTable ct; ct.readTable(countfile, false, false); nameMap = ct.getNameMap(); } if (m->control_pressed) { return 0; } #ifdef USE_MPI int pid, numSeqsPerProcessor; int tag = 2001; int startTag = 1; int endTag = 2; int lengthTag = 3; int baseTag = 4; int lhomoTag = 5; int outMode=MPI_MODE_CREATE|MPI_MODE_WRONLY; vector<unsigned long long> MPIPos; MPI_Status status; MPI_Status statusOut; MPI_File inMPI; MPI_File outMPI; MPI_Comm_size(MPI_COMM_WORLD, &processors); MPI_Comm_rank(MPI_COMM_WORLD, &pid); char tempFileName[1024]; strcpy(tempFileName, fastafile.c_str()); char sumFileName[1024]; strcpy(sumFileName, summaryFile.c_str()); MPI_File_open(MPI_COMM_WORLD, tempFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI); //comm, filename, mode, info, filepointer MPI_File_open(MPI_COMM_WORLD, sumFileName, outMode, MPI_INFO_NULL, &outMPI); if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); return 0; } if (pid == 0) { //you are the root process //print header string outputString = "seqname\tstart\tend\tnbases\tambigs\tpolymer\tnumSeqs\n"; int length = outputString.length(); char* buf2 = new char[length]; memcpy(buf2, outputString.c_str(), length); MPI_File_write_shared(outMPI, buf2, length, MPI_CHAR, &statusOut); delete buf2; MPIPos = m->setFilePosFasta(fastafile, numSeqs); //fills MPIPos, returns numSeqs for(int i = 1; i < processors; i++) { MPI_Send(&numSeqs, 1, MPI_INT, i, tag, MPI_COMM_WORLD); MPI_Send(&MPIPos[0], (numSeqs+1), MPI_LONG, i, tag, MPI_COMM_WORLD); } //figure out how many sequences you have to do numSeqsPerProcessor = numSeqs / processors; int startIndex = pid * numSeqsPerProcessor; if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; } //do your part MPICreateSummary(startIndex, numSeqsPerProcessor, startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, inMPI, outMPI, MPIPos); }else { //i am the child process MPI_Recv(&numSeqs, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status); MPIPos.resize(numSeqs+1); MPI_Recv(&MPIPos[0], (numSeqs+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status); //figure out how many sequences you have to align numSeqsPerProcessor = numSeqs / processors; int startIndex = pid * numSeqsPerProcessor; if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; } //do your part MPICreateSummary(startIndex, numSeqsPerProcessor, startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, inMPI, outMPI, MPIPos); } MPI_File_close(&inMPI); MPI_File_close(&outMPI); MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case if (pid == 0) { //get the info from the child processes for(int i = 1; i < processors; i++) { int size; MPI_Recv(&size, 1, MPI_INT, i, tag, MPI_COMM_WORLD, &status); vector<int> temp; temp.resize(size+1); for(int j = 0; j < 5; j++) { MPI_Recv(&temp[0], (size+1), MPI_INT, i, 2001, MPI_COMM_WORLD, &status); int receiveTag = temp[temp.size()-1]; //child process added a int to the end to indicate what count this is for if (receiveTag == startTag) { for (int k = 0; k < size; k++) { startPosition.push_back(temp[k]); } }else if (receiveTag == endTag) { for (int k = 0; k < size; k++) { endPosition.push_back(temp[k]); } }else if (receiveTag == lengthTag) { for (int k = 0; k < size; k++) { seqLength.push_back(temp[k]); } }else if (receiveTag == baseTag) { for (int k = 0; k < size; k++) { ambigBases.push_back(temp[k]); } }else if (receiveTag == lhomoTag) { for (int k = 0; k < size; k++) { longHomoPolymer.push_back(temp[k]); } } } } }else{ //send my counts int size = startPosition.size(); MPI_Send(&size, 1, MPI_INT, 0, tag, MPI_COMM_WORLD); startPosition.push_back(startTag); int ierr = MPI_Send(&(startPosition[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD); endPosition.push_back(endTag); ierr = MPI_Send (&(endPosition[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD); seqLength.push_back(lengthTag); ierr = MPI_Send(&(seqLength[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD); ambigBases.push_back(baseTag); ierr = MPI_Send(&(ambigBases[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD); longHomoPolymer.push_back(lhomoTag); ierr = MPI_Send(&(longHomoPolymer[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD); } MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case #else vector<unsigned long long> positions; #if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix) positions = m->divideFile(fastafile, processors); for (int i = 0; i < (positions.size()-1); i++) { lines.push_back(new linePair(positions[i], positions[(i+1)])); } #else positions = m->setFilePosFasta(fastafile, numSeqs); if (positions.size() < processors) { processors = positions.size(); } //figure out how many sequences you have to process int numSeqsPerProcessor = numSeqs / processors; for (int i = 0; i < processors; i++) { int startIndex = i * numSeqsPerProcessor; if(i == (processors - 1)){ numSeqsPerProcessor = numSeqs - i * numSeqsPerProcessor; } lines.push_back(new linePair(positions[startIndex], numSeqsPerProcessor)); } #endif if(processors == 1){ numSeqs = driverCreateSummary(startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, fastafile, summaryFile, lines[0]); }else{ numSeqs = createProcessesCreateSummary(startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, fastafile, summaryFile); } if (m->control_pressed) { return 0; } #endif #ifdef USE_MPI if (pid == 0) { #endif sort(startPosition.begin(), startPosition.end()); sort(endPosition.begin(), endPosition.end()); sort(seqLength.begin(), seqLength.end()); sort(ambigBases.begin(), ambigBases.end()); sort(longHomoPolymer.begin(), longHomoPolymer.end()); int size = startPosition.size(); //find means unsigned long long meanStartPosition, meanEndPosition, meanSeqLength, meanAmbigBases, meanLongHomoPolymer; meanStartPosition = 0; meanEndPosition = 0; meanSeqLength = 0; meanAmbigBases = 0; meanLongHomoPolymer = 0; for (int i = 0; i < size; i++) { meanStartPosition += startPosition[i]; meanEndPosition += endPosition[i]; meanSeqLength += seqLength[i]; meanAmbigBases += ambigBases[i]; meanLongHomoPolymer += longHomoPolymer[i]; } double meanstartPosition, meanendPosition, meanseqLength, meanambigBases, meanlongHomoPolymer; meanstartPosition = meanStartPosition / (double) size; meanendPosition = meanEndPosition /(double) size; meanlongHomoPolymer = meanLongHomoPolymer / (double) size; meanseqLength = meanSeqLength / (double) size; meanambigBases = meanAmbigBases /(double) size; int ptile0_25 = int(size * 0.025); int ptile25 = int(size * 0.250); int ptile50 = int(size * 0.500); int ptile75 = int(size * 0.750); int ptile97_5 = int(size * 0.975); int ptile100 = size - 1; //to compensate for blank sequences that would result in startPosition and endPostion equalling -1 if (startPosition[0] == -1) { startPosition[0] = 0; } if (endPosition[0] == -1) { endPosition[0] = 0; } if (m->control_pressed) { m->mothurRemove(summaryFile); return 0; } m->mothurOutEndLine(); m->mothurOut("\t\tStart\tEnd\tNBases\tAmbigs\tPolymer\tNumSeqs"); m->mothurOutEndLine(); m->mothurOut("Minimum:\t" + toString(startPosition[0]) + "\t" + toString(endPosition[0]) + "\t" + toString(seqLength[0]) + "\t" + toString(ambigBases[0]) + "\t" + toString(longHomoPolymer[0]) + "\t" + toString(1)); m->mothurOutEndLine(); m->mothurOut("2.5%-tile:\t" + toString(startPosition[ptile0_25]) + "\t" + toString(endPosition[ptile0_25]) + "\t" + toString(seqLength[ptile0_25]) + "\t" + toString(ambigBases[ptile0_25]) + "\t"+ toString(longHomoPolymer[ptile0_25]) + "\t" + toString(ptile0_25+1)); m->mothurOutEndLine(); m->mothurOut("25%-tile:\t" + toString(startPosition[ptile25]) + "\t" + toString(endPosition[ptile25]) + "\t" + toString(seqLength[ptile25]) + "\t" + toString(ambigBases[ptile25]) + "\t" + toString(longHomoPolymer[ptile25]) + "\t" + toString(ptile25+1)); m->mothurOutEndLine(); m->mothurOut("Median: \t" + toString(startPosition[ptile50]) + "\t" + toString(endPosition[ptile50]) + "\t" + toString(seqLength[ptile50]) + "\t" + toString(ambigBases[ptile50]) + "\t" + toString(longHomoPolymer[ptile50]) + "\t" + toString(ptile50+1)); m->mothurOutEndLine(); m->mothurOut("75%-tile:\t" + toString(startPosition[ptile75]) + "\t" + toString(endPosition[ptile75]) + "\t" + toString(seqLength[ptile75]) + "\t" + toString(ambigBases[ptile75]) + "\t" + toString(longHomoPolymer[ptile75]) + "\t" + toString(ptile75+1)); m->mothurOutEndLine(); m->mothurOut("97.5%-tile:\t" + toString(startPosition[ptile97_5]) + "\t" + toString(endPosition[ptile97_5]) + "\t" + toString(seqLength[ptile97_5]) + "\t" + toString(ambigBases[ptile97_5]) + "\t" + toString(longHomoPolymer[ptile97_5]) + "\t" + toString(ptile97_5+1)); m->mothurOutEndLine(); m->mothurOut("Maximum:\t" + toString(startPosition[ptile100]) + "\t" + toString(endPosition[ptile100]) + "\t" + toString(seqLength[ptile100]) + "\t" + toString(ambigBases[ptile100]) + "\t" + toString(longHomoPolymer[ptile100]) + "\t" + toString(ptile100+1)); m->mothurOutEndLine(); m->mothurOut("Mean:\t" + toString(meanstartPosition) + "\t" + toString(meanendPosition) + "\t" + toString(meanseqLength) + "\t" + toString(meanambigBases) + "\t" + toString(meanlongHomoPolymer)); m->mothurOutEndLine(); if ((namefile == "") && (countfile == "")) { m->mothurOut("# of Seqs:\t" + toString(numSeqs)); m->mothurOutEndLine(); } else { m->mothurOut("# of unique seqs:\t" + toString(numSeqs)); m->mothurOutEndLine(); m->mothurOut("total # of seqs:\t" + toString(startPosition.size())); m->mothurOutEndLine(); } if (m->control_pressed) { m->mothurRemove(summaryFile); return 0; } m->mothurOutEndLine(); m->mothurOut("Output File Names: "); m->mothurOutEndLine(); m->mothurOut(summaryFile); m->mothurOutEndLine(); outputNames.push_back(summaryFile); outputTypes["summary"].push_back(summaryFile); m->mothurOutEndLine(); #ifdef USE_MPI } #endif //set fasta file as new current fastafile string current = ""; itTypes = outputTypes.find("summary"); if (itTypes != outputTypes.end()) { if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setSummaryFile(current); } } return 0; } catch(exception& e) { m->errorOut(e, "SeqSummaryCommand", "execute"); exit(1); } }
int main(int argc, char ** argv) { int nprocs, mynod, errcode; options my_options = {NULL, 0, 0}; MPI_File fh; MPI_Status status; MPI_Info info; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &mynod); parse_args(argc, argv, mynod, &my_options); if (my_options.do_aggregation) { MPI_Info_create(&info); MPI_Info_set(info, "romio_no_indep_rw", "true"); MPI_Info_set(info, "cb_config_list", "leela.mcs.anl.gov:1"); } else { info = MPI_INFO_NULL; } /* create the file w/o EXCL: this must not fail */ errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname, MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh); if (errcode != MPI_SUCCESS) { handle_error(errcode, "MPI_File_open"); } errcode = MPI_File_close(&fh); if (errcode != MPI_SUCCESS) { handle_error(errcode, "MPI_File_close"); } /* now try to open w/ CREAT|EXCL: this must fail */ errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname, MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh); if (errcode == MPI_SUCCESS) { handle_error(errcode, "MPI_File_open: expected an error: got"); } /* ignore the error: File_delete is not aggregator-aware */ MPI_File_delete(my_options.fname, info); /* this must succeed: the file no longer exists */ errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname, MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh); if (errcode != MPI_SUCCESS) { handle_error(errcode, "MPI_File_open"); } errcode = MPI_File_close(&fh); if (errcode != MPI_SUCCESS) { handle_error(errcode, "MPI_File_close"); } if (mynod == 0) { printf(" No Errors\n"); } MPI_Finalize(); return 0; }
int main(int argc, char *argv[]) { int iarrayOfSizes[2], iarrayOfSubsizes[2], iarrayOfStarts[2], ilocal_size; int nproc[2], periods[2], icoord[2]; int m, n, i, j, wsize, wrank, crank, ndims, lrows, lcols, grow, gcol, err; MPI_Datatype filetype; MPI_File fh; MPI_Comm cartcomm; MPI_Info info0, info3; double t, topen, twrite, tclose, wrate; double *local_array; char nstripesStr[12], stripeUnitStr[12]; int nstripes = -1; int stripeUnit = -1; MPI_Offset headerSize = 0; MPI_Init(0,0); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); /* Get global array size */ m = n = 128; /* Set default size */ /* ioda [ n ] [ m ] [ nstripes ] [ stripeunit ] [ headersize ] */ if (argc > 0) { if (argc > 1) m = atoi(argv[1]); if (argc > 2) n = atoi(argv[2]); if (argc > 3) nstripes = atoi(argv[3]); if (argc > 4) stripeUnit = atoi(argv[4]); if (argc > 5) headerSize = atoi(argv[5]); if (argc > 6) { if (wrank == 0) fprintf(stderr,"Unrecognized argument %s\n", argv[6]); MPI_Abort(MPI_COMM_WORLD,1); } } if (wrank == 0) printf("Matrix is [%d,%d]; file dir = %s\n", m, n, MYSCRATCHDIR ); /* The default number of stripes = totalsize/1M */ if (nstripes < 0) { nstripes = n * m * sizeof(double) / (1024*1024); if (nstripes < 1) nstripes = 1; } if (wrank == 0) printf("nstripes = %d, stripeUnit = %d, header size = %d\n", nstripes, stripeUnit, (int)headerSize); /* Use topology routines to get decomposition and coordinates */ MPI_Comm_size(MPI_COMM_WORLD, &wsize); nproc[0] = 0; nproc[1] = 0; ndims = 2; MPI_Dims_create(wsize, ndims, nproc); periods[0] = 0; periods[1] = 0; MPI_Cart_create(MPI_COMM_WORLD, ndims, nproc, periods, 1, &cartcomm); MPI_Comm_rank(cartcomm, &crank); MPI_Cart_coords(cartcomm, crank, ndims, icoord); iarrayOfSizes[0] = m; iarrayOfSizes[1] = n; iarrayOfSubsizes[0] = m/nproc[0]; iarrayOfSubsizes[1] = n/nproc[1]; iarrayOfStarts[0] = icoord[0] * iarrayOfSubsizes[0]; iarrayOfStarts[1] = icoord[1] * iarrayOfSubsizes[1]; /* Initialize my block of the data */ ilocal_size = iarrayOfSubsizes[0] * iarrayOfSubsizes[1]; lrows = iarrayOfSubsizes[0]; lcols = iarrayOfSubsizes[1]; local_array = (double *)malloc(lrows*lcols*sizeof(double)); gcol = iarrayOfStarts[1]; grow = iarrayOfStarts[0]; for (i=0; i<lrows; i++) { for (j=0; j<lcols; j++) { local_array[j*lrows+i] = (grow+i) + (gcol+j)*m; } } /* Fortran order simply means the data is stored by columns */ MPI_Type_create_subarray(ndims, iarrayOfSizes, iarrayOfSubsizes, iarrayOfStarts, MPI_ORDER_FORTRAN, MPI_DOUBLE, &filetype); MPI_Type_commit(&filetype); info0 = MPI_INFO_NULL; info3 = MPI_INFO_NULL; if (nstripes > 0 || stripeUnit > 0) { MPI_Info_create(&info0); if (nstripes > 0) { snprintf(nstripesStr, sizeof(nstripesStr), "%d", nstripes); MPI_Info_set(info0, "striping_factor", nstripesStr); MPI_Info_set(info0, "cb_nodes", nstripesStr); } if (stripeUnit > 0) { snprintf(stripeUnitStr, sizeof(stripeUnitStr), "%d", stripeUnit); MPI_Info_set(info0, "striping_unit", stripeUnitStr); } MPI_Info_dup(info0, &info3); MPI_Info_set(info3, "romio_no_indep_rw", "true"); /* Other hints to consider: direct_io=true The default cb_buffer_size is 16777216 , but is overridden by the striping unit, which is smaller by default. */ } /* level - 3 */ MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-3.out", MPI_MODE_CREATE | MPI_MODE_RDWR, info3, &fh); topen = MPI_Wtime() - t; if (err != MPI_SUCCESS) myAbort(err, "open testfile-3.out"); if (headerSize > 0) { /* Simulate writing a header */ if (wrank == 0) { char *header; header = (char *)calloc(1,(size_t)headerSize); MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); free(header); } MPI_Barrier(cartcomm); } MPI_File_set_view(fh, headerSize, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL); MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); err = MPI_File_write_all(fh, local_array, ilocal_size, MPI_DOUBLE, MPI_STATUS_IGNORE); twrite = MPI_Wtime() - t; if (err != MPI_SUCCESS) myAbort(err, "collective write"); err = MPI_File_close(&fh); tclose = MPI_Wtime() - t; /* tclose is the time for the write(s) + the close, in case the implementation delays (some of) the writes until the close */ if (err != MPI_SUCCESS) myAbort(err, "close testfile-3.out"); MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if (twrite > 0) wrate = (double)m * (double)n * sizeof(double)/twrite; if (wrank == 0) printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen, twrite, tclose, wrate); /* level - 0 */ MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-0.out", MPI_MODE_CREATE | MPI_MODE_RDWR, info0, &fh); topen = MPI_Wtime() - t; if (err != MPI_SUCCESS) myAbort(err, "open testfile-0.out"); if (headerSize > 0) { /* Simulate writing a header */ if (wrank == 0) { char *header; header = (char *)calloc(1,(size_t)headerSize); MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); free(header); } MPI_Barrier(cartcomm); } MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); gcol = iarrayOfStarts[1]; grow = iarrayOfStarts[0]; for (j=0; j<lcols; j++) { MPI_Offset offset = headerSize + ((MPI_Offset)(grow) + (MPI_Offset)(gcol+j)*m) * sizeof(double); err = MPI_File_write_at(fh, offset, local_array+j*lrows, lrows, MPI_DOUBLE, MPI_STATUS_IGNORE); if (err != MPI_SUCCESS) myAbort(err, "write at"); } twrite = MPI_Wtime() - t; err = MPI_File_close(&fh); tclose = MPI_Wtime() - t; /* tclose is the time for the write(s) + the close, in case the implementation delays (some of) the writes until the close */ if (err != MPI_SUCCESS) myAbort(err, "close testfile-0"); MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if (twrite > 0) wrate = (double)m * (double)n * sizeof(double)/twrite; if (wrank == 0) printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen, twrite, tclose, wrate); if (info0 != MPI_INFO_NULL) { MPI_Info_free(&info0); MPI_Info_free(&info3); } free(local_array); MPI_Finalize(); return 0; }