void IOserver::start()
{
//starting IO server (Sinc line)
MPI_Status status,status2;
MPI_Request send_statut_request;
int flag=1;
int control;
int count=0;
int fileID;
int fileCounter;
int newFilenameLength;
int len;
bool getingDataFlag;
int getingData[IO_ClientSize_];
serverOn_flag=true;
while(serverOn_flag)
{
//cout<<count<<endl;
ostreamFile_flag=false;
if(IO_Rank_==0)
{
//cout<<count<<endl;
//count++;
//send non-blocking for status server free. (Sync line)
serverReady_flag=true;
MPI_Isend(&serverReady_flag,1,MPI::BOOL,1,SERVER_STATE_TAG,syncLineComm_,&send_statut_request);
//test if an ostream need to be open or if the server need to be stoped (Sync line) blocking rec
MPI_Recv(&control,1,MPI::INT,1,SERVER_CONTROL_TAG,syncLineComm_,&status);
}
MPI_Bcast(&control,1,MPI::INT,0,IO_Comm_);
//MPI_Bcast(&ostreamFile_flag,1,MPI::BOOL,0,IO_Comm_);
if(control==CONTROL_STOP)
{
//if(IO_Rank_==0)cout<<"stop server"<<endl;
serverOn_flag=false;
}
if(control==CONTROL_OPEN_OSTREAM)
{
//if(IO_Rank_==0)cout<<"ostream"<<endl;
ostreamFile_flag=true;
serverReady_flag=false;
if(IO_Rank_==0)MPI_Isend(&serverReady_flag,1,MPI::BOOL,1,SERVER_STATE_TAG,syncLineComm_,&send_statut_request);
fileCounter=0;
while(ostreamFile_flag)
{
//cout<<"rank: "<<IO_Rank_<<", ostream open"<<endl;
//test if there is a new file (Sync line)
if(IO_Rank_==0)MPI_Recv(&control,1,MPI::INT,1,IO_FILE_CONTROL_TAG,syncLineComm_,&status);
MPI_Bcast(&control,1,MPI::INT,0,IO_Comm_);
if(control==CONTROL_CLOSE_OSTREAM)
{
//if(IO_Rank_==0)cout<<"stop ostream"<<endl;
ostreamFile_flag=false;
}
if(control==CONTROL_CREATE_FILE)
{
//create new file ID
if(fileCounter>= MAX_FILE_NUMBER)
{
if(IO_Rank_==0)cout<<"Already to much file"<<endl;
fileID=FILE_FAIL;
}
else
{
fileID=fileCounter;
//if(IO_Rank_==0)cout<<"IO_server creating new file : "<<fileID<<endl;
}
if(IO_Rank_==0)MPI_Ssend(&fileID,1,MPI::INT,1,IO_FILE_CONTROL_FILEID_TAG,syncLineComm_);
if(fileID != FILE_FAIL)
{
if(IO_Rank_==0)
{
MPI_Probe(1,IO_FILE_CONTROL_FILENAME_TAG,syncLineComm_,&status);
MPI_Get_count(&status,MPI::CHAR,&newFilenameLength);
}
MPI_Bcast(&newFilenameLength,1,MPI::INT,0,IO_Comm_);
char * filename;
filename = (char*)malloc(newFilenameLength*sizeof(char));
if(IO_Rank_==0)MPI_Recv(filename,newFilenameLength,MPI::CHAR,1,IO_FILE_CONTROL_FILENAME_TAG,syncLineComm_,&status2);
MPI_Bcast(filename,newFilenameLength,MPI::CHAR,0,IO_Comm_);
files[fileID].filename=filename;
free(filename);
files[fileID].type=FILETYPE_UNSTRUCTURED; // no structured file implemented!!!!
//cout<< files[fileID].filename <<endl;
if(fileID==0)files[fileID].data=dataBuffer;
else files[fileID].data=&(files[fileID-1].data[files[fileID-1].size]);
files[fileID].size=0;
fileCounter++;
if(files[fileID].type==FILETYPE_UNSTRUCTURED)
{
for(int i=0;i<IO_ClientSize_;i++)getingData[i]=1;
getingDataFlag=true;
while(getingDataFlag)
{
MPI_Iprobe(MPI_ANY_SOURCE,fileID, masterClientComm_,&flag,&status);
if(flag==true)
{
char * send;
send=&(files[fileID].data[files[fileID].size]);
int size;
MPI_Get_count(&status,MPI::CHAR,&size);
MPI_Recv(send,size,MPI::CHAR,status.MPI_SOURCE,fileID,masterClientComm_,&status2);
files[fileID].size+=size;
}
else
{
MPI_Iprobe(MPI_ANY_SOURCE,IO_FILE_CONTROL_CLOSE_TAG,masterClientComm_ ,&flag,&status);
if(flag==true)
{
//cout<<"file closed"<<endl;
int tempFileID;
int total=0;
int source = status.MPI_SOURCE;
//cout<<source<<endl;
MPI_Recv(&tempFileID,1,MPI::INT,source,IO_FILE_CONTROL_CLOSE_TAG,masterClientComm_,&status2);
//cout<<source<<endl;
getingData[source-1]=0;
for(int i=0;i<IO_ClientSize_;i++)total+=getingData[i];
if(total==0)getingDataFlag=false;
//cout<<total<<endl;
}
}
}
//cout<<"file closed"<<endl;
MPI_Barrier(IO_Comm_);
}
MPI_Barrier(IO_Comm_);
}
}
}//close stream and write
if(fileCounter!=0)
{
for(int i=0;i < fileCounter;i++)
{
//cout<<"writing file: "<< i<<endl;
MPI_File ofile;
long file_Offset=0;
long temp_long=0;
string str_fname;
str_fname = files[i].filename + int2string(IO_Node_,999)+".dat";
char * fname = &(str_fname[0]);
for(int k=0;k<(IO_NodeSize_-1); k++)
{
if(IO_NodeRank_==k)
{
temp_long = file_Offset + files[i].size;
MPI_Send(&temp_long,1,MPI_LONG, k+1 , 0, IO_NodeComm_ );
}
if(IO_NodeRank_==k+1)
{
MPI_Recv( &file_Offset, 1, MPI_LONG, k, 0, IO_NodeComm_, &status);
}
}
if(files[i].size!=0)
{
MPI_File_open(IO_NodeComm_,fname,MPI_MODE_WRONLY | MPI_MODE_CREATE,MPI_INFO_NULL,&ofile);
MPI_File_set_view(ofile,file_Offset,MPI_CHAR,MPI_CHAR,(char*)"native",MPI_INFO_NULL);
MPI_File_write_all(ofile,files[i].data,files[i].size,MPI_CHAR,&status);
MPI_File_close(&ofile);
}
}
}
fileCounter=0;
MPI_Barrier(IO_Comm_);
}
}
}
int main (int argc, char *argv[])
{
int proc_num, my_rank, len;
int i, j;
double start_time, elapsed_time, all_time;
double all_time_max, all_time_avg, all_time_min;
struct timespec ts;
MPI_Status status;
MPI_File fh;
MPI_Datatype contig_type;
MPI_Init(&argc, &argv);
// get the number of procs and rank in the comm
MPI_Comm_size(MPI_COMM_WORLD, &proc_num);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
if(argc != 5) {
printf("Wrong argument number!\n");
printf("Use %s filename request_size repeat_times\n", argv[0]);
return 0;
}
int req_size = atoi(argv[2]);
int repeat_time = atoi(argv[3]);
double ht_read_time= atof(argv[4]);
ht_read_time *= 2.0;
//if(my_rank == 0)
// printf("Sleep time: %lf\n",ht_read_time);
ts.tv_sec = (int)ht_read_time;
ts.tv_nsec = (ht_read_time - ts.tv_sec) * 1000000000;
MPI_Offset stride = proc_num * req_size;
MPI_Offset tmp_pos = my_rank * req_size;
char *read_data = (char*)malloc(req_size);
MPI_Type_contiguous( req_size, MPI_CHAR, &contig_type);
MPI_Type_commit(&contig_type);
start_time = MPI_Wtime();
//MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if(fh==NULL){
printf("File not exist\n");
return -1;
}
for(i = 0; i < repeat_time; i++) {
// MPI_Barrier(MPI_COMM_WORLD);
MPI_File_read_at( fh, tmp_pos, read_data, 1, contig_type, &status );
tmp_pos += stride;
nanosleep(&ts, NULL);
}
MPI_File_close(&fh);
elapsed_time = MPI_Wtime() - start_time;
MPI_Reduce(&elapsed_time, &all_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&elapsed_time, &all_time_min, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(&elapsed_time, &all_time_avg, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
all_time_avg /= proc_num;
MPI_Barrier(MPI_COMM_WORLD);
double data_in_mb = (proc_num*(double)req_size*repeat_time)/(1024.0*1024.0);
if(my_rank == 0)
printf("Total time: %lf Min time: %lf Avg time: %lf Total data: %dM Agg Bandwidth: %lf\n", all_time, all_time_min, all_time_avg, (int)data_in_mb, data_in_mb/all_time);
// printf("%d: %lf\n",my_rank, elapsed_time);
free(read_data);
MPI_Type_free(&contig_type);
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int *buf, i, rank, nprocs, len, sum, global_sum;
char *filename;
MPI_File fh;
MPI_Status status;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: shared_fp <mpiparameter> -- -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(COUNT * sizeof(int));
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
for (i=0; i<COUNT; i++) buf[i] = COUNT*rank + i;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_write_shared(fh, buf, COUNT, MPI_INT, &status);
for (i=0; i<COUNT; i++) buf[i] = 0;
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_seek_shared(fh, 0, MPI_SEEK_SET);
MPI_File_read_shared(fh, buf, COUNT, MPI_INT, &status);
MPI_File_close(&fh);
sum = 0;
for (i=0; i<COUNT; i++) sum += buf[i];
MPI_Allreduce(&sum, &global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (global_sum != (((COUNT*nprocs - 1)*(COUNT*nprocs))/2))
printf("Error: sum %d, global_sum %d, %d\n", sum, global_sum,(((COUNT*nprocs - 1)*(COUNT*nprocs))/2));
free(buf);
free(filename);
if (!rank) printf("Done\n");
MPI_Finalize();
return 0;
}
static int test_mpio_derived_dtype(char *filename) {
MPI_File fh;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
int i;
int nerrors = 0; /* number of errors */
MPI_Datatype etype,filetype;
MPI_Datatype adv_filetype,bas_filetype[2];
MPI_Datatype etypenew, filetypenew;
MPI_Offset disp;
MPI_Status Status;
MPI_Aint adv_disp[2];
MPI_Aint offsets[1];
int blocklens[1],adv_blocklens[2];
int count,outcount;
int retcode;
int mpi_rank,mpi_size;
char buf[3],outbuf[3] = {0};
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
retcode = 0;
for(i=0;i<3;i++)
buf[i] = i+1;
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
disp = 0;
etype = MPI_BYTE;
count = 1;
blocklens[0] = 1;
offsets[0] = 0;
if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
count = 1;
blocklens[0]=1;
offsets[0] = 1;
if((mpi_err= MPI_Type_hindexed(count,blocklens,offsets,MPI_BYTE,&filetypenew))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetypenew))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
outcount = 2;
adv_blocklens[0] = 1;
adv_blocklens[1] = 1;
adv_disp[0] = 0;
adv_disp[1] = 1;
bas_filetype[0] = filetype;
bas_filetype[1] = filetypenew;
if((mpi_err= MPI_Type_struct(outcount,adv_blocklens,adv_disp,bas_filetype,&adv_filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_struct failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&adv_filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_set_view(fh,disp,etype,adv_filetype,"native",MPI_INFO_NULL))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_write(fh,buf,3,MPI_BYTE,&Status))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_write failed (%s)\n", mpi_err_str);
return 1;
;
}
if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_open(MPI_COMM_WORLD,filename,MPI_MODE_RDONLY,MPI_INFO_NULL,&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_set_view(fh,0,MPI_BYTE,MPI_BYTE,"native",MPI_INFO_NULL))!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err = MPI_File_read(fh,outbuf,3,MPI_BYTE,&Status))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_read failed (%s)\n", mpi_err_str);
return 1;
}
if(outbuf[2]==2) {
retcode = 0;
}
else {
/* if(mpi_rank == 0) {
printf("complicated derived datatype is NOT working at this platform\n");
printf("go back to hdf5/config and find the corresponding\n");
printf("configure-specific file and change ?????\n");
}
*/
retcode = -1;
}
if((mpi_err = MPI_File_close(&fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed (%s)\n", mpi_err_str);
return 1;
}
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
#ifdef H5_MPI_COMPLEX_DERIVED_DATATYPE_WORKS
if(retcode == -1) {
if(mpi_rank == 0) {
printf("Complicated derived datatype is NOT working at this platform\n");
printf("Go back to hdf5/config and find the corresponding\n");
printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n");
printf("hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype-works='no'}\n");
printf(" at the end of the file.\n");
printf(" Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
#else
if(retcode == 0) {
if(mpi_rank == 0) {
printf(" This is NOT an error, What it really says is\n");
printf("Complicated derived datatype is WORKING at this platform\n");
printf(" Go back to hdf5/config and find the corresponding \n");
printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n");
printf("hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype-works='no'}\n");
printf(" at the end of the file.\n");
printf("Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
if(retcode == -1) retcode = 0;
#endif
return retcode;
}
int main(int argc,char **argv) {
MPI_File file;
long long mapxsize,mapysize;
long long myxmin,myxmax,myymin,myymax;
int processes_in_x_dim,processes_in_y_dim;
int my_proc_id_in_x_dim,my_proc_id_in_y_dim;
long long boxxsize,boxysize; // sizes of a map fragment handled by each process
int myrank,proccount;
MPI_Offset filesize;
long long x,y; // counters to go through a map fragment
double max_similarity,my_similarity,my_temp_similarity;
double cell_val;
int provided_thread_support;
MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE, &provided_thread_support);
// first read the file name from command line
if (argc<7) {
printf("\nSyntax: 2Dmapsearch-MPIIO <map_filename> mapxsize mapysize processes_in_x_dim processes_in_y_dim pmem_path\n");
MPI_Finalize();
exit(-1);
}
mapxsize=atol(argv[2]);
mapysize=atol(argv[3]);
processes_in_x_dim=atoi(argv[4]);
processes_in_y_dim=atoi(argv[5]);
if (mapxsize*mapysize<=0) {
printf("\nWrong map size given.\n");
MPI_Finalize();
exit(-1);
}
// find out my rank and the number of processes
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
MPI_Comm_size(MPI_COMM_WORLD,&proccount);
// now check if the number of processes matches the specified processes in dims
if (proccount!=(processes_in_x_dim*processes_in_y_dim)) {
printf("\nThe number of processes started does not match processes_in_x_dim*processes_in_y_dim.\n");
MPI_Finalize();
exit(-1);
}
MPI_Info info;
MPI_Info_create(&info);
MPI_Info_set(info,"pmem_path",argv[6]);
MPI_Info_set(info,"pmem_io_mode","0");
MPI_File_open(MPI_COMM_WORLD,argv[1],MPI_MODE_RDWR,info,&file) ;
// now check the size of the file vs the given map size
MPI_File_get_size(file,&filesize);
if (filesize<mapxsize*mapysize) {
printf("\nFile too small for the specified map size.\n");
MPI_File_close(&file);
MPI_Finalize();
exit(-1);
}
// now each process should determine its bounding box for the map
// length of each box will be (mapxsize/processes_in_x_dim) and similarly for the y dimension
boxxsize=(mapxsize/processes_in_x_dim);
boxysize=(mapysize/processes_in_y_dim);
my_proc_id_in_x_dim=myrank%processes_in_x_dim;
my_proc_id_in_y_dim=myrank/processes_in_x_dim;
myxmin=my_proc_id_in_x_dim*boxxsize;
myymin=my_proc_id_in_y_dim*boxysize;
myxmax=myxmin+boxxsize;
myymax=myymin+boxysize;
// now each process should scan its fragment
// if a certain element is detected then the application scans its immediate surroundings for elements of some other types
my_similarity=0;
for(x=myxmin;x<myxmax;x++)
for(y=myymin;y<myymax;y++) {
cell_val=get_xy_cell(x,y,file,mapxsize,mapysize);
if ((cell_val>=CELL_VAL_LOW_THRESHOLD) && (cell_val<=CELL_VAL_HIGH_THRESHOLD))
my_temp_similarity=eval_surrounding(x,y,file,mapxsize,mapysize);
if (my_temp_similarity>=my_similarity)
my_similarity=my_temp_similarity;
}
// now all processes should select the highest similarity
MPI_Reduce(&my_similarity,&max_similarity,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
if (!myrank) {
printf("\nThe final similarity is %f\n",max_similarity);
}
MPI_File_close(&file);
MPI_Finalize();
}
void parallel_readwrite(char *file_name, void *dump_buffer,
int type_of_file, int is_write, long long offset)
{
#if MPI && DO_PARALLEL_WRITE
MPI_File fh;
MPI_Status status;
MPI_Datatype mpi_elementary_type, mpi_file_type;
int file_open_error, file_write_error ;
int error_string_length;
char error_string[BUFSIZ];
MPI_Offset file_size;
int count;
void *mpi_buffer;
size_t mpi_buffer_size;
int mode;
MPI_Offset mpi_offset;
MPI_Barrier(MPI_COMM_WORLD);
if (is_write) {
mode = MPI_MODE_CREATE | MPI_MODE_WRONLY | MPI_MODE_APPEND;
}
else {
mode = MPI_MODE_RDONLY;
}
file_open_error = MPI_File_open(MPI_COMM_WORLD, file_name,
mode,
MPI_INFO_NULL, &fh);
if (file_open_error != MPI_SUCCESS) {
MPI_Error_string(file_open_error, error_string,
&error_string_length);
fprintf(stderr, "parallel_readwrite(): error opening file: %3d: %s\n", mpi_rank, error_string);
MPI_Abort(MPI_COMM_WORLD, file_open_error);
/* It is still OK to abort, because we have failed to
open the file. */
}
else {
// if (i_am_the_master)
// chmod(file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (offset < 0L) {
if(is_write) {
MPI_File_get_position(fh, &mpi_offset);
offset = mpi_offset;
}
else {
offset = 0L;
}
}
MPI_Barrier(MPI_COMM_WORLD);
//differentiate data type and buffers involved based on file type
if( DUMP_FILE == type_of_file ) {
mpi_elementary_type = MPI_DUMP_TYPE;
mpi_file_type = dump_file_type;
mpi_buffer = (void*)dump_buffer;
mpi_buffer_size = dump_buffer_size;
}
else if( GDUMP_FILE == type_of_file){
mpi_elementary_type = MPI_GDUMP_TYPE;
mpi_file_type = gdump_file_type;
mpi_buffer = (void*)gdump_buffer;
mpi_buffer_size = gdump_buffer_size;
}
else if( GDUMP2_FILE == type_of_file){
mpi_elementary_type = MPI_GDUMP2_TYPE;
mpi_file_type = gdump2_file_type;
mpi_buffer = (void*)gdump2_buffer;
mpi_buffer_size = gdump2_buffer_size;
}
else if( RDUMP_FILE == type_of_file){
mpi_elementary_type = MPI_RDUMP_TYPE;
mpi_file_type = rdump_file_type;
mpi_buffer = (void*)rdump_buffer;
mpi_buffer_size = rdump_buffer_size;
}
else if( FDUMP_FILE == type_of_file){
mpi_elementary_type = MPI_FDUMP_TYPE;
mpi_file_type = fdump_file_type;
mpi_buffer = (void*)fdump_buffer;
mpi_buffer_size = fdump_buffer_size;
}
else {
if(i_am_the_master)
fprintf(stderr, "Unknown file type %d\n", type_of_file);
MPI_File_close(&fh);
MPI_Finalize();
exit(2);
}
MPI_File_set_view(fh, offset, mpi_elementary_type, mpi_file_type, "native", MPI_INFO_NULL);
if (is_write) {
file_write_error =
MPI_File_write_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type,
&status);
}
else {
file_write_error =
MPI_File_read_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type,
&status);
}
if (file_write_error != MPI_SUCCESS) {
MPI_Error_string(file_write_error, error_string,
&error_string_length);
fprintf(stderr, "parallel_readwrite(): error %s file: %3d: %s\n",
(is_write)?("writing"):("reading"), mpi_rank, error_string);
MPI_File_close(&fh);
//if (i_am_the_master) MPI_File_delete(file_name, MPI_INFO_NULL);
MPI_Finalize();
exit(1);
}
// MPI_Get_count(&status, MPI_FLOAT, &count);
// MPI_File_get_size(fh, &file_size);
// if(1) {
// printf("%3d: wrote %d floats, expected to write %lld floats\n", mpi_rank, count, (long long int)dump_buffer_size);
// printf("%3d: file size is %lld bytes, header-related offset is %lld\n", mpi_rank, file_size, offset);
// }
MPI_File_close(&fh);
}
#endif
}
static int
test_mpio_overlap_writes(char *filename)
{
int mpi_size, mpi_rank;
MPI_Comm comm;
MPI_Info info = MPI_INFO_NULL;
int color, mrc;
MPI_File fh;
int i;
int vrfyerrs, nerrs;
unsigned char buf[4093]; /* use some prime number for size */
int bufsize = sizeof(buf);
MPI_Offset stride;
MPI_Offset mpi_off;
MPI_Status mpi_stat;
if (VERBOSE_MED)
printf("MPIO independent overlapping writes test on file %s\n",
filename);
nerrs = 0;
/* set up MPI parameters */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
/* Need at least 2 processes */
if (mpi_size < 2) {
if (MAINPROCESS)
printf("Need at least 2 processes to run MPIO test.\n");
printf(" -SKIP- \n");
return 0;
}
/* splits processes 0 to n-2 into one comm. and the last one into another */
color = ((mpi_rank < (mpi_size - 1)) ? 0 : 1);
mrc = MPI_Comm_split (MPI_COMM_WORLD, color, mpi_rank, &comm);
VRFY((mrc==MPI_SUCCESS), "Comm_split succeeded");
if (color==0){
/* First n-1 processes (color==0) open a file and write it */
mrc = MPI_File_open(comm, filename, MPI_MODE_CREATE|MPI_MODE_RDWR,
info, &fh);
VRFY((mrc==MPI_SUCCESS), "");
stride = 1;
mpi_off = mpi_rank*stride;
while (mpi_off < MPIO_TEST_WRITE_SIZE){
/* make sure the write does not exceed the TEST_WRITE_SIZE */
if (mpi_off+stride > MPIO_TEST_WRITE_SIZE)
stride = MPIO_TEST_WRITE_SIZE - mpi_off;
/* set data to some trivial pattern for easy verification */
for (i=0; i<stride; i++)
buf[i] = (unsigned char)(mpi_off+i);
mrc = MPI_File_write_at(fh, mpi_off, buf, (int)stride, MPI_BYTE,
&mpi_stat);
VRFY((mrc==MPI_SUCCESS), "");
/* move the offset pointer to last byte written by all processes */
mpi_off += (mpi_size - 1 - mpi_rank) * stride;
/* Increase chunk size without exceeding buffer size. */
/* Then move the starting offset for next write. */
stride *= 2;
if (stride > bufsize)
stride = bufsize;
mpi_off += mpi_rank*stride;
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
mrc = MPI_Comm_free(&comm);
VRFY((mrc==MPI_SUCCESS), "MPI_Comm_free");
/* sync with the other waiting processes */
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync after writes");
}else{
/* last process waits till writes are done,
* then opens file to verify data.
*/
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync after writes");
mrc = MPI_File_open(comm, filename, MPI_MODE_RDONLY,
info, &fh);
VRFY((mrc==MPI_SUCCESS), "");
stride = bufsize;
for (mpi_off=0; mpi_off < MPIO_TEST_WRITE_SIZE; mpi_off += bufsize){
/* make sure it does not read beyond end of data */
if (mpi_off+stride > MPIO_TEST_WRITE_SIZE)
stride = MPIO_TEST_WRITE_SIZE - mpi_off;
mrc = MPI_File_read_at(fh, mpi_off, buf, (int)stride, MPI_BYTE,
&mpi_stat);
VRFY((mrc==MPI_SUCCESS), "");
vrfyerrs=0;
for (i=0; i<stride; i++){
unsigned char expected;
expected = (unsigned char)(mpi_off+i);
if ((expected != buf[i]) &&
(vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED)) {
printf("proc %d: found data error at [%ld], expect %u, got %u\n",
mpi_rank, (long)(mpi_off+i), expected, buf[i]);
}
}
if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED)
printf("proc %d: [more errors ...]\n", mpi_rank);
nerrs += vrfyerrs;
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
mrc = MPI_Comm_free(&comm);
VRFY((mrc==MPI_SUCCESS), "MPI_Comm_free");
}
/*
* one more sync to ensure all processes have done reading
* before ending this test.
*/
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync before leaving test");
return (nerrs);
}
int MPI_File_fclose(MPI_FFile * handle)
{
MPI_File_fflush(handle);
MPI_File_close(&(handle->fh));
return 0;
}
int main(int argc, char **argv)
{
int *writebuf, *readbuf, i, mynod, nprocs, len, err;
char *filename;
int errs=0, toterrs;
MPI_Datatype newtype;
MPI_File fh;
MPI_Status status;
MPI_Info info;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: coll_test -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+1);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
writebuf = (int *) malloc(BUFSIZE*sizeof(int));
readbuf = (int *) malloc(BUFSIZE*sizeof(int));
/* test atomicity of contiguous accesses */
/* initialize file to all zeros */
if (!mynod) {
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
MPI_File_close(&fh);
#if VERBOSE
fprintf(stderr, "\ntesting contiguous accesses\n");
#endif
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
/* set atomicity to true */
err = MPI_File_set_atomicity(fh, 1);
if (err != MPI_SUCCESS) {
fprintf(stderr, "Atomic mode not supported on this file system.\n");fflush(stderr);
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Barrier(MPI_COMM_WORLD);
/* process 0 writes and others concurrently read. In atomic mode,
the data read must be either all old values or all new values; nothing
in between. */
if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) { /* the rest must also be 0 */
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 0) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else if (readbuf[0] == 10) { /* the rest must also be 10 */
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 10) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else {
errs++;
fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
/* repeat the same test with a noncontiguous filetype */
MPI_Type_vector(BUFSIZE, 1, 2, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
MPI_Info_create(&info);
/* I am setting these info values for testing purposes only. It is
better to use the default values in practice. */
MPI_Info_set(info, "ind_rd_buffer_size", "1209");
MPI_Info_set(info, "ind_wr_buffer_size", "1107");
if (!mynod) {
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, info, &fh);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 0;
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
MPI_File_close(&fh);
#if VERBOSE
fprintf(stderr, "\ntesting noncontiguous accesses\n");
#endif
}
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i<BUFSIZE; i++) writebuf[i] = 10;
for (i=0; i<BUFSIZE; i++) readbuf[i] = 20;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE |
MPI_MODE_RDWR, info, &fh);
MPI_File_set_atomicity(fh, 1);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) MPI_File_write(fh, writebuf, BUFSIZE, MPI_INT, &status);
else {
err = MPI_File_read(fh, readbuf, BUFSIZE, MPI_INT, &status);
if (err == MPI_SUCCESS) {
if (readbuf[0] == 0) {
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 0) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 0\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else if (readbuf[0] == 10) {
for (i=1; i<BUFSIZE; i++)
if (readbuf[i] != 10) {
errs++;
fprintf(stderr, "Process %d: readbuf[%d] is %d, should be 10\n", mynod, i, readbuf[i]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
else {
errs++;
fprintf(stderr, "Process %d: readbuf[0] is %d, should be either 0 or 10\n", mynod, readbuf[0]);
}
}
}
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (mynod == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
MPI_Type_free(&newtype);
MPI_Info_free(&info);
free(writebuf);
free(readbuf);
free(filename);
MPI_Finalize();
return 0;
}
void read_unordered_shared( const char * filename, void * local_ptr, size_t size ) {
MPI_Status status;
MPI_File infile;
MPI_Datatype datatype;
MPI_Info info;
char * local_char_ptr = static_cast< char * >( local_ptr );
// Stupid MPI uses a signed integer for the count of data elements
// to write. On all the machines we use, this means the max count
// is 2^31-1. To work around this, we create a datatype large
// enough that we never need a count value larger than 2^30.
// move this many gigabyte chunks
const size_t gigabyte = 1L << 30;
size_t round_count = size / gigabyte;
// if there will be any bytes left over, move those too
if( size & (gigabyte - 1) ) round_count++;
MPI_CHECK( MPI_Info_create( &info ) );
if( FLAGS_optimize_for_lustre ) {
std::map< const std::string, const std::string > info_map = {{
// disable independent file operations, since we know this
// routine is the only one touching the file
{ "romio_no_indep_rw", "true" }
// // disable collective io on lustre for "small" files <= 1GB
// , { "romio_lustre_ds_in_coll", "1073741825" ) );
// set collective buffering block size to something reasonable
, { "cb_buffer_size", "33554432" }
// // disable collective buffering for writing
// , { "romio_cb_write", "disable" }
// // disable collective buffering for writing
// , { "romio_cb_read", "disable" }
// disable data sieving for writing
, { "romio_ds_write", "disable" }
// disable data sieving for writing
, { "romio_ds_read", "disable" }
// enable direct IO
, { "direct_read", "true" }
, { "direct_write", "true" }
// ???
// , { "romio_lustre_co_ratio", "1" }
// maybe
// , { "access_style", "read_once" }
}};
set_mpi_info( info, info_map );
}
// open file for reading
int mode = MPI_MODE_RDONLY;
MPI_CHECK( MPI_File_open( global_communicator.grappa_comm, const_cast< char * >( filename ), mode, info, &infile ) );
// make sure we will read exactly the whole file
int64_t total_size = 0;
MPI_CHECK( MPI_Reduce( &size, &total_size, 1, MPI_INT64_T, MPI_SUM, 0, global_communicator.grappa_comm ) );
if( 0 == Grappa::mycore() ) {
MPI_Offset file_size = 0;
MPI_CHECK( MPI_File_get_size( infile, &file_size ) );
CHECK_EQ( file_size, total_size ) << "Sizes don't line up to read the enitre file?";
}
// // dump file info for debugging
// if( Grappa::mycore == 0 ) {
// dump_mpi_file_info( infile );
// }
// compute number of rounds required to read entire file
MPI_CHECK( MPI_Allreduce( MPI_IN_PLACE, &round_count, 1, MPI_INT64_T, MPI_MAX, global_communicator.grappa_comm ) );
// read file in gigabyte-sized rounds
for( int i = 0; i < round_count; ++i ) {
if( size > gigabyte ) {
MPI_CHECK( MPI_File_read_shared( infile, local_char_ptr, gigabyte, MPI_BYTE, &status ) );
size -= gigabyte;
local_char_ptr += gigabyte;
} else {
MPI_CHECK( MPI_File_read_shared( infile, local_char_ptr, size, MPI_BYTE, &status ) );
}
}
MPI_CHECK( MPI_Info_free( &info ) );
MPI_CHECK( MPI_File_close( &infile ) );
}
void fileparse(unsigned int num_rows) {
int world_size, world_rank;
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
char search_phrase[] = "AdeptProject";
size_t sp_len = strlen(search_phrase);
unsigned int desired_line_len = 81;
char line[desired_line_len];
srand(time(NULL)); // Set seed
int i = 0;
int r = 0;
int m = 0;
int mismatch = 0;
int r_count = 0;
int m_count = 0;
struct timespec start, end;
/* p_num_rows is the number of rows across all processes */
unsigned int p_num_rows;
p_num_rows = (unsigned int) (num_rows * world_size);
/* Generate (on the fly) the test file for the run */
/* Make this single threaded for ease */
if (world_rank == 0) {
FILE* fp;
fp = fopen("testfile", "w+");
for (i = 0; i < p_num_rows; i++) {
r = create_line(search_phrase, sp_len, line, desired_line_len);
m = seek_match(search_phrase, sp_len, line, desired_line_len);
if (r != m) {
mismatch++;
}
if (r == 0) {
r_count++;
}
if (m == 0) {
m_count++;
}
fprintf(fp, "%s\n", line);
}
fsync(fileno(fp));
fclose(fp);
}
m = 0;
MPI_Info info;
MPI_Info_create(&info);
MPI_File fh;
MPI_Status status;
/* For holding the data from the file before parsing */
char *lb = (char*) malloc(sizeof (char)*num_rows * (desired_line_len + 1));
char *lbp = NULL;
MPI_Barrier(MPI_COMM_WORLD);
if (world_rank == 0) {
clock_gettime(CLOCK, &start);
}
m_count = 0;
/* This part should use MPI-IO */
MPI_File_open(MPI_COMM_WORLD, "testfile", MPI_MODE_RDWR | MPI_MODE_CREATE, info, &fh);
MPI_File_read_at(fh, world_rank * num_rows * (desired_line_len + 1), lb, num_rows * (desired_line_len + 1), MPI_CHAR, &status);
for (i = 0; i < num_rows; i++) {
lbp = &lb[i * (desired_line_len + 1)];
m = seek_match(search_phrase, sp_len, lbp, desired_line_len);
if (m == 0) {
m_count++;
}
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_close(&fh);
if (world_rank == 0) {
clock_gettime(CLOCK, &end);
elapsed_time_hr(start, end, "Fileparse");
}
MPI_Barrier(MPI_COMM_WORLD);
unlink("testfile"); // Use this to ensure the generated file is removed from the system upon finish
}
void write_unordered_shared( const char * filename, void * local_ptr, size_t size ) {
MPI_Status status;
MPI_File outfile;
MPI_Datatype datatype;
MPI_Info info;
char * local_char_ptr = static_cast< char * >( local_ptr );
// Stupid MPI uses a signed integer for the count of data elements
// to write. On all the machines we use, this means the max count
// is 2^31-1. To work around this, we create a datatype large
// enough that we never need a count value larger than 2^30.
// move this many gigabyte chunks
const size_t gigabyte = 1L << 30;
size_t round_count = size / gigabyte;
// if there will be any bytes left over, move those too
if( size & (gigabyte - 1) ) round_count++;
MPI_CHECK( MPI_Info_create( &info ) );
if( FLAGS_optimize_for_lustre ) {
std::map< const std::string, const std::string > info_map = {{
// disable independent file operations, since we know this
// routine is the only one touching the file
{ "romio_no_indep_rw", "true" }
// it's important to set lustre striping properly for performance.
// we're supposed to be able to do this through MPI ROMIO, but mpi installations are not always configured to allow this.
// if not, then before saving a file, run a command like this to create it:
// lfs setstripe -c -1 -s 32m <filename>
// below are what we'd like to do.
// we want 32MB lustre blocks, but this probably doesn't do anything
, { "striping_unit", "33554432" }
// we want to use all lustre OSTs, but this probably doesn't do anything
, { "striping_factor", "-1" }
// // disable collective io on lustre for "small" files <= 1GB
// , { "romio_lustre_ds_in_coll", "1073741825" ) );
// set collective buffering block size to something reasonable
, { "cb_buffer_size", "33554432" }
// // disable collective buffering for writing
// , { "romio_cb_write", "disable" }
// // disable collective buffering for writing
// , { "romio_cb_read", "disable" }
// disable data sieving for writing
, { "romio_ds_write", "disable" }
// disable data sieving for writing
, { "romio_ds_read", "disable" }
// enable direct IO
, { "direct_read", "true" }
, { "direct_write", "true" }
// ???
// , { "romio_lustre_co_ratio", "1" }
// maybe
// , { "access_style", "read_once" }
}};
set_mpi_info( info, info_map );
}
// open file for writing
int mode = MPI_MODE_CREATE | MPI_MODE_WRONLY;
MPI_CHECK( MPI_File_open( global_communicator.grappa_comm, const_cast< char * >( filename ), mode, info, &outfile ) );
// drop any data previously in file
MPI_CHECK( MPI_File_set_size( outfile, 0 ) );
// dump file info for debugging
// if( Grappa::mycore == 0 ) {
// dump_mpi_file_info( outfile );
// }
// compute number of rounds required to read entire file
MPI_CHECK( MPI_Allreduce( MPI_IN_PLACE, &round_count, 1, MPI_INT64_T, MPI_MAX, global_communicator.grappa_comm ) );
// write file in gigabyte-sized rounds
for( int i = 0; i < round_count; ++i ) {
if( size > gigabyte ) {
MPI_CHECK( MPI_File_write_shared( outfile, local_char_ptr, gigabyte, MPI_BYTE, &status ) );
size -= gigabyte;
local_char_ptr += gigabyte;
} else {
MPI_CHECK( MPI_File_write_shared( outfile, local_char_ptr, size, MPI_BYTE, &status ) );
}
}
MPI_CHECK( MPI_Info_free( &info ) );
MPI_CHECK( MPI_File_close( &outfile ) );
}
void writetofile(int rank, int a)
{
#if defined(MPI) && defined(FILEIMAGE)
// dump all the tiles to 1 file using MPI-IO
int color = 0;
MPI_Comm tilecomm;
MPI_File fhtout;
if (owneroftile == rank) color = 1;
PRINTDEBUG("%3d: tileowner=%d, color=%d\n", rank, owneroftile, color);
MPI_Comm_split(MPI_COMM_WORLD, color, rank, &tilecomm);
if (color) {
int cartrank;
MPI_Comm cartcomm;
const MPI_Status status;
// init parameters for Cart_create
int gsizes[2] = {viewport.h, viewport.w*ELEMENTSPERPIXEL};
int psizes[2] = {viewport.tiley, viewport.tilex};
int lsizes[2] = {gsizes[0]/psizes[0], (gsizes[1])/psizes[1]};
int dims[2] = {psizes[0], psizes[1]};
int periods[2] = {1,1};
int coords[2];
PRINTDEBUG("%3d: gsizes=%d,%d, psizes=%d,%d, lsizes=%d,%d\n", rank, gsizes[0], gsizes[1], psizes[0], psizes[1],
lsizes[0], lsizes[1]);
// Create cartesian coord
MPI_Cart_create(tilecomm, 2, dims, periods, 0, &cartcomm);
MPI_Comm_rank(cartcomm, &cartrank);
MPI_Cart_coords(cartcomm, cartrank, 2, coords);
// check coords
PRINTDEBUG("%3d: coords=%d,%d\n", cartrank, coords[0], coords[1]);
/* global indices of first element of local array */
int startindex[2];
startindex[0] = coords[0] * lsizes[0];
startindex[1] = coords[1] * lsizes[1];
PRINTDEBUG("%3d: start=%d,%d\n", cartrank, startindex[0], startindex[1]);
int filetype;
filetype=0;
MPI_Type_create_subarray(2, gsizes, lsizes, startindex, MPI_ORDER_C, MPI_FLOAT, &filetype);
MPI_Type_commit(&filetype);
char imagefull[32];
sprintf(imagefull,"imagefull%d.raw", a);
PRINTDEBUG("%s\n", imagefull);
MPI_File_open(cartcomm, imagefull, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fhtout);
MPI_File_set_view(fhtout, 0, MPI_FLOAT, filetype, "native", MPI_INFO_NULL);
int localarraysize = lsizes[0] * lsizes[1];
PRINTDEBUG("%3d: size=%d\n", cartrank, localarraysize);
MPI_File_write(fhtout, tilebuffer, localarraysize, MPI_FLOAT, &status);
int count;
MPI_Get_count(&status, MPI_FLOAT, &count);
PRINTDEBUG("%3d: cnt=%d\n", cartrank, count);
MPI_File_close(&fhtout);
MPI_Comm_free(&cartcomm);
MPI_Type_free(&filetype);
// free(tilebuffer);
}
MPI_Comm_free(&tilecomm);
#endif
}
int run_test (test_param_t *test) {
ADIO_Offset st_offset, end_offset;
MPI_File fh;
int is_contig;
int ind_err = 0, exp_err = 0;
MPI_Datatype filetype;
MPI_Type_struct (test->type_count, test->type_blocklens,
test->type_indices, test->type_oldtypes, &filetype);
MPI_Type_commit (&filetype);
MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
MPI_File_seek (fh, test->offset, MPI_SEEK_SET);
ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE, ADIO_INDIVIDUAL,
test->offset, &st_offset, &end_offset);
ind_err = 0;
if (st_offset != test->correct_st_offset) {
printf ("Individual st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
ind_err = 1;
}
if (end_offset != test->correct_end_offset) {
printf ("Individual st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
ind_err = 1;
}
MPI_File_close (&fh);
if (ind_err)
printf ("Individual Calc FAILED\n");
MPI_File_open (MPI_COMM_WORLD, "test_file.txt" , MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
if (!is_contig)
MPI_File_set_view (fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
MPI_File_seek (fh, 0, MPI_SEEK_SET);
ADIOI_Calc_bounds ((ADIO_File) fh, test->count, MPI_BYTE,
ADIO_EXPLICIT_OFFSET, test->offset, &st_offset,
&end_offset);
exp_err = 0;
if (st_offset != test->correct_st_offset) {
printf ("Explicit st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
exp_err = 1;
}
if (end_offset != test->correct_end_offset) {
printf ("Explicit st_offset = %lld end_offset = %lld\n",
st_offset, end_offset);
exp_err = 1;
}
if (exp_err)
printf ("Explicit Calc FAILED\n");
MPI_File_close (&fh);
if (!is_contig)
MPI_Type_free (&filetype);
return (exp_err || ind_err);
}
int main(int argc, char **argv) {
int rank, size;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
if(argc != 4 && rank == 0) {
fprintf(stderr, "Usage: %s str1_file str2_file out_file\n", argv[0]);
MPI_Abort(MPI_COMM_WORLD, 1);
}
if(size == 1) {
fprintf(stderr, "At least 2 processes expected\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Barrier(MPI_COMM_WORLD);
char *alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
int alphabet_len = strlen(alphabet);
#ifdef DEBUG_TIME
double start_t, end_t;
start_t = end_t = 0; // suppress warnings about uninitialized variables
if(rank == 0)
start_t = MPI_Wtime();
#endif
// open files with str1 and str2 and read it's contents
char *filename_str1 = argv[1];
char *filename_str2 = argv[2];
char *filename_out = argv[3];
MPI_File F_str1, F_str2;
MPI_File_open(MPI_COMM_WORLD, filename_str1, MPI_MODE_RDONLY, MPI_INFO_NULL, &F_str1);
MPI_File_open(MPI_COMM_WORLD, filename_str2, MPI_MODE_RDONLY, MPI_INFO_NULL, &F_str2);
// get lengths of str1 and str2
MPI_Offset str1_len, str2_len;
MPI_File_get_size(F_str1, &str1_len);
MPI_File_get_size(F_str2, &str2_len);
char *str1 = (char*)malloc(sizeof(char) * (str1_len + 1));
char *str2 = (char*)malloc(sizeof(char) * (str2_len + 1));
// now read str1 and str2 from files
MPI_File_read_at(F_str1, (MPI_Offset)0, str1, str1_len, MPI_CHAR, MPI_STATUS_IGNORE);
str1[str1_len] = '\0';
MPI_File_read_at(F_str2, (MPI_Offset)0, str2, str2_len, MPI_CHAR, MPI_STATUS_IGNORE);
str2[str2_len] = '\0';
MPI_File_close(&F_str1);
MPI_File_close(&F_str2);
#ifdef DEBUG_TIME
if(rank == 0) {
end_t = MPI_Wtime();
printf("%f seconds for reading from files\n", end_t - start_t);
}
#endif
int i;
int start_index, end_index;
int chunk_size = str2_len / (size - 1);
if(rank > 0) {
// every rank>0 processes part of str2
// then rank=0 reduces results
start_index = (rank - 1) * chunk_size;
end_index = rank * chunk_size - 1;
if(rank == size - 1) // last rank processes everything, what is left
end_index = str2_len - 1;
int *appearances = (int*)malloc(sizeof(int) * alphabet_len);
// appearances[0] corresponds to 'A'
// appearances[25] - 'Z'
memset(appearances, 0, sizeof(int) * alphabet_len);
int letter;
for(i = start_index; i <= end_index; i++) {
letter = str2[i] - 'A';
appearances[letter] += 1;
}
MPI_Gatherv(appearances, alphabet_len, MPI_INT, NULL, NULL, NULL, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(appearances, alphabet_len, MPI_INT, 0, MPI_COMM_WORLD);
// now every rank>0 contains _global_ count of how many times every letter from alphabet appears in str2
int *ds[alphabet_len], ds_length[alphabet_len]; // ds=decreasing sequence
for(i = 0; i < alphabet_len; i++) {
ds[i] = (int*)malloc(sizeof(int) * appearances[i]);
// ds[0] corresponds to letter 'A'
// if str2='bacdeafa', then ds[0]={8, 6, 2}
memset(ds[i], 0, sizeof(int) * appearances[i]);
ds_length[i] = 0;
}
// every rank>0 processes it's part of str2
// but now rank=1 processes _last_ part of str2
// for example, if str2='abcdefghj', size=4 (including rank=0)
// then rank=1 processes 'ghj', rank=2 - 'def', rank=3 - 'abc'
start_index = (size - (rank + 1)) * chunk_size;
end_index = (size - rank) * chunk_size - 1;
if(rank == 1) {
end_index = str2_len - 1;
}
for(i = end_index; i >= start_index; i--) {
letter = str2[i] - 'A';
ds[letter][ds_length[letter]] = i;
ds_length[letter] += 1;
}
// decreasing sequence is done, send results to rank=0
MPI_Gather(ds_length, alphabet_len, MPI_INT, NULL, 0, MPI_INT, 0, MPI_COMM_WORLD);
// now send all ds's
for(i = 0; i < alphabet_len; i++) {
MPI_Gatherv(ds[i], ds_length[i], MPI_INT, NULL, NULL, NULL, MPI_INT, 0, MPI_COMM_WORLD);
}
// cleanup
for(i = 0; i < alphabet_len; i++) {
free(ds[i]);
}
free(appearances);
}
if(rank == 0) {
#ifdef DEBUG_TIME
start_t = MPI_Wtime();
#endif
int *appearances = (int*)malloc(sizeof(int) * ((size - 1) * alphabet_len + 1)); // '+ 1' to receive value from rank=0
// but it is unused
int *recvcounts = (int*)malloc(sizeof(int) * size);
int *displs = (int*)malloc(sizeof(int) * size);
recvcounts[0] = 1;
displs[0] = 0;
for(i = 1; i < size; i++) {
start_index = (i - 1) * chunk_size;
//end_index = i * chunk_size - 1;
//if(i == size - 1)
// end_index = str2_len - 1;
recvcounts[i] = alphabet_len;
displs[i] = alphabet_len * (i - 1) + 1;
}
MPI_Gatherv(MPI_IN_PLACE, 1, MPI_INT, appearances, recvcounts, displs, MPI_INT, 0, MPI_COMM_WORLD);
// reduce data from all processes
int k;
int letter_displacement;
for(i = 1; i <= alphabet_len; i++) {
letter_displacement = i - 1;
for(k = 2; k < size; k++) {
appearances[i] += appearances[displs[k] + letter_displacement];
}
}
#ifdef DEBUG_TIME
end_t = MPI_Wtime();
printf("%f seconds for calculation of how many times every letter appears in str2\n", end_t - start_t);
start_t = MPI_Wtime();
#endif
// now appearances[1] through appearances[26] contain _global_ count of how many times every letter appears in str2
// broadcast it to all processes
MPI_Bcast(appearances + 1, alphabet_len, MPI_INT, 0, MPI_COMM_WORLD);
// receive decreasing sequences' lengths
int ds_lengths[alphabet_len * size];
// rank=0 sends 26 MPI_INTs, so ds_lengths[0] through ds_lengths[25] are not significant
MPI_Gather(MPI_IN_PLACE, alphabet_len, MPI_INT, ds_lengths, alphabet_len, MPI_INT, 0, MPI_COMM_WORLD);
int *ds[alphabet_len];
recvcounts[0] = 1;
displs[0] = 0;
for(i = 0; i < alphabet_len; i++) {
// only appearances[1] through appearances[26] contain _global_ count
ds[i] = (int*)malloc(sizeof(int) * (appearances[i + 1] + 1)); // '+1' to receive value from rank=0
for(k = 1; k < size; k++) {
recvcounts[k] = ds_lengths[alphabet_len * k + i];
//k=1 - rank=1. ds_length[26] - 'A', ds_lengths[27] - 'B'
//k=2 - rank=2, ds_length[52] - 'A', ds_lengths[53] - 'B'
if(k == 1) {
displs[k] = 1;
} else {
displs[k] = ds_lengths[alphabet_len * (k - 1) + i] + displs[k - 1];
}
}
MPI_Gatherv(MPI_IN_PLACE, 1, MPI_INT, ds[i], recvcounts, displs, MPI_INT, 0, MPI_COMM_WORLD);
}
#ifdef DEBUG_TIME
end_t = MPI_Wtime();
printf("%f seconds for creation of decreasing sequences\n", end_t - start_t);
start_t = MPI_Wtime();
#endif
int letter_appearances;
int dec_sequences_count = 1;
int **dec_sequences = (int**)malloc(sizeof(int**) * 1);
dec_sequences[0] = (int*)malloc(sizeof(int) * appearances[str1[0] - 'A' + 1]);
int *dec_sequences_lengths = (int*)malloc(sizeof(int) * 1);
dec_sequences_lengths[0] = appearances[str1[0] - 'A' + 1];
for(i = 0; i < dec_sequences_lengths[0]; i++) {
dec_sequences[0][i] = ds[(int)(str1[0] - 'A')][i + 1];
}
int seq;
int letter;
for(i = 1; i < str1_len; i++) {
letter = str1[i] - 'A';
letter_appearances = appearances[letter + 1]; // '+1' because appearances[0] is not valid
for(k = 1; k <= letter_appearances; k++) {
seq = lower_bound(dec_sequences, dec_sequences_lengths, dec_sequences_count, ds[letter][k]);
if(seq >= 0) {
dec_sequences_lengths[seq] += 1;
dec_sequences[seq] = (int*)realloc(dec_sequences[seq], sizeof(int) * dec_sequences_lengths[seq]); // it is bad, i know
dec_sequences[seq][dec_sequences_lengths[seq] - 1] = ds[letter][k]; // but would not rewrite this, because it will still be slower
} else { // than lcs_sequential
dec_sequences_count += 1;
dec_sequences_lengths = (int*)realloc(dec_sequences_lengths, sizeof(int) * dec_sequences_count);
dec_sequences_lengths[dec_sequences_count - 1] = 1;
dec_sequences = (int**)realloc(dec_sequences, sizeof(int**) * dec_sequences_count);
dec_sequences[dec_sequences_count - 1] = (int*)malloc(sizeof(int) * 1);
dec_sequences[dec_sequences_count - 1][0] = ds[letter][k];
}
}
}
#ifdef DEBUG_TIME
end_t = MPI_Wtime();
printf("%f seconds for LCS creation\n", end_t - start_t);
#endif
FILE *F_out = fopen(filename_out, "w");
if(!F_out) {
fprintf(stderr, "Unable to open file '%s' for writing\nOutput to console...\n", filename_out);
F_out = stdout;
}
fprintf(F_out, "lcs length = %d\n", dec_sequences_count);
fprintf(F_out, "lcs sequence\n");
for(i = 0; i < dec_sequences_count; i++) {
fprintf(F_out, "%c", str2[dec_sequences[i][dec_sequences_lengths[i] - 1]]);
}
fprintf(F_out, "\n");
fclose(F_out);
// cleanup
free(dec_sequences_lengths);
for(i = 0; i < dec_sequences_count; i++) {
free(dec_sequences[i]);
}
free(dec_sequences);
for(i = 0; i < alphabet_len; i++) {
free(ds[i]);
}
free(appearances);
free(recvcounts);
free(displs);
}
free(str1);
free(str2);
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
char *filename;
int i, len, nprocs, amode, err, nerrs = 0;
int blen[2], disp[2];
MPI_Datatype etype, filetype;
MPI_File fh;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (nprocs != 1) {
fprintf(stderr, "Run this program on 1 process\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
len = 8;
filename = (char *) malloc(len + 10);
strcpy(filename, "testfile");
} else {
argv++;
len = (int) strlen(*argv);
filename = (char *) malloc(len + 1);
strcpy(filename, *argv);
}
MPI_File_delete(filename, MPI_INFO_NULL);
amode = MPI_MODE_RDWR | MPI_MODE_CREATE;
err = MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh);
CHECK_ERROR(err, nerrs);
/* create etype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with overlaps */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create filetype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with overlaps */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_close(&fh);
CHECK_ERROR(err, nerrs);
/* open the file for read only */
amode = MPI_MODE_RDONLY;
err = MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh);
CHECK_ERROR(err, nerrs);
/* create etype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create etype with overlaps (should be OK for read-only) */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &etype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&etype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, etype, etype, "native", MPI_INFO_NULL);
CHECK_ERROR(err, nerrs);
err = MPI_Type_free(&etype);
CHECK_ERROR(err, nerrs);
/* create filetype with negative disp */
disp[0] = -2;
disp[1] = 2;
blen[0] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with decreasing disp */
disp[0] = 3;
blen[0] = 1;
disp[1] = 0;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_EXPECTED_ERROR(MPI_ERR_IO, err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
/* create filetype with overlaps (should be OK for read-only) */
disp[0] = 0;
blen[0] = 3;
disp[1] = 1;
blen[1] = 1;
err = MPI_Type_indexed(2, blen, disp, MPI_INT, &filetype);
CHECK_ERROR(err, nerrs);
err = MPI_Type_commit(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL);
CHECK_ERROR(err, nerrs);
err = MPI_Type_free(&filetype);
CHECK_ERROR(err, nerrs);
err = MPI_File_close(&fh);
CHECK_ERROR(err, nerrs);
if (nerrs == 0)
printf(" No Errors\n");
MPI_Finalize();
return (nerrs > 0);
}
int main(int argc, char *argv[]) {
int i, n, nlocal;
int numprocs, myrank;
MPI_File f; char* filename = "input/8";
MPI_Status status;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
if(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &f) != MPI_SUCCESS) {
fprintf(stderr, "Cannot open file %s\n", filename);
MPI_Abort(MPI_COMM_WORLD, FILE_NOT_FOUND);
MPI_Finalize();
return 1;
}
MPI_File_seek(f, 0, MPI_SEEK_SET);
MPI_File_read(f, &n, 1, MPI_INT, &status);
nlocal = n/numprocs; if(myrank == numprocs - 1) nlocal = nlocal + n % numprocs;
int *a = (int *)malloc(nlocal * n * sizeof(int));
MPI_File_seek(f, (myrank * nlocal * n + 1) * sizeof(int), MPI_SEEK_SET);
MPI_File_read(f, &a[0], nlocal * n, MPI_INT, &status);
MPI_File_close(&f);
// int j;
// if(myrank == 3) {
// for(i = 0; i < nlocal; i++) {
// for(j = 0; j < n; j++) {
// printf("%d ", a[i * n +j]);
// }
// printf("\n");
// }
// }
double start = MPI_Wtime();
floyd_all_pairs_sp_1d(n, nlocal, a);
double stop = MPI_Wtime();
printf("[%d] Completed in %1.3f seconds\n", myrank, stop-start);
// if(myrank == 3) {
// for(i = 0; i < nlocal; i++) {
// for(j = 0; j < n; j++) {
// printf("%d ", a[i * n +j]);
// }
// printf("\n");
// }
// }
if(MPI_File_open(MPI_COMM_WORLD, "output/8", MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &f) != MPI_SUCCESS) {
printf("Cannot open file %s\n", "out");
MPI_Abort(MPI_COMM_WORLD, FILE_NOT_FOUND);
MPI_Finalize();
return 1;
}
if(myrank == 0) {
MPI_File_seek(f, 0, MPI_SEEK_SET);
MPI_File_write(f, &n, 1, MPI_INT, &status);
}
for(i = 0; i < nlocal; i++) {
MPI_File_seek(f, (myrank * nlocal * n + 1) * sizeof(int), MPI_SEEK_SET);
MPI_File_write(f, &a[0], nlocal * n, MPI_INT, &status);
}
MPI_File_close(&f);
free(a);
MPI_Finalize();
return 0;
}
static void close() {
if(mpiFile != MPI_FILE_NULL)
MPI_File_close(&mpiFile);
}
/*
* Verify that MPI_Offset exceeding 2**31 can be computed correctly.
* Print any failure as information only, not as an error so that this
* won't abort the remaining test or other separated tests.
*
* Test if MPIO can write file from under 2GB to over 2GB and then
* from under 4GB to over 4GB.
* Each process writes 1MB in round robin fashion.
* Then reads the file back in by reverse order, that is process 0
* reads the data of process n-1 and vice versa.
*/
static int
test_mpio_gb_file(char *filename)
{
int mpi_size, mpi_rank;
MPI_Info info = MPI_INFO_NULL;
int mrc;
MPI_File fh;
int i, j, n;
int vrfyerrs;
int writerrs; /* write errors */
int nerrs;
int ntimes; /* how many times */
char *buf = NULL;
char expected;
MPI_Offset size;
MPI_Offset mpi_off;
MPI_Offset mpi_off_old;
MPI_Status mpi_stat;
struct stat stat_buf;
int is_signed, sizeof_mpi_offset;
nerrs = 0;
/* set up MPI parameters */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
if (VERBOSE_MED)
printf("MPI_Offset range test\n");
/* figure out the signness and sizeof MPI_Offset */
mpi_off = 0;
is_signed = ((MPI_Offset)(mpi_off - 1)) < 0;
sizeof_mpi_offset = (int)(sizeof(MPI_Offset));
/*
* Verify the sizeof MPI_Offset and correctness of handling multiple GB
* sizes.
*/
if (MAINPROCESS){ /* only process 0 needs to check it*/
printf("MPI_Offset is %s %d bytes integeral type\n",
is_signed ? "signed" : "unsigned", (int)sizeof(MPI_Offset));
if (sizeof_mpi_offset <= 4 && is_signed){
printf("Skipped 2GB range test "
"because MPI_Offset cannot support it\n");
}else {
/* verify correctness of assigning 2GB sizes */
mpi_off = 2 * 1024 * (MPI_Offset)MB;
INFO((mpi_off>0), "2GB OFFSET assignment no overflow");
INFO((mpi_off-1)==TWO_GB_LESS1, "2GB OFFSET assignment succeed");
/* verify correctness of increasing from below 2 GB to above 2GB */
mpi_off = TWO_GB_LESS1;
for (i=0; i < 3; i++){
mpi_off_old = mpi_off;
mpi_off = mpi_off + 1;
/* no overflow */
INFO((mpi_off>0), "2GB OFFSET increment no overflow");
/* correct inc. */
INFO((mpi_off-1)==mpi_off_old, "2GB OFFSET increment succeed");
}
}
if (sizeof_mpi_offset <= 4){
printf("Skipped 4GB range test "
"because MPI_Offset cannot support it\n");
}else {
/* verify correctness of assigning 4GB sizes */
mpi_off = 4 * 1024 * (MPI_Offset)MB;
INFO((mpi_off>0), "4GB OFFSET assignment no overflow");
INFO((mpi_off-1)==FOUR_GB_LESS1, "4GB OFFSET assignment succeed");
/* verify correctness of increasing from below 4 GB to above 4 GB */
mpi_off = FOUR_GB_LESS1;
for (i=0; i < 3; i++){
mpi_off_old = mpi_off;
mpi_off = mpi_off + 1;
/* no overflow */
INFO((mpi_off>0), "4GB OFFSET increment no overflow");
/* correct inc. */
INFO((mpi_off-1)==mpi_off_old, "4GB OFFSET increment succeed");
}
}
}
/*
* Verify if we can write to a file of multiple GB sizes.
*/
if (VERBOSE_MED)
printf("MPIO GB file test %s\n", filename);
if (sizeof_mpi_offset <= 4){
printf("Skipped GB file range test "
"because MPI_Offset cannot support it\n");
}else{
buf = malloc(MB);
VRFY((buf!=NULL), "malloc succeed");
/* open a new file. Remove it first in case it exists. */
/* Must delete because MPI_File_open does not have a Truncate mode. */
/* Don't care if it has error. */
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE|MPI_MODE_RDWR,
info, &fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_OPEN");
printf("MPIO GB file write test %s\n", filename);
/* instead of writing every bytes of the file, we will just write
* some data around the 2 and 4 GB boundaries. That should cover
* potential integer overflow and filesystem size limits.
*/
writerrs = 0;
for (n=2; n <= 4; n+=2){
ntimes = GB/MB*n/mpi_size + 1;
for (i=ntimes-2; i <= ntimes; i++){
mpi_off = (i*mpi_size + mpi_rank)*(MPI_Offset)MB;
if (VERBOSE_MED)
HDfprintf(stdout,"proc %d: write to mpi_off=%016llx, %lld\n",
mpi_rank, mpi_off, mpi_off);
/* set data to some trivial pattern for easy verification */
for (j=0; j<MB; j++)
*(buf+j) = i*mpi_size + mpi_rank;
if (VERBOSE_MED)
HDfprintf(stdout,"proc %d: writing %d bytes at offset %lld\n",
mpi_rank, MB, mpi_off);
mrc = MPI_File_write_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat);
INFO((mrc==MPI_SUCCESS), "GB size file write");
if (mrc!=MPI_SUCCESS)
writerrs++;
}
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync after writes");
/*
* Verify if we can read the multiple GB file just created.
*/
/* open it again to verify the data written */
/* but only if there was no write errors */
printf("MPIO GB file read test %s\n", filename);
if (errors_sum(writerrs)>0){
printf("proc %d: Skip read test due to previous write errors\n",
mpi_rank);
goto finish;
}
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh);
VRFY((mrc==MPI_SUCCESS), "");
/* Only read back parts of the file that have been written. */
for (n=2; n <= 4; n+=2){
ntimes = GB/MB*n/mpi_size + 1;
for (i=ntimes-2; i <= ntimes; i++){
mpi_off = (i*mpi_size + (mpi_size - mpi_rank - 1))*(MPI_Offset)MB;
if (VERBOSE_MED)
HDfprintf(stdout,"proc %d: read from mpi_off=%016llx, %lld\n",
mpi_rank, mpi_off, mpi_off);
mrc = MPI_File_read_at(fh, mpi_off, buf, MB, MPI_BYTE, &mpi_stat);
INFO((mrc==MPI_SUCCESS), "GB size file read");
expected = i*mpi_size + (mpi_size - mpi_rank - 1);
vrfyerrs=0;
for (j=0; j<MB; j++){
if ((*(buf+j) != expected) &&
(vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED)){
printf("proc %d: found data error at [%ld+%d], expect %d, got %d\n",
mpi_rank, (long)mpi_off, j, expected, *(buf+j));
}
}
if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED)
printf("proc %d: [more errors ...]\n", mpi_rank);
nerrs += vrfyerrs;
}
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
/*
* one more sync to ensure all processes have done reading
* before ending this test.
*/
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync before leaving test");
/*
* Check if MPI_File_get_size works correctly. Some systems (only SGI Altix
* Propack 4 so far) return wrong file size. It can be avoided by reconfiguring
* with "--disable-mpi-size".
*/
#ifdef H5_HAVE_MPI_GET_SIZE
printf("Test if MPI_File_get_size works correctly with %s\n", filename);
mrc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, info, &fh);
VRFY((mrc==MPI_SUCCESS), "");
if (MAINPROCESS){ /* only process 0 needs to check it*/
mrc = MPI_File_get_size(fh, &size);
VRFY((mrc==MPI_SUCCESS), "");
mrc=stat(filename, &stat_buf);
VRFY((mrc==0), "");
/* Hopefully this casting is safe */
if(size != (MPI_Offset)(stat_buf.st_size)) {
printf("Warning: MPI_File_get_size doesn't return correct file size. To avoid using it in the library, reconfigure and rebuild the library with --disable-mpi-size.\n");
}
}
/* close file and free the communicator */
mrc = MPI_File_close(&fh);
VRFY((mrc==MPI_SUCCESS), "MPI_FILE_CLOSE");
/*
* one more sync to ensure all processes have done reading
* before ending this test.
*/
mrc = MPI_Barrier(MPI_COMM_WORLD);
VRFY((mrc==MPI_SUCCESS), "Sync before leaving test");
#else
printf("Skipped testing MPI_File_get_size because it's disabled\n");
#endif
}
finish:
if (buf)
HDfree(buf);
return (nerrs);
}
int main(int argc, char *argv[] ) {
double time1, time2;
time1 = MPI_Wtime();
int rank, processors;
int j; // number of iterations
int k; // number of iterations to perform before creating a checkpoint
int l; // number of random samples per grid point
int checkpoint_resume = 0; // 1 = resume from last checkpoint
int c; // used to hold a character
int i=0, row = 0, col = 0, pln = 0; // array iterators
char ***local_array;
char **local_array_2nd;
char *local_array_pointer;
char ***local_array_copy;
char **local_array_copy_2nd;
char *local_array_copy_pointer;
char ***temp, *temp_pointer;
int file_open_error;
int command_line_incomplete = 0;
int grid_size[3] = {0,0,0};
int proc_size[3] = {0,0,0};
int local_size[3] = {0,0,0};
int remainder_size[3] = {0,0,0};
int coords[3] = {0,0,0};
int start_indices[3] = {0,0,0};
int periods[3] = {0,0,0};
int mem_size[3] = {0,0,0};
MPI_Status status;
MPI_Datatype filetype, memtype;
MPI_File fh;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &processors);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// Interpret the command line arguments --------------------------------
if (rank == 0) {
if (argc < 6 || argc > 8) {
fputs("usage: x y z j k l r\n", stderr);
fputs("where: x,y,z = x, y and z dimensions\n", stderr);
fputs(" j = how many times the game of life is played\n", stderr);
fputs(" k = checkpoint every k iterations\n", stderr);
fputs(" l = number of random samples per grid point\n", stderr);
fputs(" r = resume from the last checkpoint\n", stderr);
fputs(INITIAL, stderr);
fputs(" must be present.\n", stderr);
fputs(CHECKPOINT, stderr);
fputs(" must be present if resuming from the last checkpoint.\n", stderr);
exit(EXIT_FAILURE);
}
}
j = (int) strtol(argv[4], NULL, 10);
k = (int) strtol(argv[5], NULL, 10);
l = (int) strtol(argv[6], NULL, 10);
if ( argc == 7 )
if ( argv[6][0] == 'r' )
checkpoint_resume = 1;
if (rank == 0)
printf("%d iterations \ncheckpoint every %d iterations \n%d samples per grid point \ncheckpoint resume = %d\n", j,k,l,checkpoint_resume);
grid_size[0] = (int) strtol(argv[1], NULL, 10);
grid_size[1] = (int) strtol(argv[2], NULL, 10);
grid_size[2] = (int) strtol(argv[3], NULL, 10);
if (rank==0) printf("grid_size: %d, %d, %d\n", grid_size[0], grid_size[1], grid_size[2]);
MPI_Dims_create(processors, 3, proc_size);
if (rank==0) printf("proc_size: %d, %d, %d\n", proc_size[0], proc_size[1], proc_size[2]);
local_size[0] = grid_size[0] / proc_size[0];
local_size[1] = grid_size[1] / proc_size[1];
local_size[2] = grid_size[2] / proc_size[2];
if (rank==0) printf("local_size: %d, %d, %d\n", local_size[0], local_size[1], local_size[2]);
remainder_size[0] = grid_size[0] % proc_size[0];
remainder_size[1] = grid_size[1] % proc_size[1];
remainder_size[2] = grid_size[2] % proc_size[2];
if (rank==0) printf("remainder_size: %d, %d, %d\n", remainder_size[0], remainder_size[1], remainder_size[2]);
if (remainder_size[0] != 0 || remainder_size[1] != 0 || remainder_size[2] != 0) {
fputs("remainder size != 0, check your dimensions", stderr);
MPI_Finalize();
exit(EXIT_FAILURE);
}
MPI_Comm comm;
MPI_Cart_create(MPI_COMM_WORLD, 3, proc_size, periods, 0, &comm);
MPI_Comm_rank(comm, &rank);
MPI_Cart_coords(comm, rank, 3, coords);
start_indices[0] = coords[0] * local_size[0];
start_indices[1] = coords[1] * local_size[1];
start_indices[2] = coords[2] * local_size[2];
/* printf("A coords R%d: (%d, %d, %d) (%d, %d, %d)\n", rank, coords[0], coords[1], coords[2], start_indices[0], start_indices[1], start_indices[2]);*/
fflush(stdout);
// create the file type ---------------------------------------------------
MPI_Type_create_subarray(3, grid_size, local_size, start_indices, MPI_ORDER_C, MPI_CHAR, &filetype);
MPI_Type_commit(&filetype);
// create a local memory type with ghost rows -----------------------------
mem_size[0] = local_size[0] + 2;
mem_size[1] = local_size[1] + 2;
mem_size[2] = local_size[2] + 2;
start_indices[0] = start_indices[1] = start_indices[2] = 1;
MPI_Type_create_subarray(3, mem_size, local_size, start_indices, MPI_ORDER_C, MPI_CHAR, &memtype);
MPI_Type_commit(&memtype);
// find my neighbors ------------------------------------------------------
int nxminus, nxplus, nyminus, nyplus, nzminus, nzplus, tag = 333, *neighbors;
// Neighbors Array: row- col- col+ row+ plane- plane+
neighbors = (int *) malloc(6 * sizeof(int));
for(i=0; i<6; i++)
neighbors[i] = rank;
MPI_Cart_shift(comm, 0, 1, &nxminus, &nxplus);
MPI_Cart_shift(comm, 1, 1, &nyminus, &nyplus);
MPI_Cart_shift(comm, 2, 1, &nzminus, &nzplus);
// printf(" %d sending south to %d receiving from %d \n",rank,nxplus,nxminus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nxplus, tag,
&(neighbors[0]), 1, MPI_INT, nxminus, tag, comm, &status);
// printf(" %d sending North to %d receiving from %d \n",rank,nxminus,nxplus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nxminus, tag,
&(neighbors[3]), 1, MPI_INT, nxplus, tag, comm, &status);
// printf(" %d sending East to %d receiving from %d \n",rank,nyplus,nyminus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nyplus, tag,
&neighbors[1], 1, MPI_INT, nyminus, tag, comm, &status);
// printf(" %d sending West to %d receiving from %d \n",rank,nyminus,nyplus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nyminus, tag,
&neighbors[2], 1, MPI_INT, nyplus, tag, comm, &status);
// printf(" %d sending backwards to %d receiving from %d \n",rank,nzplus,nzminus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nzplus, tag,
&(neighbors[4]), 1, MPI_INT, nzminus, tag, comm, &status);
// printf(" %d sending forward to %d receiving from %d \n",rank,nzminus,nzplus);
// fflush(stdout);
MPI_Sendrecv(&rank, 1, MPI_INT, nzminus, tag,
&(neighbors[5]), 1, MPI_INT, nzplus, tag, comm, &status);
/* printf("neighboors R%d : (row-) %d (col-) %d (col+) %d (row+) %d (plane-) %d (plane+) %d\n",rank,neighbors[0],neighbors[1],neighbors[2],neighbors[3],neighbors[4],neighbors[5]);*/
fflush(stdout);
//init_sprng(1,time(0),SPRNG_DEFAULT);
srand((unsigned int)time(NULL));
// Open the initial condition (checkpoint or not) ----------------------
if ( checkpoint_resume ) {
file_open_error =
MPI_File_open(MPI_COMM_WORLD, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh,0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
}
else {
file_open_error =
MPI_File_open(MPI_COMM_WORLD, INITIAL, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh,0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
}
if (file_open_error != MPI_SUCCESS) {
if (checkpoint_resume)
fputs(CHECKPOINT, stderr);
else
fputs(INITIAL, stderr);
fputs(" could not be opened.\n", stderr);
exit(EXIT_FAILURE);
}
// Allocate and Populate the local array ----------------------------------
local_array_copy_pointer = (char *) malloc(mem_size[0] * mem_size[1] * mem_size[2] * sizeof(char));
local_array_copy_2nd = (char **) malloc(mem_size[0] * mem_size[1] * sizeof(char*));
local_array_copy = (char ***) malloc(mem_size[0] * sizeof(char*));
for(i = 0; i < mem_size[0] * mem_size[1]; i++)
local_array_copy_2nd[i] = &local_array_copy_pointer[i * mem_size[2]];
for(i = 0; i < mem_size[0]; i++)
local_array_copy[i] = &local_array_copy_2nd[i * mem_size[1]];
local_array_pointer = (char *) malloc(mem_size[0] * mem_size[1] * mem_size[2] * sizeof(char));
local_array_2nd = (char **) malloc(mem_size[0] * mem_size[1] * sizeof(char*));
local_array = (char ***) malloc(mem_size[0] * sizeof(char*));
for(i = 0; i < mem_size[0] * mem_size[1]; i++)
local_array_2nd[i] = &local_array_pointer[i * mem_size[2]];
for(i = 0; i < mem_size[0]; i++)
local_array[i] = &local_array_2nd[i * mem_size[1]];
// if (rank==0) printf("Malloc complete\n");
for(row=0; row<mem_size[0]; row++) {
for(col=0; col<mem_size[1]; col++) {
for(pln=0; pln<mem_size[2]; pln++) {
local_array[row][col][pln] = local_array_copy[row][col][pln] = '0';
}
}
}
// if (rank==0) printf("Setup complete\n");
MPI_File_read_all(fh, local_array_pointer, 1, memtype, &status);
if (rank==0) printf("File Read\n");
// if (rank==0) {
// for(row=0; row<mem_size[0]; row++) {
// for(col=0; col<mem_size[1]; col++) {
// for(pln=0; pln<mem_size[2]; pln++) {
// printf("%c", local_array[row][col][pln]);
// }
// printf("\n");
// }
// printf("-----------------------\n");
// }
// }
MPI_File_close(&fh);
// Construct the plane data types
MPI_Datatype yzplane;
MPI_Type_vector(local_size[1], local_size[2], local_size[2]+2, MPI_CHAR, &yzplane);
MPI_Type_commit(&yzplane);
MPI_Datatype xzplane;
MPI_Type_vector(local_size[0], local_size[2], ((local_size[2]+2)*local_size[1])+((local_size[2]+2)*2), MPI_CHAR, &xzplane);
MPI_Type_commit(&xzplane);
// this type will also copy the corner x columns, can't skip blocks intermittently
// since we aren't worrying about the corner data, it's ok
MPI_Datatype xyplane;
MPI_Type_vector((local_size[0]*local_size[1])+((local_size[0]*2)-2), 1, local_size[2]+2, MPI_CHAR, &xyplane);
MPI_Type_commit(&xyplane);
MPI_Barrier(comm);
// start the iteration loop
int iterations;
int kCounter = k;
for (iterations = 0; iterations < j; iterations++) {
// send updated planes
// Neighbors Array:
// 0 1 2 3 4 5
// row- col- col+ row+ plane- plane+
// Note: corners are not handled
// send top yzplane
if (rank != neighbors[0]) MPI_Send(&local_array[1][1][1], 1, yzplane, neighbors[0], 0, comm);
// recv bottom yzplane
if (rank != neighbors[3]) MPI_Recv(&local_array[local_size[0]+1][1][1], 1, yzplane, neighbors[3], 0, comm, &status);
// send bottom yzplane
if (rank != neighbors[3]) MPI_Send(&local_array[local_size[0]][1][1], 1, yzplane, neighbors[3], 0, comm);
// recv top yzplane
if (rank != neighbors[0]) MPI_Recv(&local_array[0][1][1], 1, yzplane, neighbors[0], 0, comm, &status);
// send left xzplane
if (rank != neighbors[1]) MPI_Send(&local_array[1][1][1], 1, xzplane, neighbors[1], 0, comm);
// recv right xzplane
if (rank != neighbors[2]) MPI_Recv(&local_array[1][local_size[1]+1][1], 1, xzplane, neighbors[2], 0, comm, &status);
// send right xzplane
if (rank != neighbors[2]) MPI_Send(&local_array[1][local_size[1]][1], 1, xzplane, neighbors[2], 0, comm);
// recv left xzplane
if (rank != neighbors[1]) MPI_Recv(&local_array[1][0][1], 1, xzplane, neighbors[1], 0, comm, &status);
// send front xyplane
if (rank != neighbors[4]) MPI_Send(&local_array[1][1][1], 1, xyplane, neighbors[4], 0, comm);
// recv back xyplane
if (rank != neighbors[5]) MPI_Recv(&local_array[1][1][local_size[2]+1], 1, xyplane, neighbors[5], 0, comm, &status);
// send back xyplane
if (rank != neighbors[5]) MPI_Send(&local_array[1][1][local_size[2]], 1, xyplane, neighbors[5], 0, comm);
// recv front xyplane
if (rank != neighbors[4]) MPI_Recv(&local_array[1][1][0], 1, xyplane, neighbors[4], 0, comm, &status);
// if (rank==0) {
// for(row=0; row<mem_size[0]; row++) {
// for(col=0; col<mem_size[1]; col++) {
// for(pln=0; pln<mem_size[2]; pln++) {
// printf("%c", local_array[row][col][pln]);
// }
// printf("\n");
// }
// printf("-----------------------\n");
// }
// }
// run the game of life
// gameOfLife(local_array, local_array_copy, local_size[0], local_size[1], l, rank);
// swap the arrays
// temp1 = local_array;
// local_array = local_array_copy;
// local_array_copy = temp1;
//
// temp2 = local_array_pointer;
// local_array_pointer = local_array_copy_pointer;
// local_array_copy_pointer = temp2;
// check to see if this iteration needs a checkpoint
kCounter--;
if (kCounter == 0) {
kCounter = k;
// checkpoint code
MPI_File_open(MPI_COMM_WORLD, CHECKPOINT, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
MPI_File_write_all(fh, local_array_pointer, 1, memtype, &status);
MPI_File_close(&fh);
if (rank == 0)
printf("Checkpoint made: Iteration %d\n", iterations+1);
} // end if kCounter == 0
} // end iteration loop
iterations--;
// all done! repeat the checkpoint process
MPI_File_open(MPI_COMM_WORLD, FINAL_RESULTS, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_CHAR, filetype, "native", MPI_INFO_NULL);
MPI_File_write_all(fh, local_array_pointer, 1, memtype, &status);
MPI_File_close(&fh);
if (rank == 0)
printf("Final Results made: Iteration %d\n", iterations+1);
time2 = MPI_Wtime();
if (rank == 0)
printf("Elapsed Seconds: %f\n", time2-time1);fflush(stdout);
MPI_Finalize();
return EXIT_SUCCESS;
}
static int
test_mpio_1wMr(char *filename, int special_request)
{
char hostname[128];
int mpi_size, mpi_rank;
MPI_File fh;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
unsigned char writedata[DIMSIZE], readdata[DIMSIZE];
unsigned char expect_val;
int i, irank;
int nerrs = 0; /* number of errors */
int atomicity;
MPI_Offset mpi_off;
MPI_Status mpi_stat;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
if (MAINPROCESS && VERBOSE_MED){
printf("Testing one process writes, all processes read.\n");
printf("Using %d processes accessing file %s\n", mpi_size, filename);
printf(" (Filename can be specified via program argument)\n");
}
/* show the hostname so that we can tell where the processes are running */
if (VERBOSE_DEF){
if (gethostname(hostname, 128) < 0){
PRINTID;
printf("gethostname failed\n");
return 1;
}
PRINTID;
printf("hostname=%s\n", hostname);
}
/* Delete any old file in order to start anew. */
/* Must delete because MPI_File_open does not have a Truncate mode. */
/* Don't care if it has error. */
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD); /* prevent racing condition */
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE ,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
if (special_request & USEATOM){
/* ==================================================
* Set atomcity to true (1). A POSIX compliant filesystem
* should not need this.
* ==================================================*/
if ((mpi_err = MPI_File_get_atomicity(fh, &atomicity)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_get_atomicity failed (%s)\n", mpi_err_str);
}
if (VERBOSE_HI)
printf("Initial atomicity = %d\n", atomicity);
if ((mpi_err = MPI_File_set_atomicity(fh, 1)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_set_atomicity failed (%s)\n", mpi_err_str);
}
if ((mpi_err = MPI_File_get_atomicity(fh, &atomicity)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_get_atomicity failed (%s)\n", mpi_err_str);
}
if (VERBOSE_HI)
printf("After set_atomicity atomicity = %d\n", atomicity);
}
/* This barrier is not necessary but do it anyway. */
MPI_Barrier(MPI_COMM_WORLD);
if (VERBOSE_HI){
PRINTID;
printf("between MPI_Barrier and MPI_File_write_at\n");
}
/* ==================================================
* Each process calculates what to write but
* only process irank(0) writes.
* ==================================================*/
irank=0;
for (i=0; i < DIMSIZE; i++)
writedata[i] = irank*DIMSIZE + i;
mpi_off = irank*DIMSIZE;
/* Only one process writes */
if (mpi_rank==irank){
if (VERBOSE_HI){
PRINTID; printf("wrote %d bytes at %ld\n", DIMSIZE, (long)mpi_off);
}
if ((mpi_err = MPI_File_write_at(fh, mpi_off, writedata, DIMSIZE,
MPI_BYTE, &mpi_stat))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, DIMSIZE, mpi_err_str);
return 1;
};
};
/* Bcast the return code and */
/* make sure all writing are done before reading. */
MPI_Bcast(&mpi_err, 1, MPI_INT, irank, MPI_COMM_WORLD);
if (VERBOSE_HI){
PRINTID;
printf("MPI_Bcast: mpi_err = %d\n", mpi_err);
}
if (special_request & USEFSYNC){
/* ==================================================
* Do a file sync. A POSIX compliant filesystem
* should not need this.
* ==================================================*/
if (VERBOSE_HI)
printf("Apply MPI_File_sync\n");
/* call file_sync to force the write out */
if ((mpi_err = MPI_File_sync(fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_sync failed (%s)\n", mpi_err_str);
}
MPI_Barrier(MPI_COMM_WORLD);
/* call file_sync to force the write out */
if ((mpi_err = MPI_File_sync(fh)) != MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_sync failed (%s)\n", mpi_err_str);
}
}
/* This barrier is not necessary because the Bcase or File_sync above */
/* should take care of it. Do it anyway. */
MPI_Barrier(MPI_COMM_WORLD);
if (VERBOSE_HI){
PRINTID;
printf("after MPI_Barrier\n");
}
/* ==================================================
* Each process reads what process 0 wrote and verify.
* ==================================================*/
irank=0;
mpi_off = irank*DIMSIZE;
if ((mpi_err = MPI_File_read_at(fh, mpi_off, readdata, DIMSIZE, MPI_BYTE,
&mpi_stat))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
PRINTID;
printf("MPI_File_read_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, DIMSIZE, mpi_err_str);
return 1;
};
for (i=0; i < DIMSIZE; i++){
expect_val = irank*DIMSIZE + i;
if (readdata[i] != expect_val){
PRINTID;
printf("read data[%d:%d] got %02x, expect %02x\n", irank, i,
readdata[i], expect_val);
nerrs++;
}
}
MPI_File_close(&fh);
if (VERBOSE_HI){
PRINTID;
printf("%d data errors detected\n", nerrs);
}
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
return nerrs;
}
int main(int argc, char **argv)
{
MPI_File fp;
LemonWriter *w;
LemonReader *r;
LemonRecordHeader *h;
double *data;
double tick, tock;
double *timesRead;
double *timesWrite;
double stdRead = 0.0;
double stdWrite = 0.0;
int mpisize;
int rank;
char const *type;
int ldsize;
unsigned long long int fsize;
int *hashMatch, *hashMatchAll;
double const rscale = 1.0 / RAND_MAX;
int ME_flag=1, MB_flag=1, status=0;
int latDist[] = {0, 0, 0, 0};
int periods[] = {1, 1, 1, 1};
int locSizes[4];
int latSizes[4];
int localVol = 1;
int latVol = localVol;
MPI_Comm cartesian;
int i, j;
md5_state_t state;
md5_byte_t before[16];
md5_byte_t after[16];
int L;
int iters;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpisize);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (argc != 3)
{
usage(rank, argv);
MPI_Finalize();
return 1;
}
L = atoi(argv[1]);
if (L <= 0)
usage(rank, argv);
iters = atoi(argv[2]);
if (iters <= 0)
usage(rank, argv);
timesWrite = (double*)calloc(iters, sizeof(double));
if (timesWrite == (double*)NULL)
{
fprintf(stderr, "ERROR: Could not allocate memory.\n");
return 1;
}
timesRead = (double*)calloc(iters, sizeof(double));
if (timesRead == (double*)NULL)
{
fprintf(stderr, "ERROR: Could not allocate memory.\n");
return 1;
}
hashMatch = (int*)calloc(iters, sizeof(int));
if (hashMatch == (int*)NULL)
{
fprintf(stderr, "ERROR: Could not allocate memory.\n");
return 1;
}
hashMatchAll = (int*)calloc(iters, sizeof(int));
if (hashMatchAll == (int*)NULL)
{
fprintf(stderr, "ERROR: Could not allocate memory.\n");
return 1;
}
/* Construct a Cartesian topology, adjust lattice sizes where needed */
MPI_Dims_create(mpisize, 4, latDist);
for (i = 0; i < 4; ++i)
{
int div = (i == 3 ? (2 * L) : L) / latDist[i];
locSizes[i] = div ? div : 1;
localVol *= locSizes[i];
latSizes[i] = locSizes[i] * latDist[i];
}
latVol = mpisize * localVol;
ldsize = localVol * 72 * sizeof(double);
fsize = (unsigned long long int)latVol * 72 * sizeof(double);
MPI_Cart_create(MPI_COMM_WORLD, 4, latDist, periods, 1, &cartesian);
MPI_Comm_rank(cartesian, &rank);
if (rank == 0)
{
fprintf(stdout, "Benchmark on a block of data %s in size,\n", humanForm(fsize));
fprintf(stdout, "representing a %u x %u x %u x %u lattice", latSizes[0], latSizes[1], latSizes[2], latSizes[3]);
if (mpisize == 1)
fprintf(stdout, ".\n\n");
else
{
fprintf(stdout, ",\ndistributed over %u MPI processes\n", mpisize);
fprintf(stdout, "for a local %u x %u x %u x %u lattice.\n\n", locSizes[0], locSizes[1], locSizes[2], locSizes[3]);
}
}
/* Allocate a block of memory for dummy data to write */
data = (double*)malloc(ldsize);
if (data == (double*)NULL)
{
fprintf(stderr, "ERROR: Could not allocate memory.\n");
return 1;
}
srand(time(NULL) + rank);
/* Start of test */
for (i = 0; i < iters; ++i)
{
if (rank == 0)
fprintf(stdout, "Measurement %d of %d.\n", i + 1, iters);
/* Create a block of dummy data to write out
Fill with some random numbers to make sure we don't get coincidental matches here */
for (j = 0; j < (localVol * 72); ++j)
data[j] = rscale * (double)rand();
/* Calculate a hash of the data, to check integrity against */
md5_init(&state);
md5_append(&state, (md5_byte_t const *)data, ldsize);
md5_finish(&state, before);
/* Note that the following is the only (?) way to truncate the file with MPI */
MPI_File_open(cartesian, "benchmark.test", MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fp);
MPI_File_set_size(fp, 0);
w = lemonCreateWriter(&fp, cartesian);
h = lemonCreateHeader(MB_flag, ME_flag, "benchmark", latVol);
status = lemonWriteRecordHeader(h, w);
lemonDestroyHeader(h);
MPI_Barrier(cartesian);
tick = MPI_Wtime();
lemonWriteLatticeParallel(w, data, 72 * sizeof(double), latSizes);
tock = MPI_Wtime();
MPI_Barrier(cartesian);
timesWrite[i] = tock - tick;
if (rank == 0)
fprintf(stdout, "Time spent writing was %4.2g s.\n", timesWrite[i]);
lemonWriterCloseRecord(w);
lemonDestroyWriter(w);
MPI_File_close(&fp);
/* Clear data to avoid an utterly failed read giving md5 hash matches from the old data */
memset(data, 0, ldsize);
/* Start of reading test */
MPI_File_open(cartesian, "benchmark.test", MPI_MODE_RDONLY | MPI_MODE_DELETE_ON_CLOSE, MPI_INFO_NULL, &fp);
r = lemonCreateReader(&fp, cartesian);
if (lemonReaderNextRecord(r))
fprintf(stderr, "Node %d reports: next record failed.\n", rank);
type = lemonReaderType(r);
if (strncmp(type, "benchmark", 13))
fprintf(stderr, "Node %d reports: wrong type read.\n", rank);
MPI_Barrier(cartesian);
tick = MPI_Wtime();
lemonReadLatticeParallel(r, data, 72 * sizeof(double), latSizes);
tock = MPI_Wtime();
timesRead[i] = tock - tick;
MPI_Barrier(cartesian);
if (rank == 0)
fprintf(stdout, "Time spent reading was %4.2g s.\n", timesRead[i]);
lemonDestroyReader(r);
MPI_File_close(&fp);
md5_init(&state);
md5_append(&state, (md5_byte_t const *)data, ldsize);
md5_finish(&state, after);
hashMatch[i] = strncmp((char const *)before, (char const *)after, 16) != 0 ? 1 : 0;
MPI_Reduce(hashMatch + i, hashMatchAll + i, 1, MPI_INT, MPI_SUM, 0, cartesian);
if (rank == 0)
{
if (hashMatchAll[i] == 0)
fprintf(stdout, "All nodes report that MD5 hash matches.\n\n");
else
fprintf(stdout, "WARNING: MD5 hash failure detected!\n\n");
}
}
/* Aggregate the data */
hashMatch[0] = 0;
stdWrite = timesWrite[0] * timesWrite[0];
stdRead = timesRead[0] * timesRead[0];
for (i = 1; i < iters; ++i)
{
hashMatchAll[0] += hashMatchAll[i];
timesWrite[0] += timesWrite[i];
stdWrite += timesWrite[i] * timesWrite[i];
timesRead[0] += timesRead[i];
stdRead += timesRead[i] * timesRead[i];
}
stdWrite /= iters;
stdRead /= iters;
timesWrite[0] /= iters;
timesRead[0] /= iters;
stdWrite -= timesWrite[0] * timesWrite[0];
stdRead -= timesRead[0] * timesRead[0];
if (rank == 0)
{
fprintf(stdout, "Average time spent writing was %4.2e s, ", timesWrite[0]);
fprintf(stdout, "with a standard deviation of %4.2e s.\n", sqrt(stdWrite));
fprintf(stdout, "Average time spent reading was %4.2e s, ", timesRead[0]);
fprintf(stdout, "with a standard deviation of %4.2e s.\n\n", sqrt(stdRead));
stdWrite *= (double)fsize / (timesWrite[0] * timesWrite[0]);
stdRead *= (double)fsize / (timesRead[0] * timesRead[0]);
fprintf(stdout, "Average writing speed was %s/s\n", humanForm((unsigned long long int)(fsize / timesWrite[0])));
fprintf(stdout, "Average reading speed was %s/s\n", humanForm((unsigned long long int)(fsize / timesRead[0])));
if (hashMatchAll[0] == 0)
fprintf(stdout, "All data hashed correctly.\n");
else
fprintf(stdout, "WARNING: %d hash mismatches detected!.\n", hashMatchAll[0]);
}
MPI_Finalize();
free(data);
free(timesWrite);
free(timesRead);
free(hashMatch);
free(hashMatchAll);
return(0);
}
static int
test_mpio_special_collective(char *filename)
{
int mpi_size, mpi_rank;
MPI_File fh;
MPI_Datatype etype,buftype,filetype;
char mpi_err_str[MPI_MAX_ERROR_STRING];
int mpi_err_strlen;
int mpi_err;
char writedata[2];
char *buf;
int i;
int count,bufcount;
int blocklens[2];
MPI_Aint offsets[2];
MPI_Offset mpi_off;
MPI_Status mpi_stat;
int retcode;
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
retcode = 0;
/* create MPI data type */
etype = MPI_BYTE;
if(mpi_rank == 0 || mpi_rank == 1) {
count = DIMSIZE;
bufcount = 1;
}
else {
count = 0;
bufcount = 0;
}
blocklens[0] = count;
offsets[0] = mpi_rank*count;
blocklens[1] = count;
offsets[1] = (mpi_size+mpi_rank)*count;
if(count !=0) {
if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&filetype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&filetype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err= MPI_Type_hindexed(2,blocklens,offsets,etype,&buftype))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_contiguous failed (%s)\n", mpi_err_str);
return 1;
}
if((mpi_err=MPI_Type_commit(&buftype))!=MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_Type_commit failed (%s)\n", mpi_err_str);
return 1;
}
}
else {
filetype = MPI_BYTE;
buftype = MPI_BYTE;
}
/* Open a file */
if ((mpi_err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_RDWR | MPI_MODE_CREATE ,
MPI_INFO_NULL, &fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_open failed (%s)\n", mpi_err_str);
return 1;
}
/* each process writes some data */
for (i=0; i < 2*DIMSIZE; i++)
writedata[i] = mpi_rank*DIMSIZE + i;
mpi_off = 0;
if((mpi_err = MPI_File_set_view(fh, mpi_off, MPI_BYTE, filetype, "native", MPI_INFO_NULL))
!= MPI_SUCCESS) {
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_set_view failed (%s)\n", mpi_err_str);
return 1;
}
buf = writedata;
if ((mpi_err = MPI_File_write_at_all(fh, mpi_off, buf, bufcount, buftype,
&mpi_stat))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_write_at offset(%ld), bytes (%d), failed (%s)\n",
(long) mpi_off, bufcount, mpi_err_str);
return 1;
};
if ((mpi_err = MPI_File_close(&fh))
!= MPI_SUCCESS){
MPI_Error_string(mpi_err, mpi_err_str, &mpi_err_strlen);
printf("MPI_File_close failed. \n");
return 1;
};
mpi_err = MPI_Barrier(MPI_COMM_WORLD);
#ifdef H5_MPI_SPECIAL_COLLECTIVE_IO_WORKS
if(retcode != 0) {
if(mpi_rank == 0) {
printf("special collective IO is NOT working at this platform\n");
printf("Go back to hdf5/config and find the corresponding\n");
printf("configure-specific file (for example, powerpc-ibm-aix5.x) and add\n");
printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n");
printf(" at the end of the file.\n");
printf(" Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
#else
if(retcode == 0) {
if(mpi_rank == 0) {
printf(" This is NOT an error, What it really says is\n");
printf("special collective IO is WORKING at this platform\n");
printf(" Go back to hdf5/config and find the corresponding \n");
printf(" configure-specific file (for example, powerpc-ibm-aix5.x) and delete the line\n");
printf("hdf5_cv_mpi_special_collective_io_works=${hdf5_cv_mpi_special_collective_io_works='no'}\n");
printf(" at the end of the file.\n");
printf("Please report to [email protected] about this problem.\n");
}
retcode = 1;
}
#endif
return retcode;
}
/* *************************** *
* Main computational kernel *
* *************************** */
int correlationKernel(int rank,
int size,
double* dataMatrixX,
double* dataMatrixY,
int columns,
int rows,
char *out_filename,
int distance_flag) {
int local_check = 0, global_check = 0;
int i = 0, j, taskNo;
int err, count = 0;
unsigned long long fair_chunk = 0, coeff_count = 0;
unsigned int init_and_cleanup_loop_iter=0;
unsigned long long cor_cur_size = 0;
double start_time, end_time;
// Variables needed by the Indexed Datatype
MPI_Datatype coeff_index_dt;
MPI_File fh;
int *blocklens, *indices;
MPI_Status stat;
MPI_Comm comm = MPI_COMM_WORLD;
// Master processor keeps track of tasks
if (rank == 0) {
// Make sure everything will work fine even if there are
// less genes than available workers (there are size-1 workers
// master does not count)
if ( (size-1) > rows )
init_and_cleanup_loop_iter = rows+1;
else
init_and_cleanup_loop_iter = size;
// Start timer
start_time = MPI_Wtime();
// Send out initial tasks (remember you have size-1 workers, master does not count)
for (i=1; i<init_and_cleanup_loop_iter; i++) {
taskNo = i-1;
err = MPI_Send(&taskNo, 1, MPI_INT, i, 0, comm);
}
// Terminate any processes that were not working due to the fact
// that the number of rows where less than the actual available workers
for(i=init_and_cleanup_loop_iter; i < size; i++) {
PROF(rank, "\nPROF_idle : Worker %d terminated due to insufficient work load", i);
err = -1;
err = MPI_Send(&err, 1, MPI_INT, i, 0, comm);
}
// Wait for workers to finish their work assignment and ask for more
for (i=init_and_cleanup_loop_iter-1; i<rows; i++) {
err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat);
// Check taskNo to make sure everything is ok. Negative means there is problem
// thus terminate gracefully all remaining working workers
if ( taskNo < 0 ) {
// Reduce by one because one worker is already terminated
init_and_cleanup_loop_iter--;
// Break and cleanup
break;
}
// The sending processor is ready to work:
// It's ID is in stat.MPI_SOURCE
// Send it the current task (i)
err = MPI_Send(&i, 1, MPI_INT, stat.MPI_SOURCE, 0, comm);
}
// Clean up processors
for (i=1; i<init_and_cleanup_loop_iter; i++) {
// All tasks complete - shutdown workers
err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat);
// If process failed then it will not be waiting to receive anything
// We have to ignore the send because it will deadlock
if ( taskNo < 0 )
continue;
err = -1;
err = MPI_Send(&err, 1, MPI_INT, stat.MPI_SOURCE, 0, comm);
}
// Master is *always* OK
local_check = 0;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Check failed, abort
if ( global_check != 0 ) {
return -1;
}
// Stop timer
end_time = MPI_Wtime();
PROF(rank, "\nPROF_comp (workers=%d) : Time taken by correlation coefficients computations : %g\n", size-1, end_time - start_time);
// Start timer
start_time = MPI_Wtime();
// Master process must call MPI_File_set_view as well, it's a collective call
// Open the file handler
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
// Create the file view
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
// Write data to disk
MPI_File_write_all(fh, &cor[0], 0, MPI_DOUBLE, &stat);
// Stop timer
end_time = MPI_Wtime();
PROF(rank, "\nPROF_write (workers=%d) : Time taken for global write-file : %g\n", size-1, end_time - start_time);
} else {
// Compute how many workers will share the work load
// Two scenarios exist:
// (1) more OR equal number of workers and rows exist
// (2) more rows than workers
if ( (size-1) > rows ) {
// For this scenario each worker will get exaclty one work asssignment.
// There is not going to be any other work so it only compute "rows" number
// of coefficients
fair_chunk = rows;
cor_cur_size = fair_chunk;
} else {
// For this scenario we are going to allocate space equal to a fair
// distribution of work assignments *plus* an extra amount of space to
// cover any load imbalancing. This amount is expressed as a percentage
// of the fair work distribution (see on top, 20% for now)
// Plus 1 to round it up or just add some extra space, both are fine
fair_chunk = (rows / (size-1)) + 1;
DEBUG("fair_chunk %d \n", fair_chunk);
// We can use "j" as temporary variable.
// Plus 1 to avoid getting 0 from the multiplication.
j = (fair_chunk * MEM_PERC) + 1;
cor_cur_size = (fair_chunk + j) * rows;
DEBUG("cor_cur_size %lld \n", cor_cur_size);
}
// Allocate memory
DEBUG("cor_cur_size %lld \n", cor_cur_size);
long long double_size = sizeof(double);
DEBUG("malloc size %lld \n", (double_size * cor_cur_size));
cor = (double *)malloc(double_size * cor_cur_size);
blocklens = (int *)malloc(sizeof(int) * rows);
indices = (int *)malloc(sizeof(int) * rows);
mean_value_vectorX = (double *)malloc(sizeof(double) * rows);
Sxx_vector = (double *)malloc(sizeof(double) * rows);
mean_value_vectorY = (double *)malloc(sizeof(double) * rows);
Syy_vector = (double *)malloc(sizeof(double) * rows);
// Check that all memory is successfully allocated
if ( ( cor == NULL ) || ( blocklens == NULL ) || ( indices == NULL ) ||
( mean_value_vectorX == NULL ) || ( Sxx_vector == NULL ) ||
( mean_value_vectorY == NULL ) || ( Syy_vector == NULL ) ) {
ERR("**ERROR** : Memory allocation failed on worker process %d. Aborting.\n", rank);
// Free allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
// We have to receive a work assignment first and then terminate
// otherwise the master will deadlock trying to give work to this worker
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
return -1;
}
// Compute necessary parameters for Pearson method
// (this will transform the values of the input array to more meaningful data
// and save us from a lot of redundant computations)
compute_parameters(dataMatrixX, dataMatrixY, rows, columns);
// Main loop for workers. They get work from master, compute coefficients,
// save them to their *local* vector and ask for more work
for(;;) {
// Get work
err = 0;
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
// If received task is -1, function is terminated
if ( taskNo == -1 ) break;
// Check if there is enough memory to store the new coefficients, if not reallocate
// the current memory and expand it by MEM_PERC of the approximated size
if ( cor_cur_size < (coeff_count + rows) ) {
PROF(0, "\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank);
DEBUG("\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank);
// Use j as temporary again. Add two (or any other value) to avoid 0.
// (two is just a random value, you can put any value really...)
j = (fair_chunk * MEM_PERC) + 2;
cor_cur_size += (j * rows);
// Reallocate and check
cor = (double *)realloc(cor, sizeof(double) * cor_cur_size);
if ( cor == NULL ) {
ERR("**ERROR** : Memory re-allocation failed on worker process %d. Aborting.\n", rank);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
return -1;
}
}
// Compute the correlation coefficients
if(dataMatrixY != NULL) {
for (j=0; j < rows; j++) {
cor[coeff_count] = pearson_XY(dataMatrixX, dataMatrixY, j, taskNo, columns);
coeff_count++;
}
} else {
for (j=0; j < rows; j++) {
// Set main diagonal to 1
if ( j == taskNo ) {
cor[coeff_count] = 1.0;
coeff_count++;
continue;
}
cor[coeff_count] = pearson(dataMatrixX, taskNo, j, columns);
coeff_count++;
}
}
// The value of blocklens[] represents the number of coefficients on each
// row of the corellation array
blocklens[count] = rows;
// The value of indices[] represents the offset of each row in the data file
indices[count] = (taskNo * rows);
count++;
// Give the master the taskID
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
}
// There are two possibilities
// (a) everything went well and all workers finished ok
// (b) some processes finished ok but one or more of the remaining working workers failed
// To make sure all is well an all-reduce will be performed to sync all workers and guarantee success
// before moving on to write the output file
// This worker is OK
local_check = 0;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Check failed
if ( global_check != 0 ) {
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
return -1;
}
PROF(0, "\nPROF_stats (thread %3d) : Fair chunk of work : %d \t\t Allocated : %d \t\t Computed : %d\n",
rank, fair_chunk, cor_cur_size, coeff_count);
// If the distance_flag is set, then transform all correlation coefficients to distances
if ( distance_flag == 1 ) {
for(j=0; j < coeff_count; j++) {
cor[j] = 1 - cor[j];
}
}
// Create and commit the Indexed datatype *ONLY* if there are data available
if ( coeff_count != 0 ) {
MPI_Type_indexed(count, blocklens, indices, MPI_DOUBLE, &coeff_index_dt);
MPI_Type_commit(&coeff_index_dt);
}
// Open the file handler
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
// Create the file view
if ( coeff_count != 0 ) {
MPI_File_set_view(fh, 0, MPI_DOUBLE, coeff_index_dt, "native", MPI_INFO_NULL);
} else {
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
}
// Write data to disk
// TODO coeff_count cannot be greater than max int (for use in the MPI_File_write_all call).
// A better fix should be possible, for now throw error.
DEBUG("\ncoeff_count is %lld\n", coeff_count);
DEBUG("\INT_MAX is %d\n", INT_MAX);
if(coeff_count>INT_MAX)
{
ERR("**ERROR** : Could not run as the chunks of data are too large. Try running again with more MPI processes.\n");
// Free allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
// We have to receive a work assignment first and then terminate
// otherwise the master will deadlock trying to give work to this worker
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
return -1;
}
DEBUG("\nWriting %d to disk\n", coeff_count);
MPI_File_write_all(fh, &cor[0], coeff_count, MPI_DOUBLE, &stat);
if (coeff_count != 0 )
MPI_Type_free(&coeff_index_dt);
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
}
DEBUG("\nAbout to write to disk %d\n", rank);
MPI_File_sync( fh ) ; // Causes all previous writes to be transferred to the storage device
DEBUG("\nWritten to disk %d\n",rank);
// MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine.
DEBUG("\nAfter barrier \n", rank);
// Close file handler
MPI_File_close(&fh);
DEBUG("\nAfter file closed /n");
// MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine.
DEBUG("\nAbout to return from kernel /n");
return 0;
}
void readGraph_singleFile_MPI(graph_t *G, char *filename)
{
uint8_t align;
int rank, size;
int offset,offset_row ,offset_col,offset_weight;
edge_id_t my_edges[2];
int local_n=0;
int local_m=0;
int k;
uint32_t TotVertices;
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
G->rank = rank;
G->nproc = size;
G->filename[0] = '\0';
sprintf(G->filename, "%s", filename);
int lgsize;
for (lgsize = 0; lgsize < size; ++lgsize) {
if ((1 << lgsize) == size) break;
}
MPI_File fh;
MPI_Status status;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
MPI_File_read(fh, &G->n, 1, MPI_UINT32_T, &status);
offset = sizeof(vertex_id_t);
TotVertices = G->n;
MPI_File_read_at(fh, offset, &G->m, 1, MPI_UINT64_T, &status);
offset += sizeof(edge_id_t);
MPI_File_read_at(fh, offset, &G->directed, 1, MPI_C_BOOL, &status);
offset += sizeof(bool);
MPI_File_read_at(fh, offset, &align, 1, MPI_UINT8_T, &status);
offset += sizeof(uint8_t);
offset_row = offset;
for( uint32_t i = 0; i < G->n; i++ ) {
if( rank == VERTEX_OWNER(i,TotVertices,size) ) {
MPI_File_read_at(fh,offset_row + (i)*sizeof(edge_id_t),&my_edges[0], 2, MPI_UINT64_T, &status);
local_n++;
local_m += my_edges[1] - my_edges[0];
}
}
G->local_n = local_n;
G->local_m = local_m;
offset_col = offset_row + (G->n+1) * sizeof(edge_id_t);
offset_weight = offset_col + G->m * sizeof(vertex_id_t);
G->rowsIndices = (edge_id_t *)malloc((G->local_n + 1) * sizeof(edge_id_t) );
G->endV = (vertex_id_t *)malloc((G->local_m)*sizeof(vertex_id_t));
G->weights = (weight_t *)malloc((G->local_m)*sizeof(weight_t));
G->rowsIndices[0] = 0;
k = 1;
for( uint32_t i = 0; i < G->n; i++ ) {
if( rank == VERTEX_OWNER(i,TotVertices,size) ) {
MPI_File_read_at(fh,offset_row + (i)*sizeof(edge_id_t),&my_edges[0], 2, MPI_UINT64_T, &status);
G->rowsIndices[k] = G->rowsIndices[k-1] + my_edges[1] - my_edges[0];
MPI_File_read_at(fh,offset_col + my_edges[0] * sizeof(vertex_id_t), &G->endV[G->rowsIndices[k-1]], G->rowsIndices[k]-G->rowsIndices[k-1], MPI_UINT32_T, &status);
MPI_File_read_at(fh,offset_weight + my_edges[0] * sizeof(weight_t), &G->weights[G->rowsIndices[k-1]], G->rowsIndices[k]-G->rowsIndices[k-1], MPI_DOUBLE, &status);
k++;
}
}
MPI_File_close(&fh);
}
int main(int argc, char** argv)
{
int my_rank, p;
int i, dest;
mpz_t currentPrime;
unsigned long int product;
sscanf(argv[1], "%lu", &product);
int secondFactor = 0;
int bcastStatus;
int equals;
/** GMP library variables **/
mpz_t nextPrimeNumber;
mpz_t testFactor;
mpz_init(nextPrimeNumber);
mpz_init_set_str (nextPrimeNumber, argv[1], 10);
mpz_init(testFactor);
mpz_init_set_ui(currentPrime, 2);
mpz_nextprime(nextPrimeNumber, nextPrimeNumber);
mpz_t testProduct;
mpz_init(testProduct);
/** MPI Initialization **/
MPI_Request finalValue;
MPI_File out;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &p);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
MPI_Status status;
/** Get Ready to receive a factor if another process finds one */
MPI_Irecv(&secondFactor, 1, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &finalValue);
/** Prepare initial offset for each process **/
for (i=0 ; i < my_rank ; i++) {
mpz_nextprime(currentPrime, currentPrime);
}
/** Start Timing **/
double start = MPI_Wtime(), diff;
while (!secondFactor) {
/** Check if another process has found the factors **/
MPI_Test (&finalValue, &bcastStatus, &status);
if(bcastStatus) {
/** Somebody else has found the factors, we are done **/
MPI_Wait(&finalValue, &status);
break;
}
/** Skip P primes before checking again **/
for (i=0 ; i < p ; i++) {
mpz_nextprime(currentPrime, currentPrime);
}
/** Brute force check if the current working prime is a factor of the input number **/
for (mpz_set_ui(testFactor , 2) ; mpz_get_ui(testFactor) <= mpz_get_ui(currentPrime); mpz_nextprime(testFactor, testFactor)) {
/** Check if another process has found the factors **/
MPI_Test (&finalValue, &bcastStatus, &status);
if(bcastStatus) {
MPI_Wait(&finalValue, &status);
break;
}
mpz_mul_ui(testProduct, currentPrime, mpz_get_ui(testFactor));
equals = mpz_cmp_ui(testProduct, product);
if (equals == 0){
/** We've found the factor, find the second number, secnd it to the other processes **/
secondFactor = mpz_get_ui(testFactor);
printf("done by process %d, factors are %lu and %d \n", my_rank, mpz_get_ui(currentPrime), secondFactor);
fflush(stdout);
for (dest = 0 ; dest < p ; dest++) {
if (dest != my_rank) {
MPI_Send(&secondFactor, 1, MPI_UNSIGNED_LONG, dest, 0, MPI_COMM_WORLD);
}
}
}
}
}
diff = MPI_Wtime() - start;
/** End Timing **/
/** Prepare file contents **/
char fileName[200], fileContents[200];
sprintf(fileName, "time_%lu", product);
sprintf(fileContents, "%d\t%f\n", my_rank, diff);
/** Write File **/
MPI_File_open( MPI_COMM_WORLD, fileName, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &out );
MPI_File_seek(out, my_rank*strlen ( fileContents ) , MPI_SEEK_SET);
MPI_File_write_all(out , &fileContents, strlen ( fileContents ), MPI_CHAR, &status );
MPI_File_close(&out);
/** Fin **/
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return(0);
}
/**
* \brief Measures the time to write once to a file.
*
* Only one process is active. It writes once to a file.
*
* Remark:<br>
* With the <tt>O_DIRECT</tt> flag set, cache effects are minimized, because I/O
* is done directly to/from user space buffers. The operation system's page
* cache is bypassed. Under Linux 2.6 alignment to 512-byte boundaries is
* required for buffer and file offset. Thus the following parameters should be
* set in a SKaMPI input file:
* - <tt>set_send_buffert_alignment (512)</tt>
* - <tt>set_recv_buffert_alignment (512)</tt>
* - <tt>switch_buffer_cycling_off ()</tt><br>
*
* <tt>O_DIRECT</tt> is only relevant if the POSIX-API is used for I/O.
*
* For more information please refer to the <tt>open ()</tt> man pages.
*
* \param[in] size size of memory buffer, i.e. number of <tt>MPI_BYTE</tt>s
* \param[in] api POSIX-API or MPI-API for I/O accesses
* \param[in] create_flag write into existing file (FALSE) or create it (TRUE)
* \param[in] directio_flag open file with <tt>O_DIRECT</tt> flag to minimize
* cache effects
*
* \return measured time
*/
double measure_MPI_IO_write_file_once (int size, char *api, int create_flag, int directio_flag){
double start_time = 1.0, end_time = 0.0;
int open_flags;
char *error_string;
if (get_measurement_rank () == 0){
if (strcmp (api, POSIX_API) == 0){
if (directio_flag != 0)
open_flags = O_WRONLY | O_DIRECT;
else
open_flags = O_WRONLY;
errno = 0;
if (create_flag == 0){ /* open existing file */
if ((io_fd = open (io_filename, open_flags)) < 0){
error_string = strerror (errno);
error_with_abort (errno,
"\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed."
"\nCannot open local file (write only mode)."
"\nError: %s\n",
size, api, create_flag, directio_flag, error_string);
}
}
else { /* open nonexisting file and create it */
if ((io_fd = open (io_filename, open_flags|O_CREAT, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0){
error_string = strerror (errno);
error_with_abort (errno,
"\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed."
"\nCannot open local file (write only mode)."
"\nError: %s\n",
size, api, create_flag, directio_flag, error_string);
}
}
start_time = start_synchronization ();
write (io_fd, get_send_buffer (), size);
fsync (io_fd);
end_time = MPI_Wtime ();
close (io_fd);
}
else{ /* if strcmp (api, POSIX_API) != 0 */
if (create_flag == 0){
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &io_fh);
}
else{ /* if create_flag != 0*/
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &io_fh);
}
MPI_File_set_view (io_fh, (MPI_Offset)0,
MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
start_time = start_synchronization ();
MPI_File_write (io_fh, get_send_buffer (), size, MPI_BYTE, MPI_STATUS_IGNORE);
MPI_File_sync (io_fh);
end_time = MPI_Wtime ();
MPI_File_close (&io_fh);
}
}
else if (get_measurement_rank () != 0) {
start_synchronization ();
}
stop_synchronization ();
if (get_measurement_rank () == 0)
return end_time - start_time;
else
return -1.0;
}
int SeqSummaryCommand::execute(){
try{
if (abort == true) { if (calledHelp) { return 0; } return 2; }
//set current fasta to fastafile
m->setFastaFile(fastafile);
map<string, string> variables;
variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(fastafile));
string summaryFile = getOutputFileName("summary",variables);
int numSeqs = 0;
vector<int> startPosition;
vector<int> endPosition;
vector<int> seqLength;
vector<int> ambigBases;
vector<int> longHomoPolymer;
if (namefile != "") { nameMap = m->readNames(namefile); }
else if (countfile != "") {
CountTable ct;
ct.readTable(countfile, false, false);
nameMap = ct.getNameMap();
}
if (m->control_pressed) { return 0; }
#ifdef USE_MPI
int pid, numSeqsPerProcessor;
int tag = 2001;
int startTag = 1; int endTag = 2; int lengthTag = 3; int baseTag = 4; int lhomoTag = 5;
int outMode=MPI_MODE_CREATE|MPI_MODE_WRONLY;
vector<unsigned long long> MPIPos;
MPI_Status status;
MPI_Status statusOut;
MPI_File inMPI;
MPI_File outMPI;
MPI_Comm_size(MPI_COMM_WORLD, &processors);
MPI_Comm_rank(MPI_COMM_WORLD, &pid);
char tempFileName[1024];
strcpy(tempFileName, fastafile.c_str());
char sumFileName[1024];
strcpy(sumFileName, summaryFile.c_str());
MPI_File_open(MPI_COMM_WORLD, tempFileName, MPI_MODE_RDONLY, MPI_INFO_NULL, &inMPI); //comm, filename, mode, info, filepointer
MPI_File_open(MPI_COMM_WORLD, sumFileName, outMode, MPI_INFO_NULL, &outMPI);
if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); return 0; }
if (pid == 0) { //you are the root process
//print header
string outputString = "seqname\tstart\tend\tnbases\tambigs\tpolymer\tnumSeqs\n";
int length = outputString.length();
char* buf2 = new char[length];
memcpy(buf2, outputString.c_str(), length);
MPI_File_write_shared(outMPI, buf2, length, MPI_CHAR, &statusOut);
delete buf2;
MPIPos = m->setFilePosFasta(fastafile, numSeqs); //fills MPIPos, returns numSeqs
for(int i = 1; i < processors; i++) {
MPI_Send(&numSeqs, 1, MPI_INT, i, tag, MPI_COMM_WORLD);
MPI_Send(&MPIPos[0], (numSeqs+1), MPI_LONG, i, tag, MPI_COMM_WORLD);
}
//figure out how many sequences you have to do
numSeqsPerProcessor = numSeqs / processors;
int startIndex = pid * numSeqsPerProcessor;
if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; }
//do your part
MPICreateSummary(startIndex, numSeqsPerProcessor, startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, inMPI, outMPI, MPIPos);
}else { //i am the child process
MPI_Recv(&numSeqs, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
MPIPos.resize(numSeqs+1);
MPI_Recv(&MPIPos[0], (numSeqs+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status);
//figure out how many sequences you have to align
numSeqsPerProcessor = numSeqs / processors;
int startIndex = pid * numSeqsPerProcessor;
if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; }
//do your part
MPICreateSummary(startIndex, numSeqsPerProcessor, startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, inMPI, outMPI, MPIPos);
}
MPI_File_close(&inMPI);
MPI_File_close(&outMPI);
MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
if (pid == 0) {
//get the info from the child processes
for(int i = 1; i < processors; i++) {
int size;
MPI_Recv(&size, 1, MPI_INT, i, tag, MPI_COMM_WORLD, &status);
vector<int> temp; temp.resize(size+1);
for(int j = 0; j < 5; j++) {
MPI_Recv(&temp[0], (size+1), MPI_INT, i, 2001, MPI_COMM_WORLD, &status);
int receiveTag = temp[temp.size()-1]; //child process added a int to the end to indicate what count this is for
if (receiveTag == startTag) {
for (int k = 0; k < size; k++) { startPosition.push_back(temp[k]); }
}else if (receiveTag == endTag) {
for (int k = 0; k < size; k++) { endPosition.push_back(temp[k]); }
}else if (receiveTag == lengthTag) {
for (int k = 0; k < size; k++) { seqLength.push_back(temp[k]); }
}else if (receiveTag == baseTag) {
for (int k = 0; k < size; k++) { ambigBases.push_back(temp[k]); }
}else if (receiveTag == lhomoTag) {
for (int k = 0; k < size; k++) { longHomoPolymer.push_back(temp[k]); }
}
}
}
}else{
//send my counts
int size = startPosition.size();
MPI_Send(&size, 1, MPI_INT, 0, tag, MPI_COMM_WORLD);
startPosition.push_back(startTag);
int ierr = MPI_Send(&(startPosition[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD);
endPosition.push_back(endTag);
ierr = MPI_Send (&(endPosition[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD);
seqLength.push_back(lengthTag);
ierr = MPI_Send(&(seqLength[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD);
ambigBases.push_back(baseTag);
ierr = MPI_Send(&(ambigBases[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD);
longHomoPolymer.push_back(lhomoTag);
ierr = MPI_Send(&(longHomoPolymer[0]), (size+1), MPI_INT, 0, 2001, MPI_COMM_WORLD);
}
MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
#else
vector<unsigned long long> positions;
#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
positions = m->divideFile(fastafile, processors);
for (int i = 0; i < (positions.size()-1); i++) { lines.push_back(new linePair(positions[i], positions[(i+1)])); }
#else
positions = m->setFilePosFasta(fastafile, numSeqs);
if (positions.size() < processors) { processors = positions.size(); }
//figure out how many sequences you have to process
int numSeqsPerProcessor = numSeqs / processors;
for (int i = 0; i < processors; i++) {
int startIndex = i * numSeqsPerProcessor;
if(i == (processors - 1)){ numSeqsPerProcessor = numSeqs - i * numSeqsPerProcessor; }
lines.push_back(new linePair(positions[startIndex], numSeqsPerProcessor));
}
#endif
if(processors == 1){
numSeqs = driverCreateSummary(startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, fastafile, summaryFile, lines[0]);
}else{
numSeqs = createProcessesCreateSummary(startPosition, endPosition, seqLength, ambigBases, longHomoPolymer, fastafile, summaryFile);
}
if (m->control_pressed) { return 0; }
#endif
#ifdef USE_MPI
if (pid == 0) {
#endif
sort(startPosition.begin(), startPosition.end());
sort(endPosition.begin(), endPosition.end());
sort(seqLength.begin(), seqLength.end());
sort(ambigBases.begin(), ambigBases.end());
sort(longHomoPolymer.begin(), longHomoPolymer.end());
int size = startPosition.size();
//find means
unsigned long long meanStartPosition, meanEndPosition, meanSeqLength, meanAmbigBases, meanLongHomoPolymer;
meanStartPosition = 0; meanEndPosition = 0; meanSeqLength = 0; meanAmbigBases = 0; meanLongHomoPolymer = 0;
for (int i = 0; i < size; i++) {
meanStartPosition += startPosition[i];
meanEndPosition += endPosition[i];
meanSeqLength += seqLength[i];
meanAmbigBases += ambigBases[i];
meanLongHomoPolymer += longHomoPolymer[i];
}
double meanstartPosition, meanendPosition, meanseqLength, meanambigBases, meanlongHomoPolymer;
meanstartPosition = meanStartPosition / (double) size; meanendPosition = meanEndPosition /(double) size; meanlongHomoPolymer = meanLongHomoPolymer / (double) size; meanseqLength = meanSeqLength / (double) size; meanambigBases = meanAmbigBases /(double) size;
int ptile0_25 = int(size * 0.025);
int ptile25 = int(size * 0.250);
int ptile50 = int(size * 0.500);
int ptile75 = int(size * 0.750);
int ptile97_5 = int(size * 0.975);
int ptile100 = size - 1;
//to compensate for blank sequences that would result in startPosition and endPostion equalling -1
if (startPosition[0] == -1) { startPosition[0] = 0; }
if (endPosition[0] == -1) { endPosition[0] = 0; }
if (m->control_pressed) { m->mothurRemove(summaryFile); return 0; }
m->mothurOutEndLine();
m->mothurOut("\t\tStart\tEnd\tNBases\tAmbigs\tPolymer\tNumSeqs"); m->mothurOutEndLine();
m->mothurOut("Minimum:\t" + toString(startPosition[0]) + "\t" + toString(endPosition[0]) + "\t" + toString(seqLength[0]) + "\t" + toString(ambigBases[0]) + "\t" + toString(longHomoPolymer[0]) + "\t" + toString(1)); m->mothurOutEndLine();
m->mothurOut("2.5%-tile:\t" + toString(startPosition[ptile0_25]) + "\t" + toString(endPosition[ptile0_25]) + "\t" + toString(seqLength[ptile0_25]) + "\t" + toString(ambigBases[ptile0_25]) + "\t"+ toString(longHomoPolymer[ptile0_25]) + "\t" + toString(ptile0_25+1)); m->mothurOutEndLine();
m->mothurOut("25%-tile:\t" + toString(startPosition[ptile25]) + "\t" + toString(endPosition[ptile25]) + "\t" + toString(seqLength[ptile25]) + "\t" + toString(ambigBases[ptile25]) + "\t" + toString(longHomoPolymer[ptile25]) + "\t" + toString(ptile25+1)); m->mothurOutEndLine();
m->mothurOut("Median: \t" + toString(startPosition[ptile50]) + "\t" + toString(endPosition[ptile50]) + "\t" + toString(seqLength[ptile50]) + "\t" + toString(ambigBases[ptile50]) + "\t" + toString(longHomoPolymer[ptile50]) + "\t" + toString(ptile50+1)); m->mothurOutEndLine();
m->mothurOut("75%-tile:\t" + toString(startPosition[ptile75]) + "\t" + toString(endPosition[ptile75]) + "\t" + toString(seqLength[ptile75]) + "\t" + toString(ambigBases[ptile75]) + "\t" + toString(longHomoPolymer[ptile75]) + "\t" + toString(ptile75+1)); m->mothurOutEndLine();
m->mothurOut("97.5%-tile:\t" + toString(startPosition[ptile97_5]) + "\t" + toString(endPosition[ptile97_5]) + "\t" + toString(seqLength[ptile97_5]) + "\t" + toString(ambigBases[ptile97_5]) + "\t" + toString(longHomoPolymer[ptile97_5]) + "\t" + toString(ptile97_5+1)); m->mothurOutEndLine();
m->mothurOut("Maximum:\t" + toString(startPosition[ptile100]) + "\t" + toString(endPosition[ptile100]) + "\t" + toString(seqLength[ptile100]) + "\t" + toString(ambigBases[ptile100]) + "\t" + toString(longHomoPolymer[ptile100]) + "\t" + toString(ptile100+1)); m->mothurOutEndLine();
m->mothurOut("Mean:\t" + toString(meanstartPosition) + "\t" + toString(meanendPosition) + "\t" + toString(meanseqLength) + "\t" + toString(meanambigBases) + "\t" + toString(meanlongHomoPolymer)); m->mothurOutEndLine();
if ((namefile == "") && (countfile == "")) { m->mothurOut("# of Seqs:\t" + toString(numSeqs)); m->mothurOutEndLine(); }
else { m->mothurOut("# of unique seqs:\t" + toString(numSeqs)); m->mothurOutEndLine(); m->mothurOut("total # of seqs:\t" + toString(startPosition.size())); m->mothurOutEndLine(); }
if (m->control_pressed) { m->mothurRemove(summaryFile); return 0; }
m->mothurOutEndLine();
m->mothurOut("Output File Names: "); m->mothurOutEndLine();
m->mothurOut(summaryFile); m->mothurOutEndLine(); outputNames.push_back(summaryFile); outputTypes["summary"].push_back(summaryFile);
m->mothurOutEndLine();
#ifdef USE_MPI
}
#endif
//set fasta file as new current fastafile
string current = "";
itTypes = outputTypes.find("summary");
if (itTypes != outputTypes.end()) {
if ((itTypes->second).size() != 0) { current = (itTypes->second)[0]; m->setSummaryFile(current); }
}
return 0;
}
catch(exception& e) {
m->errorOut(e, "SeqSummaryCommand", "execute");
exit(1);
}
}
int main(int argc, char ** argv) {
int nprocs, mynod, errcode;
options my_options = {NULL, 0, 0};
MPI_File fh;
MPI_Status status;
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
parse_args(argc, argv, mynod, &my_options);
if (my_options.do_aggregation) {
MPI_Info_create(&info);
MPI_Info_set(info, "romio_no_indep_rw", "true");
MPI_Info_set(info, "cb_config_list", "leela.mcs.anl.gov:1");
} else {
info = MPI_INFO_NULL;
}
/* create the file w/o EXCL: this must not fail */
errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
MPI_MODE_CREATE|MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_close");
}
/* now try to open w/ CREAT|EXCL: this must fail */
errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh);
if (errcode == MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open: expected an error: got");
}
/* ignore the error: File_delete is not aggregator-aware */
MPI_File_delete(my_options.fname, info);
/* this must succeed: the file no longer exists */
errcode = MPI_File_open(MPI_COMM_WORLD, my_options.fname,
MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_close");
}
if (mynod == 0) {
printf(" No Errors\n");
}
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
int iarrayOfSizes[2], iarrayOfSubsizes[2], iarrayOfStarts[2], ilocal_size;
int nproc[2], periods[2], icoord[2];
int m, n, i, j, wsize, wrank, crank, ndims, lrows, lcols, grow, gcol, err;
MPI_Datatype filetype;
MPI_File fh;
MPI_Comm cartcomm;
MPI_Info info0, info3;
double t, topen, twrite, tclose, wrate;
double *local_array;
char nstripesStr[12], stripeUnitStr[12];
int nstripes = -1;
int stripeUnit = -1;
MPI_Offset headerSize = 0;
MPI_Init(0,0);
MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
/* Get global array size */
m = n = 128; /* Set default size */
/* ioda [ n ] [ m ] [ nstripes ] [ stripeunit ] [ headersize ] */
if (argc > 0) {
if (argc > 1) m = atoi(argv[1]);
if (argc > 2) n = atoi(argv[2]);
if (argc > 3) nstripes = atoi(argv[3]);
if (argc > 4) stripeUnit = atoi(argv[4]);
if (argc > 5) headerSize = atoi(argv[5]);
if (argc > 6) {
if (wrank == 0)
fprintf(stderr,"Unrecognized argument %s\n", argv[6]);
MPI_Abort(MPI_COMM_WORLD,1);
}
}
if (wrank == 0) printf("Matrix is [%d,%d]; file dir = %s\n", m, n, MYSCRATCHDIR );
/* The default number of stripes = totalsize/1M */
if (nstripes < 0) {
nstripes = n * m * sizeof(double) / (1024*1024);
if (nstripes < 1) nstripes = 1;
}
if (wrank == 0) printf("nstripes = %d, stripeUnit = %d, header size = %d\n",
nstripes, stripeUnit, (int)headerSize);
/* Use topology routines to get decomposition and coordinates */
MPI_Comm_size(MPI_COMM_WORLD, &wsize);
nproc[0] = 0; nproc[1] = 0;
ndims = 2;
MPI_Dims_create(wsize, ndims, nproc);
periods[0] = 0; periods[1] = 0;
MPI_Cart_create(MPI_COMM_WORLD, ndims, nproc, periods, 1, &cartcomm);
MPI_Comm_rank(cartcomm, &crank);
MPI_Cart_coords(cartcomm, crank, ndims, icoord);
iarrayOfSizes[0] = m;
iarrayOfSizes[1] = n;
iarrayOfSubsizes[0] = m/nproc[0];
iarrayOfSubsizes[1] = n/nproc[1];
iarrayOfStarts[0] = icoord[0] * iarrayOfSubsizes[0];
iarrayOfStarts[1] = icoord[1] * iarrayOfSubsizes[1];
/* Initialize my block of the data */
ilocal_size = iarrayOfSubsizes[0] * iarrayOfSubsizes[1];
lrows = iarrayOfSubsizes[0];
lcols = iarrayOfSubsizes[1];
local_array = (double *)malloc(lrows*lcols*sizeof(double));
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (i=0; i<lrows; i++) {
for (j=0; j<lcols; j++) {
local_array[j*lrows+i] = (grow+i) + (gcol+j)*m;
}
}
/* Fortran order simply means the data is stored by columns */
MPI_Type_create_subarray(ndims, iarrayOfSizes, iarrayOfSubsizes,
iarrayOfStarts, MPI_ORDER_FORTRAN, MPI_DOUBLE,
&filetype);
MPI_Type_commit(&filetype);
info0 = MPI_INFO_NULL;
info3 = MPI_INFO_NULL;
if (nstripes > 0 || stripeUnit > 0) {
MPI_Info_create(&info0);
if (nstripes > 0) {
snprintf(nstripesStr, sizeof(nstripesStr), "%d", nstripes);
MPI_Info_set(info0, "striping_factor", nstripesStr);
MPI_Info_set(info0, "cb_nodes", nstripesStr);
}
if (stripeUnit > 0) {
snprintf(stripeUnitStr, sizeof(stripeUnitStr), "%d", stripeUnit);
MPI_Info_set(info0, "striping_unit", stripeUnitStr);
}
MPI_Info_dup(info0, &info3);
MPI_Info_set(info3, "romio_no_indep_rw", "true");
/* Other hints to consider:
direct_io=true
The default cb_buffer_size is 16777216 , but is overridden by the
striping unit, which is smaller by default.
*/
}
/* level - 3 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-3.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info3, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-3.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_File_set_view(fh, headerSize, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_write_all(fh, local_array, ilocal_size, MPI_DOUBLE,
MPI_STATUS_IGNORE);
twrite = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "collective write");
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-3.out");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
/* level - 0 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-0.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info0, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-0.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (j=0; j<lcols; j++) {
MPI_Offset offset = headerSize +
((MPI_Offset)(grow) + (MPI_Offset)(gcol+j)*m) * sizeof(double);
err = MPI_File_write_at(fh, offset, local_array+j*lrows, lrows, MPI_DOUBLE,
MPI_STATUS_IGNORE);
if (err != MPI_SUCCESS) myAbort(err, "write at");
}
twrite = MPI_Wtime() - t;
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-0");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
if (info0 != MPI_INFO_NULL) {
MPI_Info_free(&info0);
MPI_Info_free(&info3);
}
free(local_array);
MPI_Finalize();
return 0;
}
