void
qpb_read_gauge(qpb_gauge_field gauge_field, size_t offset, size_t precision, char fname[])
{
MPI_Datatype mpi_dtype_link, filetype;
if(precision == 32)
{
MPI_Type_contiguous(4*2*NC*NC*ND, MPI_BYTE, &mpi_dtype_link);
}
else if(precision == 64)
{
MPI_Type_contiguous(8*2*NC*NC*ND, MPI_BYTE, &mpi_dtype_link);
}
else
{
fprintf(stderr, "%s: precision should be either 32 or 64\n", __func__);
exit(QPB_NOT_IMPLEMENTED_ERROR);
}
MPI_Type_commit(&mpi_dtype_link);
int starts[ND], l_dim[ND], g_dim[ND];
for(int i=0; i<ND; i++)
{
starts[i] = problem_params.coords[i]*problem_params.l_dim[i];
l_dim[i] = problem_params.l_dim[i];
g_dim[i] = problem_params.g_dim[i];
};
int ierr = MPI_Type_create_subarray(ND, g_dim, l_dim, starts, MPI_ORDER_C,
mpi_dtype_link, &filetype);
MPI_Type_commit(&filetype);
MPI_File fhandle;
ierr = MPI_File_open(MPI_COMM_WORLD, fname,
MPI_MODE_RDONLY, MPI_INFO_NULL, &fhandle);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_open() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
ierr = MPI_File_set_view(fhandle, (MPI_Offset)offset, mpi_dtype_link, filetype,
"native", MPI_INFO_NULL);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_set_view() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
void *buffer = NULL;
if(precision == 32)
{
buffer = qpb_alloc(problem_params.l_vol*ND*sizeof(qpb_link_float));
}
else if(precision == 64)
{
buffer = qpb_alloc(problem_params.l_vol*ND*sizeof(qpb_link_double));
}
MPI_Status status;
ierr = MPI_File_read_all(fhandle, buffer, problem_params.l_vol,
mpi_dtype_link, &status);
MPI_Type_free(&mpi_dtype_link);
MPI_Type_free(&filetype);
ierr = MPI_File_close(&fhandle);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_close() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_read() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
if(!qpb_is_bigendian())
{
if(precision == 32)
qpb_byte_swap_float(buffer, problem_params.l_vol*ND*NC*NC*2);
if(precision == 64)
qpb_byte_swap_double(buffer, problem_params.l_vol*ND*NC*NC*2);
}
for(int mu=0; mu<ND; mu++)
for(int v=0; v<problem_params.l_vol; v++)
for(int col=0; col<NC*NC; col++)
{
if(precision == 32)
gauge_field.bulk[v][ND-1 - mu][col] = (qpb_complex) {
((float *) buffer)[2*col + 2*mu*NC*NC + 2*v*NC*NC*ND],
((float *) buffer)[1 + 2*col + 2*mu*NC*NC + 2*v*NC*NC*ND]
};
if(precision == 64)
gauge_field.bulk[v][ND-1 - mu][col] = (qpb_complex) {
(float)((double *) buffer)[2*col + 2*mu*NC*NC + 2*v*NC*NC*ND],
(float)((double *) buffer)[1 + 2*col + 2*mu*NC*NC + 2*v*NC*NC*ND]
};
}
free(buffer);
return;
}
int main(int argc, char **argv)
{
MPI_Datatype newtype;
int i, ndims, array_of_gsizes[3], array_of_distribs[3];
int order, nprocs, len, flag, err;
int array_of_dargs[3], array_of_psizes[3];
int *readbuf, *writebuf, bufcount, mynod;
char filename[1024];
MPI_File fh;
MPI_Status status;
MPI_Aint size_with_aint;
MPI_Offset size_with_offset;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!mynod) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
printf("\n*# Usage: large_array -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
printf("This program creates a 4 Gbyte file. Don't run it if you don't have that much disk space!\n");
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
/* create the distributed array filetype */
ndims = 3;
order = MPI_ORDER_C;
array_of_gsizes[0] = 1024;
array_of_gsizes[1] = 1024;
array_of_gsizes[2] = 4*1024/sizeof(int);
array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
for (i=0; i<ndims; i++) array_of_psizes[i] = 0;
MPI_Dims_create(nprocs, ndims, array_of_psizes);
/* check if MPI_Aint is large enough for size of global array.
if not, complain. */
size_with_aint = sizeof(int);
for (i=0; i<ndims; i++) size_with_aint *= array_of_gsizes[i];
size_with_offset = sizeof(int);
for (i=0; i<ndims; i++) size_with_offset *= array_of_gsizes[i];
if (size_with_aint != size_with_offset) {
printf("Can't use an array of this size unless the MPI implementation defines a 64-bit MPI_Aint\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes,
array_of_distribs, array_of_dargs,
array_of_psizes, order, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
/* initialize writebuf */
MPI_Type_size(newtype, &bufcount);
bufcount = bufcount/sizeof(int);
writebuf = (int *) malloc(bufcount * sizeof(int));
if (!writebuf) printf("Process %d, not enough memory for writebuf\n", mynod);
for (i=0; i<bufcount; i++) writebuf[i] = mynod*1024 + i;
/* write the array to the file */
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
MPI_File_write_all(fh, writebuf, bufcount, MPI_INT, &status);
MPI_File_close(&fh);
free(writebuf);
/* now read it back */
readbuf = (int *) calloc(bufcount, sizeof(int));
if (!readbuf) printf("Process %d, not enough memory for readbuf\n", mynod);
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL);
MPI_File_read_all(fh, readbuf, bufcount, MPI_INT, &status);
MPI_File_close(&fh);
/* check the data read */
flag = 0;
for (i=0; i<bufcount; i++)
if (readbuf[i] != mynod*1024 + i) {
printf("Process %d, readbuf=%d, writebuf=%d\n", mynod, readbuf[i], mynod*1024 + i);
flag = 1;
}
if (!flag) printf("Process %d: data read back is correct\n", mynod);
MPI_Type_free(&newtype);
free(readbuf);
MPI_Barrier(MPI_COMM_WORLD);
if (!mynod) {
err = MPI_File_delete(filename, MPI_INFO_NULL);
if (err == MPI_SUCCESS) printf("file deleted\n");
}
MPI_Finalize();
return 0;
}
double eigenvalues_Jacobi(int * nr_of_eigenvalues, const int max_iterations,
const double precision, const int maxmin,int tslice,
const int nstore) {
double returnvalue;
static int allocated = 0;
#ifdef HAVE_LAPACK
int verbosity = 1, converged = 0, blocksize = 1 , blockwise=0;
int solver_it_max = 50, j_max, j_min;
double decay_min = 1.7, decay_max = 1.5, prec, threshold_min = 1.e-3, threshold_max = 5.e-2;
int v0dim = 0;
matrix_mult_su3vect f;
int N=SPACEVOLUME, N2=(SPACEVOLUME + SPACERAND);
su3_vector * max_eigenvector_ = NULL, *max_eigenvector;
int returncode=0;
int returncode2=0;
su3_vector *s;
double sqnorm;
char filename[200];
char eigvl_filename[200];
// int dims[]={T*g_nproc_t, LX*g_nproc_x, LY*g_nproc_y, LZ*g_nproc_z};
int dims[]={1, LX*g_nproc_x, LY*g_nproc_y, LZ*g_nproc_z};
FILE *efp;
#ifdef MPI
double atime, etime;
MPI_File fp;
MPI_Offset siteSize=3*2*sizeof(double);
LemonRecordHeader *header;
LemonWriter *writer;
#else
FILE *fp;
int siteSize=3*2*sizeof(double);
#endif
f = &Jacobi;
evlength_su3v = N2;
if(g_proc_id == g_stdio_proc && g_debug_level >0)
{
printf("Number of %s eigenvalues to compute = %d\n",
maxmin ? "maximal" : "minimal",(*nr_of_eigenvalues));
printf("Using Jacobi-Davidson method! \n");
}
if((*nr_of_eigenvalues) < 8){
j_max = 15;
j_min = 8;
}
else{
j_max = 2*(*nr_of_eigenvalues);
j_min = (*nr_of_eigenvalues);
}
if(precision < 1.e-14){
prec = 1.e-14;
}
else{
prec = precision;
}
max_eigenvector_= calloc(N2, sizeof(su3_vector));
max_eigenvector = max_eigenvector_;
if(allocated == 0)
{
allocated = 1;
eigenvectors_su3v = calloc(N2*(*nr_of_eigenvalues), sizeof(su3_vector));;
eigenvls_su3v = (double*)malloc((*nr_of_eigenvalues)*sizeof(double));
inv_eigenvls_su3v = (double*)malloc((*nr_of_eigenvalues)*sizeof(double));
}
solver_it_max = 64;
/* compute the maximal one first */
/* DEBUG
jdher_su3vect(N*sizeof(su3_vector)/sizeof(_Complex double), N2*sizeof(su3_vector)/sizeof(_Complex double),
50., 1.e-12,
1, 15, 8, max_iterations, 1, 0, 0, NULL,
CG, solver_it_max,
threshold_max, decay_max, verbosity,
&converged, (_Complex double*) max_eigenvector, (double*) &max_eigenvalue_su3v,
&returncode2, JD_MAXIMAL, 1,tslice,f);
*/
atime = gettime();
/* (re-) compute minimal eigenvalues */
converged = 0;
solver_it_max = 256;
if(maxmin)
jdher_su3vect(N*sizeof(su3_vector)/sizeof(_Complex double), N2*sizeof(su3_vector)/sizeof(_Complex double),
50., prec,
(*nr_of_eigenvalues), j_max, j_min,
max_iterations, blocksize, blockwise, v0dim, (_Complex double*) eigenvectors_su3v,
CG, solver_it_max,
threshold_max, decay_max, verbosity,
&converged, (_Complex double*) eigenvectors_su3v, eigenvls_su3v,
&returncode, JD_MAXIMAL, 1,tslice,
f);
else
jdher_su3vect(N*sizeof(su3_vector)/sizeof(_Complex double), N2*sizeof(su3_vector)/sizeof(_Complex double),
0., prec,
(*nr_of_eigenvalues), j_max, j_min,
max_iterations, blocksize, blockwise, v0dim, (_Complex double*) eigenvectors_su3v,
CG, solver_it_max,
threshold_min, decay_min, verbosity,
&converged, (_Complex double*) eigenvectors_su3v, eigenvls_su3v,
&returncode, JD_MINIMAL, 1,tslice,
f);
etime = gettime();
if(g_proc_id == 0) {
printf("Eigenvalues computed in %e sec. (gettime)\n", etime-atime);
}
/* Printout eigenvalues. */
if(g_proc_id == 0) {
sprintf(eigvl_filename,"eigenvalues.%.3d.%.4d", tslice, nstore);
efp=fopen(eigvl_filename,"w");
for(v0dim = 0; v0dim < (*nr_of_eigenvalues); v0dim++) {
fprintf(efp,"%e\n",eigenvls_su3v[v0dim]);
}
fclose(efp);
}
/* Printout eigenvectors. */
for(v0dim = 0; v0dim < (*nr_of_eigenvalues); v0dim++) {
sprintf(filename, "eigenvector.%.3d.%.3d.%.4d", v0dim, tslice, nstore);
s=(su3_vector*)&eigenvectors_su3v[v0dim*N2];
#ifdef MPI
# ifdef HAVE_LIBLEMON
// SEGNO: dovrebbe stampare 8*2*3*SPACEVOLUME data per file, ma ne stampa 8*2*4n*SPACEVOLUME (n=4-1 per ev 0-3)
MPI_File_open(g_cart_grid, filename, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fp);
writer = lemonCreateWriter(&fp, g_cart_grid);
header = lemonCreateHeader(1 /* MB */, 1 /* ME */, "lattice-su3_vector-data",SPACEVOLUME*3*sizeof(_Complex double));
lemonWriteRecordHeader(header, writer);
lemonDestroyHeader(header);
lemonWriteLatticeParallel(writer, s, siteSize, dims);
lemonWriterCloseRecord(writer);
lemonDestroyWriter(writer);
MPI_File_close(&fp);
# else
if(g_proc_id == 0) {
printf("Cannot write eigenvectors: you need LEMON for writing eigenvectors with MPI\n");
}
# endif
#else
fp=fopen(filename,"wb");
fwrite(s,siteSize,SPACEVOLUME,fp);
fclose(fp);
#endif // MPI
sqnorm=square_norm_su3vect(s,SPACEVOLUME,1);
if(g_proc_id == 0) {
printf("wrote eigenvector | |^2 = %e \n",sqnorm);
}
}
returnvalue=eigenvls_su3v[0];
free(max_eigenvector_);
#else
fprintf(stderr, "lapack not available, so JD method for EV computation not available \n");
#endif // LAPACK
return(returnvalue);
}
void writecomputetrace_(int* pcompstep, double* pdtime, double* pcpu_t)
#endif
{
COMPUTE_TRACE_FLAG = 1; // only for param print
// now the control is off to the solver, i/o kernel just provides the function
// it's up to solver to decide when to use compute trace
// if(COMPUTE_TRACE_FLAG != 1)
// return;
char tracefname[kMaxPathLen];
int formatparam = trace_ioop;
int nfile = trace_nf;
int stepnum = *pcompstep;
double dtime = *pdtime;
double cpu_t = *pcpu_t;
// only write every few steps TODO: get param(13) IOCOMM and compare with it
if(stepnum%100 != 0) return;
//printf("iostep is %d, dtime = %lf\n", *pcompstep, *pdtime);
int temp_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &temp_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mysize);
MPI_Barrier(MPI_COMM_WORLD);
memset((void*)tracefname, 0, kMaxPathLen);
//sprintf(tracefname, "%s/compute-trace-%d-proc-ioop-%d-nf-%d-t%.5d.dat",
// kOutputPath, mysize, formatparam, nfile, stepnum);
// note: this might be called before going into any io func, so "path" is not set yet
sprintf(tracefname, "%s/compute-trace-%d-proc-istep-%.5d-ioop-%d-nf-%d.dat",
kOutputPath, mysize, stepnum, trace_ioop, trace_nf);
//printf("my filename %s (myrank=%d) \n", tracefname, temp_rank);
// write the actual file
if (1) {
MPI_File timefile;
int rc;
rc = MPI_File_open(MPI_COMM_WORLD, tracefname,
MPI_MODE_CREATE | MPI_MODE_WRONLY , MPI_INFO_NULL, &timefile);
if(rc) {
if(temp_rank == 0) printf("Unble to open file %s, error code:%d! \n", tracefname, rc);
}
char mytime[128];
memset(mytime, 0, 128);
sprintf(mytime, "%10d %10.3lf %10.3lf\n",
temp_rank, dtime, cpu_t);
int len = strlen(mytime);
//printf("str len = %d\n", len);
long long offsets = temp_rank * len ;
MPI_Status write_data_status;
MPI_File_write_at_all_begin(timefile,
offsets,
mytime,
len,
MPI_CHAR);
MPI_File_write_at_all_end(timefile,
mytime,
&write_data_status);
MPI_File_close( & timefile );
}
//printf("writecomputetrace() finished, myrank = %d\n", temp_rank);
}
int main(int argc, char *argv[])
{
int width, height, maxiter, flag;
double x[2], y[2], c[2];
char *image, *stats;
int comm_sz, my_rank;
double t1, t2, delta;
// Get and parse the program parameters
getParams(argv, &flag, c, x, y, &width, &height, &maxiter, &image, &stats);
// Allocate space for the image
int *iterations = (int*)malloc( sizeof(int) * width * height );
assert(iterations != NULL);
// Start MPI
MPI_Init(NULL, NULL);
MPI_Comm_size(MPI_COMM_WORLD, &comm_sz);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
// Begin process timer
t1 = MPI_Wtime();
/* compute set */
int maxCount = parallelJulia(x, width, y, height, c, flag, maxiter, iterations, my_rank, comm_sz, MPI_COMM_WORLD);
// Stop timer and compute time elapse
t2 = MPI_Wtime();
delta = t2 - t1;
if (my_rank == 0)
{
/* save our picture for the viewer */
printf("\nMaster process %d creating image...\n", my_rank);
saveBMP(image, iterations, width, height);
printf("\nFinished image creation\n");
}
// Wait for all processes to finish Julia computations
MPI_Barrier(MPI_COMM_WORLD);
// Open stats file
MPI_File statsFile;
if (MPI_File_open(MPI_COMM_WORLD, stats, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &statsFile) == MPI_SUCCESS)
{
// Generate statistic string
char message[100];
sprintf(message, "process %d: max iterations reached = %d, time elapsed = %lf\n", my_rank, maxCount, delta);
MPI_File_write_ordered(statsFile, message, strlen(message), MPI_CHAR, MPI_STATUS_IGNORE);
MPI_File_close(&statsFile);
}
else printf("Problem opening file on process %d\n", my_rank);
// Close MPI environment
MPI_Finalize();
// Free reserved memory
free(iterations);
return 0;
}
int main(int argc, char *argv[])
{
int rank, size;
const int N = atoi(argv[1]);
// printf("Number of testcase = %d\n", N);
MPI_Init (&argc, &argv);
double start_time, end_time;
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
MPI_Comm_size (MPI_COMM_WORLD, &size);
// printf("My rank is %d \n", rank);
//start_time = MPI_Wtime();
MPI_File fin, fout;
MPI_Status status;
int *root_arr;
int max_arr_size = size > N ? size : N;
int ret = MPI_File_open(MPI_COMM_WORLD, argv[2],
MPI_MODE_RDONLY, MPI_INFO_NULL, &fin);
if (rank == ROOT) {
root_arr = new int[max_arr_size+3];
// printf("Enter rank 0 statement ... \n");
MPI_File_read(fin, root_arr, N, MPI_INT, &status);
/*
for (int i = 0; i < N; ++i)
printf("[START] [Rank %d] root_arr[%d] = %d\n", rank, i, root_arr[i]);
printf("Out Rank 0 statement ... \n");
*/
}
MPI_File_close(&fin);
MPI_Barrier(MPI_COMM_WORLD); // Wait for rank0 to read file
int rank_num = size > N ? N : size;
const int LAST = rank_num - 1;
int num_per_node = N / rank_num;
int *local_arr;
int num_per_node_diff = N - num_per_node * rank_num;
int diff = num_per_node_diff;
bool has_remain = false;
bool has_remain_rank = rank_num % 2 ? true : false;
if (num_per_node_diff > 0) {
// Send remaining elements to size - 1
has_remain = true;
if (rank == ROOT) {
MPI_Send(root_arr + N - diff, diff, MPI_INT, LAST, 0, MPI_COMM_WORLD);
} else if (rank == LAST) {
// Handle special case
num_per_node += num_per_node_diff;
local_arr = new int[num_per_node+1];
MPI_Recv(local_arr + num_per_node - diff, diff,
MPI_INT, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
}
} else if(rank == rank_num - 1) {
local_arr = new int[num_per_node+1];
}
MPI_Barrier(MPI_COMM_WORLD); // Wait for rank0 to read file
if (rank != rank_num - 1)
local_arr = new int[num_per_node+1];
// MPI_Scatter (send_buf, send_count, send_type, recv_buf, recv_count, recv_type, root, comm)
if (rank < LAST)
MPI_Scatter(root_arr, num_per_node, MPI_INT, local_arr,
num_per_node, MPI_INT, ROOT, MPI_COMM_WORLD);
else
MPI_Scatter(root_arr, num_per_node-diff, MPI_INT, local_arr,
num_per_node-diff, MPI_INT, ROOT, MPI_COMM_WORLD);
// printf("[Rank %d] num_per_node_size = %d\n" ,rank, num_per_node);
MPI_Barrier(MPI_COMM_WORLD);
/*
for (int i = 0; i < num_per_node; ++i)
printf("[BEFORE] [Rank %d] local_arr[%d] = %d\n", rank, i, local_arr[i]);
*/
if (rank < rank_num) {
std::sort(local_arr, local_arr + num_per_node);
}
MPI_Barrier(MPI_COMM_WORLD);
/*
for (int i = 0; i < num_per_node; ++i)
printf("[AFTER] [Rank %d] local_arr[%d] = %d\n", rank, i, local_arr[i]);
*/
// printf("rank %d is arrived\n", rank);
MPI_Barrier(MPI_COMM_WORLD); // Wait for rank0 to read file
int *recv_buf, *send_buf;
int recv_len, send_len, success;
if (rank_num > 1 && rank < rank_num) {
if (rank == ROOT) {
send_len = num_per_node;
MPI_Send(&send_len, 1, MPI_INT, rank+1, 0, MPI_COMM_WORLD);
MPI_Recv(&success, 1, MPI_INT, rank+1,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
MPI_Send(local_arr, send_len, MPI_INT, rank+1, 0, MPI_COMM_WORLD);
} else {
MPI_Recv(&recv_len, 1, MPI_INT, rank-1,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
success = 1;
MPI_Send(&success, 1, MPI_INT, rank-1, 0, MPI_COMM_WORLD);
send_len = recv_len + num_per_node;
recv_buf = new int[recv_len];
send_buf = new int[send_len];
// printf("RANK %d recv_len = %d SUCCESS\n", rank, recv_len);
MPI_Recv(recv_buf, recv_len, MPI_INT,
rank-1, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
// printf("RANK %d complete recevice array SUCCESS\n", rank);
int i = 0, j = 0, cur = 0;
while (i < recv_len && j < num_per_node) {
// Do MERGE array
if (recv_buf[i] < local_arr[j]) {
send_buf[cur++] = recv_buf[i++];
} else {
send_buf[cur++] = local_arr[j++];
}
}
while (i < recv_len)
send_buf[cur++] = recv_buf[i++];
while (j < num_per_node)
send_buf[cur++] = local_arr[j++];
/*
for (int k = 0; k < cur; k++) {
printf("[RANK %d] send_buf[%d] = %d\n", rank, k, send_buf[k]);
}
*/
if(rank != LAST) {
MPI_Send(&send_len, 1, MPI_INT, rank+1, 0, MPI_COMM_WORLD);
// printf("RANK %d send_len SUCCESS\n", rank);
MPI_Recv(&success, 1, MPI_INT, rank+1,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
MPI_Send(send_buf, send_len, MPI_INT, rank+1, 0, MPI_COMM_WORLD);
// printf("RANK %d complete sending array SUCCESS\n", rank);
}
if(rank != LAST)
delete [] send_buf;
delete [] recv_buf;
}
}
// printf("rank %d is arrived\n", rank);
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_open(MPI_COMM_WORLD, argv[3],
MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fout);
if (rank == LAST) {
if (rank == 0) send_buf = local_arr;
MPI_File_write(fout, send_buf, N, MPI_INT, &status);
/*
for (int i = 0; i < N; ++i) {
printf("[FINAL] [Rank %d] ans[%d] = %d\n", rank, i, send_buf[i]);
}
*/
}
MPI_File_close(&fout);
// printf("CLOSE rank %d is arrived\n", rank);
MPI_Barrier(MPI_COMM_WORLD);
if (rank != 0) {
delete [] local_arr;
// printf("[FREE] [RANK %d] SUCCESS FREE\n", rank);
} else {
delete [] root_arr;
delete [] local_arr;;
}
MPI_Finalize();
return 0;
}
void *
IOR_Open_MPIIO(char * testFileName,
IOR_param_t * param)
{
int fd_mode = (int)0,
offsetFactor,
tasksPerFile,
transfersPerBlock = param->blockSize
/ param->transferSize;
struct fileTypeStruct {
int globalSizes[2],
localSizes[2],
startIndices[2];
} fileTypeStruct;
MPI_File * fd;
MPI_Comm comm;
MPI_Info mpiHints = MPI_INFO_NULL;
fd = (MPI_File *)malloc(sizeof(MPI_File));
if (fd == NULL) ERR("Unable to malloc MPI_File");
*fd = 0;
/* set IOR file flags to MPIIO flags */
/* -- file open flags -- */
if (param->openFlags & IOR_RDONLY) {fd_mode |= MPI_MODE_RDONLY;}
if (param->openFlags & IOR_WRONLY) {fd_mode |= MPI_MODE_WRONLY;}
if (param->openFlags & IOR_RDWR) {fd_mode |= MPI_MODE_RDWR;}
if (param->openFlags & IOR_APPEND) {fd_mode |= MPI_MODE_APPEND;}
if (param->openFlags & IOR_CREAT) {fd_mode |= MPI_MODE_CREATE;}
if (param->openFlags & IOR_EXCL) {fd_mode |= MPI_MODE_EXCL;}
if (param->openFlags & IOR_TRUNC) {
fprintf(stdout, "File truncation not implemented in MPIIO\n");
}
if (param->openFlags & IOR_DIRECT) {
fprintf(stdout, "O_DIRECT not implemented in MPIIO\n");
}
/*
* MPI_MODE_UNIQUE_OPEN mode optimization eliminates the overhead of file
* locking. Only open a file in this mode when the file will not be con-
* currently opened elsewhere, either inside or outside the MPI environment.
*/
fd_mode |= MPI_MODE_UNIQUE_OPEN;
if (param->filePerProc) {
comm = MPI_COMM_SELF;
} else {
comm = testComm;
}
SetHints(&mpiHints, param->hintsFileName);
/*
* note that with MP_HINTS_FILTERED=no, all key/value pairs will
* be in the info object. The info object that is attached to
* the file during MPI_File_open() will only contain those pairs
* deemed valid by the implementation.
*/
/* show hints passed to file */
if (rank == 0 && param->showHints) {
fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd),
"cannot open file");
/* show hints actually attached to file handle */
if (rank == 0 && param->showHints) {
MPI_CHECK(MPI_File_get_info(*fd, &mpiHints),
"cannot get file info");
fprintf(stdout, "\nhints returned from opened file {\n");
ShowHints(&mpiHints);
fprintf(stdout, "}\n");
}
/* preallocate space for file */
if (param->preallocate && param->open == WRITE) {
MPI_CHECK(MPI_File_preallocate(*fd,
(MPI_Offset)(param->segmentCount*param->blockSize*param->numTasks)),
"cannot preallocate file");
}
/* create file view */
if (param->useFileView) {
/* create contiguous transfer datatype */
MPI_CHECK(MPI_Type_contiguous(param->transferSize / sizeof(IOR_size_t),
MPI_LONG_LONG_INT, ¶m->transferType),
"cannot create contiguous datatype");
MPI_CHECK(MPI_Type_commit(¶m->transferType),
"cannot commit datatype");
if (param->filePerProc) {
offsetFactor = 0;
tasksPerFile = 1;
} else {
offsetFactor = (rank + rankOffset) % param->numTasks;
tasksPerFile = param->numTasks;
}
/*
* create file type using subarray
*/
fileTypeStruct.globalSizes[0] = 1;
fileTypeStruct.globalSizes[1] = transfersPerBlock * tasksPerFile;
fileTypeStruct.localSizes[0] = 1;
fileTypeStruct.localSizes[1] = transfersPerBlock;
fileTypeStruct.startIndices[0] = 0;
fileTypeStruct.startIndices[1] = transfersPerBlock * offsetFactor;
MPI_CHECK(MPI_Type_create_subarray(2, fileTypeStruct.globalSizes,
fileTypeStruct.localSizes,
fileTypeStruct.startIndices,
MPI_ORDER_C, param->transferType,
¶m->fileType),
"cannot create subarray");
MPI_CHECK(MPI_Type_commit(¶m->fileType), "cannot commit datatype");
MPI_CHECK(MPI_File_set_view(*fd, (MPI_Offset)0,
param->transferType, param->fileType,
"native", (MPI_Info)MPI_INFO_NULL),
"cannot set file view");
}
return((void *)fd);
} /* IOR_Open_MPIIO() */
int PIDX_aggregated_io(PIDX_file_io_id io_id, Agg_buffer agg_buf, PIDX_block_layout block_layout, int MODE)
{
int64_t data_offset = 0;
char file_name[PATH_MAX];
int i = 0, k = 0;
uint32_t *headers;
int total_header_size = 0;
#ifdef PIDX_RECORD_TIME
double t1, t2, t3, t4, t5;
#endif
#if PIDX_HAVE_MPI
int mpi_ret;
MPI_File fh;
MPI_Status status;
#else
int fh;
#endif
int total_chunk_size = (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]);
if (enable_caching == 1 && agg_buf->var_number == io_id->init_index && agg_buf->sample_number == 0)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) /*agg_buf->file_number*/adjusted_file_index, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#else
fh = open(file_name, O_WRONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = (uint32_t*)malloc(total_header_size);
memset(headers, 0, total_header_size);
#if !SIMULATE_IO
if (enable_caching == 1)
memcpy (headers, cached_header_copy, total_header_size);
else
{
//TODO
}
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
uint64_t header_size = (io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size);
#if !SIMULATE_IO
unsigned char* temp_buffer = (unsigned char*)realloc(agg_buf->buffer, agg_buf->buffer_size + header_size);
if (temp_buffer == NULL)
{
fprintf(stderr, "[%s] [%d] realloc() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
else
{
agg_buf->buffer = temp_buffer;
memmove(agg_buf->buffer + header_size, agg_buf->buffer, agg_buf->buffer_size);
memcpy(agg_buf->buffer, headers, total_header_size);
memset(agg_buf->buffer + total_header_size, 0, (header_size - total_header_size));
}
#endif
free(headers);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_write_at(fh, 0, agg_buf->buffer, agg_buf->buffer_size + header_size, MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed for filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size + header_size, 0);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t5 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V0. [R %d] [O 0 C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long)agg_buf->buffer_size + header_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t3-t2), (t4-t3), (t5-t4));
#else
#endif
}
else if (agg_buf->var_number != -1 && agg_buf->sample_number != -1 && agg_buf->file_number != -1)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) adjusted_file_index/*agg_buf->file_number*/, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
else
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
#else
if (MODE == PIDX_WRITE)
fh = open(file_name, O_WRONLY);
else
fh = open(file_name, O_RDONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
if (MODE == PIDX_WRITE)
{
for (k = 0; k < agg_buf->var_number; k++)
{
PIDX_variable vark = io_id->idx->variable[k];
int bytes_per_datatype = ((vark->bits_per_value/8) * total_chunk_size) / (io_id->idx->compression_factor);
int64_t prev_var_sample = (int64_t) block_layout->block_count_per_file[agg_buf->file_number] * io_id->idx_d->samples_per_block * bytes_per_datatype * io_id->idx->variable[k]->values_per_sample;
data_offset = (int64_t) data_offset + prev_var_sample;
}
for (i = 0; i < agg_buf->sample_number; i++)
data_offset = (int64_t) data_offset + agg_buf->buffer_size;
}
else
{
int total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = malloc(total_header_size);
memset(headers, 0, total_header_size);
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_read_at(fh, 0, headers, total_header_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#endif
}
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
//int rank;
//MPI_Comm_rank(io_id->comm, &rank);
//printf("W [%d] [%d %d %d] size = %d and offset = %d\n", rank, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, agg_buf->buffer_size, data_offset);
mpi_ret = MPI_File_write_at(fh, data_offset, agg_buf->buffer, agg_buf->buffer_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count = 0;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
int data_size = 0;
int block_count = 0;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
{
if (PIDX_blocks_is_block_present(agg_buf->file_number * io_id->idx->blocks_per_file + i, block_layout))
{
data_offset = htonl(headers[12 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
data_size = htonl(headers[14 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
mpi_ret = MPI_File_read_at(fh, data_offset, agg_buf->buffer + (block_count * io_id->idx_d->samples_per_block * (io_id->idx->variable[agg_buf->var_number]->bits_per_value/8) * io_id->idx->variable[agg_buf->var_number]->values_per_sample * io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2]) / io_id->idx->compression_factor, /*agg_buf->buffer_size*/data_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int read_count = 0;
MPI_Get_count(&status, MPI_BYTE, &read_count);
if (read_count != /*agg_buf->buffer_size*/data_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed. %d != %lldd\n", __FILE__, __LINE__, read_count, (long long)agg_buf->buffer_size);
return PIDX_err_io;
}
block_count++;
}
}
free(headers);
}
#else
if (MODE == PIDX_WRITE)
{
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
ssize_t read_count = pread(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (read_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pread() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V. [R %d] [O %lld C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long) data_offset, (long long)agg_buf->buffer_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t2-t2), (t3-t2), (t4-t3));
#endif
}
return PIDX_success;
}
static void writePLY(
MPI_Comm comm, std::string fname,
int nvertices, int nverticesPerObject,
int ntriangles, int ntrianglesPerObject,
int nObjects,
const std::vector<int3>& mesh,
const std::vector<float3>& vertices)
{
int rank;
MPI_Check( MPI_Comm_rank(comm, &rank) );
int totalVerts = 0;
MPI_Check( MPI_Reduce(&nvertices, &totalVerts, 1, MPI_INT, MPI_SUM, 0, comm) );
int totalTriangles = 0;
MPI_Check( MPI_Reduce(&ntriangles, &totalTriangles, 1, MPI_INT, MPI_SUM, 0, comm) );
MPI_File f;
MPI_Check( MPI_File_open(comm, fname.c_str(), MPI_MODE_CREATE|MPI_MODE_DELETE_ON_CLOSE|MPI_MODE_WRONLY, MPI_INFO_NULL, &f) );
MPI_Check( MPI_File_close(&f) );
MPI_Check( MPI_File_open(comm, fname.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &f) );
int headerSize = 0;
MPI_Offset fileOffset = 0;
if (rank == 0)
{
std::stringstream ss;
ss << "ply\n";
ss << "format binary_little_endian 1.0\n";
ss << "element vertex " << totalVerts << "\n";
ss << "property float x\nproperty float y\nproperty float z\n";
//ss << "property float xnormal\nproperty float ynormal\nproperty float znormal\n";
ss << "element face " << totalTriangles << "\n";
ss << "property list int int vertex_index\n";
ss << "end_header\n";
std::string content = ss.str();
headerSize = content.length();
MPI_Check( MPI_File_write_at(f, fileOffset, content.c_str(), headerSize, MPI_CHAR, MPI_STATUS_IGNORE) );
}
MPI_Check( MPI_Bcast(&headerSize, 1, MPI_INT, 0, comm) );
fileOffset += headerSize;
fileOffset += writeToMPI(vertices, f, fileOffset, comm);
int verticesOffset = 0;
MPI_Check( MPI_Exscan(&nvertices, &verticesOffset, 1, MPI_INT, MPI_SUM, comm));
std::vector<int4> connectivity;
for(int j = 0; j < nObjects; ++j)
for(int i = 0; i < ntrianglesPerObject; ++i)
{
int3 vertIds = mesh[i] + nverticesPerObject * j + verticesOffset;
connectivity.push_back({3, vertIds.x, vertIds.y, vertIds.z});
}
fileOffset += writeToMPI(connectivity, f, fileOffset, comm);
MPI_Check( MPI_File_close(&f));
}
int ChimeraCheckCommand::execute(){
try{
if (abort == true) { if (calledHelp) { return 0; } return 2; }
for (int i = 0; i < fastaFileNames.size(); i++) {
m->mothurOut("Checking sequences from " + fastaFileNames[i] + " ..." ); m->mothurOutEndLine();
int start = time(NULL);
string thisNameFile = "";
if (nameFileNames.size() != 0) { thisNameFile = nameFileNames[i]; }
chimera = new ChimeraCheckRDP(fastaFileNames[i], templatefile, thisNameFile, svg, increment, ksize, outputDir);
if (m->control_pressed) { delete chimera; return 0; }
if (outputDir == "") { outputDir = m->hasPath(fastaFileNames[i]); }//if user entered a file with a path then preserve it
map<string, string> variables;
variables["[filename]"] = outputDir + m->getRootName(m->getSimpleName(fastaFileNames[i]));
string outputFileName = getOutputFileName("chimera", variables);
outputNames.push_back(outputFileName); outputTypes["chimera"].push_back(outputFileName);
#ifdef USE_MPI
int pid, numSeqsPerProcessor;
int tag = 2001;
vector<unsigned long long> MPIPos;
MPI_Status status;
MPI_Comm_rank(MPI_COMM_WORLD, &pid); //find out who we are
MPI_Comm_size(MPI_COMM_WORLD, &processors);
MPI_File inMPI;
MPI_File outMPI;
int outMode=MPI_MODE_CREATE|MPI_MODE_WRONLY;
int inMode=MPI_MODE_RDONLY;
char outFilename[1024];
strcpy(outFilename, outputFileName.c_str());
char inFileName[1024];
strcpy(inFileName, fastaFileNames[i].c_str());
MPI_File_open(MPI_COMM_WORLD, inFileName, inMode, MPI_INFO_NULL, &inMPI); //comm, filename, mode, info, filepointer
MPI_File_open(MPI_COMM_WORLD, outFilename, outMode, MPI_INFO_NULL, &outMPI);
if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
if (pid == 0) { //you are the root process
MPIPos = m->setFilePosFasta(fastaFileNames[i], numSeqs); //fills MPIPos, returns numSeqs
//send file positions to all processes
for(int j = 1; j < processors; j++) {
MPI_Send(&numSeqs, 1, MPI_INT, j, tag, MPI_COMM_WORLD);
MPI_Send(&MPIPos[0], (numSeqs+1), MPI_LONG, j, tag, MPI_COMM_WORLD);
}
//figure out how many sequences you have to align
numSeqsPerProcessor = numSeqs / processors;
int startIndex = pid * numSeqsPerProcessor;
if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; }
//align your part
driverMPI(startIndex, numSeqsPerProcessor, inMPI, outMPI, MPIPos);
if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
//wait on chidren
for(int j = 1; j < processors; j++) {
char buf[5];
MPI_Recv(buf, 5, MPI_CHAR, j, tag, MPI_COMM_WORLD, &status);
}
}else{ //you are a child process
MPI_Recv(&numSeqs, 1, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
MPIPos.resize(numSeqs+1);
MPI_Recv(&MPIPos[0], (numSeqs+1), MPI_LONG, 0, tag, MPI_COMM_WORLD, &status);
//figure out how many sequences you have to align
numSeqsPerProcessor = numSeqs / processors;
int startIndex = pid * numSeqsPerProcessor;
if(pid == (processors - 1)){ numSeqsPerProcessor = numSeqs - pid * numSeqsPerProcessor; }
//align your part
driverMPI(startIndex, numSeqsPerProcessor, inMPI, outMPI, MPIPos);
if (m->control_pressed) { MPI_File_close(&inMPI); MPI_File_close(&outMPI); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
//tell parent you are done.
char buf[5];
strcpy(buf, "done");
MPI_Send(buf, 5, MPI_CHAR, 0, tag, MPI_COMM_WORLD);
}
//close files
MPI_File_close(&inMPI);
MPI_File_close(&outMPI);
MPI_Barrier(MPI_COMM_WORLD); //make everyone wait - just in case
#else
//break up file
#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux) || (__linux__) || (__unix__) || (__unix)
vector<unsigned long long> positions = m->divideFile(fastaFileNames[i], processors);
for (int s = 0; s < (positions.size()-1); s++) {
lines.push_back(new linePair(positions[s], positions[(s+1)]));
}
if(processors == 1){
numSeqs = driver(lines[0], outputFileName, fastaFileNames[i]);
if (m->control_pressed) { for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } for (int j = 0; j < lines.size(); j++) { delete lines[j]; } outputTypes.clear(); lines.clear(); delete chimera; return 0; }
}else{
processIDS.resize(0);
numSeqs = createProcesses(outputFileName, fastaFileNames[i]);
rename((outputFileName + toString(processIDS[0]) + ".temp").c_str(), outputFileName.c_str());
//append output files
for(int j=1;j<processors;j++){
m->appendFiles((outputFileName + toString(processIDS[j]) + ".temp"), outputFileName);
m->mothurRemove((outputFileName + toString(processIDS[j]) + ".temp"));
}
if (m->control_pressed) {
for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear();
for (int j = 0; j < lines.size(); j++) { delete lines[j]; } lines.clear();
delete chimera;
return 0;
}
}
#else
lines.push_back(new linePair(0, 1000));
numSeqs = driver(lines[0], outputFileName, fastaFileNames[i]);
if (m->control_pressed) { for (int j = 0; j < lines.size(); j++) { delete lines[j]; } lines.clear(); for (int j = 0; j < outputNames.size(); j++) { m->mothurRemove(outputNames[j]); } outputTypes.clear(); delete chimera; return 0; }
#endif
#endif
delete chimera;
for (int j = 0; j < lines.size(); j++) { delete lines[j]; } lines.clear();
m->mothurOutEndLine(); m->mothurOut("This method does not determine if a sequence is chimeric, but allows you to make that determination based on the IS values."); m->mothurOutEndLine();
m->mothurOutEndLine(); m->mothurOut("It took " + toString(time(NULL) - start) + " secs to check " + toString(numSeqs) + " sequences."); m->mothurOutEndLine(); m->mothurOutEndLine();
}
m->mothurOutEndLine();
m->mothurOut("Output File Names: "); m->mothurOutEndLine();
for (int i = 0; i < outputNames.size(); i++) { m->mothurOut(outputNames[i]); m->mothurOutEndLine(); }
m->mothurOutEndLine();
return 0;
}
catch(exception& e) {
m->errorOut(e, "ChimeraCheckCommand", "execute");
exit(1);
}
}
static int write_read_samples(PIDX_file_io_id io_id, int variable_index, uint64_t hz_start_index, uint64_t hz_count, unsigned char* hz_buffer, int64_t buffer_offset, PIDX_block_layout layout, int MODE)
{
int samples_per_file, block_number, file_index, file_count, ret = 0, block_negative_offset = 0, file_number;
int bytes_per_sample, bytes_per_datatype;
int i = 0;
char file_name[PATH_MAX];
off_t data_offset = 0;
samples_per_file = io_id->idx_d->samples_per_block * io_id->idx->blocks_per_file;
bytes_per_datatype = (io_id->idx->variable[variable_index]->bits_per_value / 8) * (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]) / (io_id->idx->compression_factor);
#if !SIMULATE_IO
hz_buffer = hz_buffer + buffer_offset * bytes_per_datatype * io_id->idx->variable[variable_index]->values_per_sample;
#endif
while (hz_count)
{
block_number = hz_start_index / io_id->idx_d->samples_per_block;
file_number = hz_start_index / samples_per_file;
file_index = hz_start_index % samples_per_file;
file_count = samples_per_file - file_index;
if ((int64_t)file_count > hz_count)
file_count = hz_count;
// build file name
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
ret = generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, /*file_number*/adjusted_file_index, file_name, PATH_MAX);
if (ret == 1)
{
fprintf(stderr, "[%s] [%d] generate_file_name() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
data_offset = 0;
bytes_per_sample = io_id->idx->variable[variable_index]->bits_per_value / 8;
data_offset = file_index * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
block_negative_offset = PIDX_blocks_find_negative_offset(io_id->idx->blocks_per_file, block_number, layout);
data_offset -= block_negative_offset * io_id->idx_d->samples_per_block * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample;
for (l = 0; l < variable_index; l++)
{
bytes_per_sample = io_id->idx->variable[l]->bits_per_value / 8;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
if (PIDX_blocks_is_block_present((i + (io_id->idx->blocks_per_file * file_number)), layout))
data_offset = data_offset + (io_id->idx->variable[l]->values_per_sample * bytes_per_sample * io_id->idx_d->samples_per_block);
}
if(MODE == PIDX_WRITE)
{
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
#ifdef PIDX_DUMP_IO
if (io_id->idx_d->dump_io_info == 1 && io_id->idx->current_time_step == 0)
{
fprintf(io_dump_fp, "[A] Count %lld Target Disp %d (%d %d)\n", (long long)file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), (file_index * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample - block_negative_offset * io_id->idx_d->samples_per_block * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample)/8, (int)io_id->idx_d->start_fs_block, (int)io_id->idx_d->fs_block_size);
fflush(io_dump_fp);
}
#endif
if (io_id->idx_d->parallel_mode == 1)
{
int rank = 0;
MPI_Comm_rank(io_id->comm, &rank);
MPI_File fh;
MPI_Status status;
int mpi_ret;
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed. (%s) [%d]\n", __FILE__, __LINE__, file_name, file_number);
return PIDX_err_io;
}
/*
printf("[%d] Data Offset %d Count %d\n", rank, data_offset, (file_count));
int x = 0;
for (x = 0; x < file_count; x++)
{
double x1;
memcpy(&x1, hz_buffer + x * sizeof(double), sizeof(double));
printf("Values %d %f\n", x, x1);
}
*/
mpi_ret = MPI_File_write_at(fh, data_offset, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
MPI_File_close(&fh);
}
else
{
int fh;
fh = open(file_name, O_WRONLY);
ssize_t write_count = pwrite(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
}
#else
int fh;
fh = open(file_name, O_WRONLY);
/*
double x1, x2, x3, x4;
memcpy(&x1, hz_buffer, sizeof(double));
memcpy(&x2, hz_buffer + sizeof(double), sizeof(double));
memcpy(&x3, hz_buffer + 2*sizeof(double), sizeof(double));
memcpy(&x4, hz_buffer + 3*sizeof(double), sizeof(double));
printf("[%d] [%d %d] Values %f %f %f %f\n", variable_index, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset, x1, x2, x3, x4);
*/
ssize_t write_count = pwrite(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
#endif
#endif
}
if(MODE == PIDX_READ)
{
#if PIDX_HAVE_MPI
if (io_id->idx_d->parallel_mode == 1)
{
MPI_File fh;
MPI_Status status;
int mpi_ret;
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
mpi_ret = MPI_File_read_at(fh, data_offset, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
MPI_File_close(&fh);
}
else
{
int fh;
fh = open(file_name, O_RDONLY);
ssize_t read_count = pread(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (read_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
}
#else
int fh;
fh = open(file_name, O_RDONLY);
ssize_t read_count = pread(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (read_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
#endif
}
hz_count -= file_count;
hz_start_index += file_count;
hz_buffer += file_count * io_id->idx->variable[variable_index]->values_per_sample * bytes_per_datatype;
#if PIDX_HAVE_MPI
#else
#endif
}
return PIDX_success;
}
int main(int argc, char **argv) {
int rank;
int size;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
printf("Moin. I'm process %d of %d processes.\n", rank, size);
//FILE *ifile = stdin;
//MPI_File ifile = stdin;
MPI_File ifile;
FILE *ofile = stdout;
int c, spok = 0, aopt = 0;
unsigned char *bopt = NULL, *iopt = NULL, *oopt = NULL;
unsigned char *topt = NULL, *sopt = NULL, *popt = NULL;
unsigned char *Wopt = NULL;
while ((c = getopt(argc, argv, "ab:hi:o:p:s:t:W:")) != -1) {
switch (c) {
case 'a':
aopt = 1; // open output file in append mode
break;
case 'b':
bopt = optarg; // bloom filter file
break;
case 'i':
iopt = optarg; // input file
break;
case 'o':
oopt = optarg; // output file
break;
case 's':
sopt = optarg; // salt
break;
case 'p':
popt = optarg; // passphrase
break;
case 't':
topt = optarg; // type of input
break;
case 'W':
Wopt = optarg;
break;
case 'h':
// show help
usage(argv[0]);
return 0;
case '?':
// show error
return 1;
default:
// should never be reached...
printf("got option '%c' (%d)\n", c, c);
return 1;
}
}
if (optind < argc) {
if (optind == 1 && argc == 2) {
// older versions of brainflayer had the bloom filter file as a
// single optional argument, this keeps compatibility with that
bopt = argv[1];
} else {
fprintf(stderr, "Invalid arguments:\n");
while (optind < argc) {
fprintf(stderr, " '%s'\n", argv[optind++]);
}
exit(1);
}
}
if (topt != NULL) {
if (strcmp(topt, "str") == 0) {
input2hash160 = &pass2hash160;
} else if (strcmp(topt, "hex") == 0) {
input2hash160 = &hexpass2hash160;
} else if (strcmp(topt, "priv") == 0) {
input2hash160 = &hexpriv2hash160;
} else if (strcmp(topt, "warp") == 0) {
spok = 1;
input2hash160 = popt ? &warpsalt2hash160 : &warppass2hash160;
} else if (strcmp(topt, "bwio") == 0) {
spok = 1;
if (popt && strcmp(popt, "hello world") == 0
&& (Wopt == NULL
|| strcmp(Wopt, "NEVER_TELL_ME_THE_ODDS") != 0)) {
// https://www.youtube.com/watch?v=NHWjlCaIrQo
for (c = 0; c < 100; ++c)
fprintf(stderr, "\n"); // try not to clobber scrollback
fprintf(stderr, "\033[2J\033[0;0H\033[0;0f"); // clear terminal and send cursor to top left
fflush(stderr);
sleep(1);
fprintf(stderr, "A STRANGE GAME.\n");
sleep(2);
fprintf(stderr, "THE ONLY WINNING MOVE IS NOT TO PLAY.\n");
sleep(2);
fprintf(stderr,
"\n"
"So, you're looking for that sweet, sweet 0.5BTC bounty? Brainflayer's\n"
"cracking speed against brainwallet.io had been communicated to them before\n"
"the challange was created. It is likely that the salt was chosen to be\n"
"infeasible to crack in the given timeframe by a significant margin. I advise\n"
"against trying it - it's probably a waste time and money. If you want to do it\n"
"anyway, run with `-W NEVER_TELL_ME_THE_ODDS`.\n");
sleep(2);
fprintf(stderr,
"\nAs for me, I have better things to do with my CPU cycles.\n");
sleep(3);
bail(83, "CONNECTION TERMINATED\n");
}
input2hash160 = popt ? &bwiosalt2hash160 : &bwiopass2hash160;
} else if (strcmp(topt, "bv2") == 0) {
spok = 1;
input2hash160 = popt ? &brainv2salt2hash160 : &brainv2pass2hash160;
} else {
bail(1, "Unknown input type '%s'.\n", topt);
}
} else {
topt = "str";
input2hash160 = &pass2hash160;
}
if (spok) {
if (sopt && popt) {
bail(1, "Cannot specify both a salt and a passphrase\n");
}
if (popt) {
kdfpass = popt;
kdfpass_sz = strlen(popt);
} else {
if (sopt) {
kdfsalt = sopt;
kdfsalt_sz = strlen(kdfsalt);
} else {
kdfsalt = malloc(0);
kdfsalt_sz = 0;
}
}
} else {
if (popt) {
bail(1,
"Specifying a passphrase not supported with input type '%s'\n",
topt);
} else if (sopt) {
bail(1,
"Specifying a salt not supported with this input type '%s'\n",
topt);
}
}
if (bopt) {
if ((bloom = bloom_open(bopt)) == NULL) {
bail(1, "failed to open bloom filter.\n");
}
}
if (iopt) {
if (MPI_File_open(MPI_COMM_WORLD, iopt, MPI_MODE_RDONLY, MPI_INFO_NULL,
&ifile)) {
bail(1, "failed to open '%s' for reading: %s\n", iopt,
strerror(errno));
}
}
if (oopt) {
if ((ofile = fopen(oopt, (aopt ? "a" : "w"))) == NULL) {
bail(1, "failed to open '%s' for writing: %s\n", oopt,
strerror(errno));
}
}
/* use line buffered output */
setvbuf(ofile, NULL, _IOLBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
secp256k1_start();
const int overlap = 100;
char **lines;
int nlines;
readlines(&ifile, rank, size, overlap, &lines, &nlines);
fprintf(ofile, "----Welcome %d! %d Lines for you.----\n", rank, nlines);
int index = 0;
time_t start, end;
double length;
time(&start);
for (int i = 0; i < nlines - 1; i++) {
++index;
input2hash160(lines[i], strlen(lines[i]));
if (bloom) {
if (bloom_chk_hash160(bloom, hash160_uncmp.ul)) {
fprintresult(ofile, &hash160_uncmp, 'u', topt, lines[i]);
}
if (bloom_chk_hash160(bloom, hash160_compr.ul)) {
fprintresult(ofile, &hash160_compr, 'c', topt, lines[i]);
}
} else {
fprintresult(ofile, &hash160_uncmp, 'u', topt, lines[i]);
fprintresult(ofile, &hash160_compr, 'c', topt, lines[i]);
}
}
time(&end);
length = difftime(end, start);
double perSecond = index / length;
fprintf(ofile, "----Process: %d, Lines: %d, speed: %.0f/sec!----\n", rank, index, perSecond);
secp256k1_stop();
MPI_File_close(&ifile);
MPI_Finalize();
return 0;
}
int main (int argc, char **argv)
{
struct arguments arguments;
/* Parse our arguments; every option seen by parse_opt will
be reflected in arguments. */
argp_parse (&argp, argc, argv, 0, 0, &arguments);
int run_type;
run_type = 0; //default is serial
if (sscanf (arguments.args[0], "%i", &run_type)!=1) {}
int iterations;
iterations = 0; //default is serial
if (sscanf (arguments.args[1], "%i", &iterations)!=1) {}
int count_when;
count_when = 1000;
if (sscanf (arguments.args[2], "%i", &count_when)!=1) {}
char print_list[200]; //used for input list
if (sscanf (arguments.args[3], "%s", &print_list)!=1) {}
// printf("Print list = %s\n", print_list);
//Extract animation list from arguments
char char_array[20][12] = { NULL }; //seperated input list
int animation_list[20][2] = { NULL }; //integer input list start,range
char *tok = strtok(print_list, ",");
//counters
int i,j,k,x,y,ii,jj;
ii = 0;
jj = 0;
//Loop over tokens parsing our commas
int tok_len = 0;
while (tok != NULL)
{
//first loop parses out commas
tok_len = strlen(tok);
for (jj=0;jj<tok_len;jj++)
{
char_array[ii][jj] = tok[jj];
}
// printf("Tok = %s\n", char_array[ii]);
tok = strtok(NULL, ",");
ii++;
}
//looking for a range input, convert to ints
int stop;
for (ii=0;ii<20;ii++)
{
//convert first number to int
tok = strtok(char_array[ii], "-");
if (tok != NULL)
{
animation_list[ii][0] = atoi(tok);
tok = strtok(NULL, ",");
}
//look for second number, add to range
if (tok != NULL)
{
stop = atoi(tok);
animation_list[ii][1] = stop - animation_list[ii][0];
}
// if (rank == 0)
// {
// printf("Animation_list = %i, %i\n",
// animation_list[ii][0], animation_list[ii][1]);
// }
}
//should an animation be generated
//prints a bunch of .pgm files, have to hand
//make the gif...
int animation;
animation = arguments.animation;
//verbose?
int verbose;
verbose = arguments.verbose;
// printf("VERBOSE = %i",verbose);
if (verbose>0 && verbose<=10)
{
verbose = 1;
}
// Initialize the MPI environment
MPI_Init(NULL, NULL);
// Get the number of processes
int world_size;
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
// Get the rank of the process
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// Get the name of the processor
char processor_name[MPI_MAX_PROCESSOR_NAME];
int name_len;
MPI_Get_processor_name(processor_name, &name_len);
//Print run information, exit on bad command line input
if (rank == 0 && verbose == 1)
{
printf("Verbose=%i, RunType=%i, Iterations=%i, CountWhen=%i, Animation=%i\n",
verbose,run_type,iterations,count_when, animation);
}
if (world_size>1 && run_type ==0)
{
printf("Runtype and processors count not consistant\n");
MPI_Finalize();
exit(0);
}
if (world_size==1 && run_type>0)
{
printf("Runtype and processors count not consistant\n");
MPI_Finalize();
exit(0);
}
if (count_when <= 0)
{
if (rank == 0)
{
printf("Invalid count interval, positive integers only\n");
}
MPI_Finalize();
exit(0);
}
//serial
if (world_size == 1 && run_type == 0)
{
ncols=1;
nrows=1;
}
//Blocked
else if (world_size>1 && run_type == 1)
{
ncols = 1;
nrows = world_size;
my_col = 0;
my_row = rank;
}
//Checker
else if (world_size>1 && run_type == 2)
{
ncols = (int)sqrt(world_size);
nrows = (int)sqrt(world_size);
my_row = rank/nrows;
my_col = rank-my_row*nrows;
if (ncols*nrows!=world_size)
{
if (rank == 0)
{
printf("Number of processors must be square, Exiting\n");
}
MPI_Finalize();
exit(0);
}
}
// if (verbose == 1)
// {
// printf("WR,row,col=%i,%i,%i\n",rank,my_row,my_col);
// }
//////////////////////READ IN INITIAL PGM////////////////////////////////
if(!readpgm("life.pgm"))
{
// printf("WR=%d,HERE2\n",rank);
if( rank==0 )
{
pprintf( "An error occured while reading the pgm file\n" );
}
MPI_Finalize();
return 1;
}
// Count the life forms. Note that we count from [1,1] - [height+1,width+1];
// we need to ignore the ghost row!
i = 0;
for(y=1; y<local_height+1; y++ )
{
for(x=1; x<local_width+1; x++ )
{
if( field_a[ y * field_width + x ] )
{
i++;
}
}
}
// pprintf( "%i local buggies\n", i );
int total;
MPI_Allreduce( &i, &total, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if( rank==0 && verbose == 1 )
{
pprintf( "%i total buggies\n", total );
}
// printf("WR=%d, Row=%d, Col=%d\n",rank,my_row,my_col);
//Row and column size per processor
int rsize, csize;
rsize = local_width;
csize = local_height;
if (rank == 0 && verbose == 1)
{
printf("rsize,csize,NP = %d, %d, %d\n",rsize,csize,world_size);
}
//Create new derived datatype for writing to files
MPI_Datatype submatrix;
int array_of_gsizes[2];
int array_of_distribs[2];
int array_of_dargs[2];
int array_of_psize[2];
if (run_type == 1)
{
if (rank == 0)
{
printf("g0,g1 = %i,%i\n", local_height*ncols, local_width);
printf("p0,p1 = %i,%i\n", nrows, ncols);
}
array_of_gsizes[0] = local_height*ncols;
array_of_gsizes[1] = local_width;
array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_psize[0] = nrows;
array_of_psize[1] = ncols;
// int order = MPI_ORDER_C;
//size,rank,ndims,array_gsizes,array_distribs,array_args,array_psizes
//order,oldtype,*newtype
MPI_Type_create_darray(world_size, rank, 2, array_of_gsizes, array_of_distribs,
array_of_dargs, array_of_psize, MPI_ORDER_C, MPI_UNSIGNED_CHAR, &submatrix);
MPI_Type_commit(&submatrix);
}
else if (run_type == 2)
{
if (rank == 0)
{
printf("g0,g1 = %i,%i\n", local_height*ncols, local_width*nrows);
printf("p0,p1 = %i,%i\n", nrows, ncols);
}
array_of_gsizes[0] = local_height*ncols;
array_of_gsizes[1] = local_width*nrows;
array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_psize[0] = nrows;
array_of_psize[1] = ncols;
// int order = MPI_ORDER_C;
//size,rank,ndims,array_gsizes,array_distribs,array_args,array_psizes
//order,oldtype,*newtype
MPI_Type_create_darray(world_size, rank, 2, array_of_gsizes, array_of_distribs,
array_of_dargs, array_of_psize, MPI_ORDER_C, MPI_UNSIGNED_CHAR, &submatrix);
MPI_Type_commit(&submatrix);
}
MPI_Barrier(MPI_COMM_WORLD);
//////////////////ALLOCATE ARRAYS, CREATE DATATYPES/////////////////////
//Create new column derived datatype
MPI_Datatype column;
//count, blocklength, stride, oldtype, *newtype
MPI_Type_hvector(csize, 1, sizeof(unsigned char), MPI_UNSIGNED_CHAR, &column);
MPI_Type_commit(&column);
//Create new row derived datatype
MPI_Datatype row;
//count, blocklength, stride, oldtype, *newtype
MPI_Type_hvector(rsize, 1, sizeof(unsigned char), MPI_UNSIGNED_CHAR, &row);
MPI_Type_commit(&row);
//allocate arrays and corner storage
unsigned char *section;
unsigned char *neighbors;
//to use
unsigned char *top;
unsigned char *bot;
unsigned char *left;
unsigned char *right;
//to send
unsigned char *ttop;
unsigned char *tbot;
unsigned char *tleft;
unsigned char *tright;
//MALLOC!!
section = (unsigned char*)malloc(rsize*csize*sizeof(unsigned char));
neighbors = (unsigned char*)malloc(rsize*csize*sizeof(unsigned char));
top = (unsigned char*)malloc(rsize*sizeof(unsigned char));
bot = (unsigned char*)malloc(rsize*sizeof(unsigned char));
left = (unsigned char*)malloc(csize*sizeof(unsigned char));
right = (unsigned char*)malloc(csize*sizeof(unsigned char));
ttop = (unsigned char*)malloc(rsize*sizeof(unsigned char));
tbot = (unsigned char*)malloc(rsize*sizeof(unsigned char));
tleft = (unsigned char*)malloc(csize*sizeof(unsigned char));
tright = (unsigned char*)malloc(csize*sizeof(unsigned char));
//corners
unsigned char topleft,topright,botleft,botright; //used in calculations
unsigned char ttopleft,ttopright,tbotleft,tbotright;
topleft = 255;
topright = 255;
botleft = 255;
botright = 255;
//used for animation, each process will put there own result in and then
//each will send to process 1 which will add them up
unsigned char* full_matrix;
unsigned char* full_matrix_buffer;
if (animation == 1)
{
int msize1 = rsize*ncols*csize*nrows;
full_matrix = (unsigned char*)malloc(msize1*sizeof(unsigned char));
full_matrix_buffer = (unsigned char*)malloc(msize1*sizeof(unsigned char));
for (i=0; i<msize1; i++)
{
full_matrix[i] = 0;
full_matrix_buffer[i] = 0;
}
}
// printf("Rsize,Lsize,Fsize=%i %i %i,Csize,Lsize,Fsize=%i %i %i\n",rsize,local_width,field_width,csize,local_height,field_height);
//Serial initialize vars
int count = 0;
if (world_size == 1 && run_type == 0)
{
for (i=0;i<csize;i++)
{
for (j=0;j<rsize;j++)
{
section[i*rsize + j] = 255;
if (field_a[(i+1)*(2+rsize) + j + 1])
{
section[i*rsize + j] = 0;
count += 1;
}
else
{
section[i*rsize + j] = 255;
}
top[j] = 255;
bot[j] = 255;
ttop[j] = 255;
tbot[j] = 255;
}
right[i] = 255;
left[i] = 255;
tright[i] = 255;
tleft[i] = 255;
}
// printf("COUNT 4 = %d\n", count);
}
//Blocked/Checkered initializing variables
else if (world_size > 1 && (run_type == 1 || run_type == 2))
{
//initialize
for (i=0;i<csize;i++)
{
for (j=0;j<rsize;j++)
{
section[i*rsize + j] = 255;
if (field_a[(i+1)*(2+rsize) + j + 1])
{
section[i*rsize + j] = 0;
count += 1;
}
else
{
section[i*rsize + j] = 255;
}
top[j] = 255;
bot[j] = 255;
ttop[j] = 255;
tbot[j] = 255;
}
right[i] = 255;
left[i] = 255;
tright[i] = 255;
tleft[i] = 255;
}
// MPI_Allreduce( &count, &total, 1, MPI_UNSIGNED_CHAR, MPI_SUM, MPI_COMM_WORLD );
// if (rank == 0)
// {
// printf("COUNT 4 = %d\n", total);
// }
}
//header/footer for mpio writes
char header1[15];
header1[0] = 0x50;
header1[1] = 0x35;
header1[2] = 0x0a;
header1[3] = 0x35;
header1[4] = 0x31;
header1[5] = 0x32;
header1[6] = 0x20;
header1[7] = 0x35;
header1[8] = 0x31;
header1[9] = 0x32;
header1[10] = 0x0a;
header1[11] = 0x32;
header1[12] = 0x35;
header1[13] = 0x35;
header1[14] = 0x0a;
char footer;
footer = 0x0a;
//make a frame or not?
int create_frame = 0;
//send to
int send_to;
int receive_from;
int info[5];
info[2] = rank;
info[3] = rsize;
info[4] = csize;
unsigned char info2[4];
info2[0] = topleft;
info2[1] = topright;
info2[2] = botleft;
info2[3] = botright;
int current_count;
int location;
//Gameplay
for (k=0;k<iterations;k++)
{
//Count buggies
if (k%count_when==0)
{
if (verbose == 1)
{
current_count = rsize*csize-count_buggies(rsize,csize,section);
MPI_Allreduce( ¤t_count, &total, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (rank == 0)
{
printf("Iteration=%5d, Count=%6d\n", k,total);
}
////corner debug
// printf("WR,tl,tr,bl,br = %d %d %d %d %d\n", rank, topleft, topright, botleft, botright);
}
}
//Write to file serially for comparison
//If animation is requested
if (animation == 1 && run_type == 0)
{
//Put smaller matrix part into larger matrix
for (i=0; i<csize; i++)
{
for (j=0; j<rsize; j++)
{
location = (my_row*csize*rsize*ncols + my_col*rsize +
i*rsize*ncols + j);
full_matrix_buffer[location] = section[i*rsize+j];
}
// if (rank == 0)
// {
// printf("Location = %d\n", location);
// }
}
//Gather matrix
MPI_Reduce(full_matrix_buffer, full_matrix, rsize*ncols*csize*nrows,
MPI_UNSIGNED_CHAR, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && run_type == 0)
{
write_matrix_to_pgm(k, rsize*ncols, csize*nrows, full_matrix);
}
}
//mpio write pgm
else if (animation == 1 && (run_type == 1 || run_type == 2))
{
//default is no frame
create_frame = 0;
for (ii=0;ii<20;ii++)
{
for (jj=0;jj<animation_list[ii][1]+1;jj++)
{
// if (rank == 0)
// {
// printf("a,ii,j,k= %i,%i,%i,%i, Frame? = %i\n",
// animation_list[ii][0],ii,jj,k,(animation_list[ii][0]+jj-k)==0);
// }
if ((animation_list[ii][0] + jj - k) == 0)
{
create_frame = 1;
break;
}
}
}
if (create_frame == 1)
{
//dynamic filename with leading zeroes for easy conversion to gif
char buffer[128];
snprintf(buffer, sizeof(char)*128, "Animation/frame%04d.pgm", k);
/* open the file, and set the view */
MPI_File file;
MPI_File_open(MPI_COMM_WORLD, buffer,
MPI_MODE_CREATE|MPI_MODE_WRONLY,
MPI_INFO_NULL, &file);
MPI_File_set_view(file, 0, MPI_UNSIGNED_CHAR, MPI_UNSIGNED_CHAR,
"native", MPI_INFO_NULL);
//write header
MPI_File_write(file, &header1, 15, MPI_CHAR, MPI_STATUS_IGNORE);
//write matrix
MPI_File_set_view(file, 15, MPI_UNSIGNED_CHAR, submatrix,
"native", MPI_INFO_NULL);
MPI_File_write_all(file, section, rsize*csize,
MPI_UNSIGNED_CHAR, MPI_STATUS_IGNORE);
//write footer (trailing newline)
MPI_File_set_view(file, 15+rsize*ncols*csize*nrows,
MPI_UNSIGNED_CHAR, MPI_UNSIGNED_CHAR,
"native", MPI_INFO_NULL);
MPI_File_write(file, &footer, 1, MPI_CHAR, MPI_STATUS_IGNORE);
}
}
// BLOCKED COMMUNITATION //
if (run_type == 1)
{
//change bot (send top) to account for middle area
//alternate to avoid locking
send_to = rank - 1;
receive_from = rank + 1;
//figure out what to send
//top and bottom
for (i=0;i<rsize;i++)
{
ttop[i] = section[i];
tbot[i] = section[rsize*(csize-1)+i];
}
//left n right
for (i=0;i<csize;i++)
{
tleft[i] = section[0 + rsize*i];
tright[i] = section[rsize-1 + rsize*i];
}
//send top, receive bot
if (rank%2==0)
{
if (send_to<world_size && send_to>=0)
{
MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from >= 0)
{
MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from >= 0)
{
MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0)
{
MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD);
}
}
//change top to account for middle area
//alternate to avoid locking
send_to = rank + 1;
receive_from = rank - 1;
//send bot, receive top
if (rank%2==0)
{
// printf("%d, %d, %d\n", rank, send_to, receive_from);
if (send_to<world_size && send_to>=0)
{
MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from >= 0)
{
MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
// printf("%d, %d, %d\n", rank, send_to, receive_from);
if (receive_from<world_size && receive_from >= 0)
{
//*data,count,type,from,tag,comm,mpi_status
MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0)
{
//*data,count,type,to,tag,comm
MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD);
}
}
}
// CHECKERED COMMUNITATION //
else if (run_type == 2)
{
//figure out what to send
//top and bottom
for (i=0;i<rsize;i++)
{
ttop[i] = section[i];
tbot[i] = section[rsize*(csize-1)+i];
}
//left n right
for (i=0;i<csize;i++)
{
tleft[i] = section[0 + rsize*i];
tright[i] = section[rsize-1 + rsize*i];
}
//corners
ttopleft = tleft[0];
tbotleft = tleft[csize-1];
ttopright = tright[0];
tbotright = tright[csize-1];
//Send top, receive bot
send_to = rank - nrows;
receive_from = rank + nrows;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0)
{
MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0)
{
MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0)
{
MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0)
{
MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD);
}
}
//Send bot, receive top
send_to = rank + nrows;
receive_from = rank - nrows;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0)
{
MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0)
{
MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0)
{
MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0)
{
MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD);
}
}
//Send left, receive right
send_to = rank - 1;
receive_from = rank + 1;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row)
{
MPI_Send(tleft, 1, column, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row)
{
MPI_Recv(right, 1, column, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row)
{
MPI_Recv(right, 1, column, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row)
{
MPI_Send(tleft, 1, column, send_to, 0, MPI_COMM_WORLD);
}
}
//Send right, receive left
send_to = rank + 1;
receive_from = rank - 1;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row)
{
MPI_Send(tright, 1, row, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row)
{
MPI_Recv(left, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row)
{
MPI_Recv(left, 1, row, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row)
{
MPI_Send(tright, 1, row, send_to, 0, MPI_COMM_WORLD);
}
}
//Send topright, receive botleft
send_to = rank - ncols + 1;
receive_from = rank + ncols - 1;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1)
{
MPI_Send(&ttopright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1)
{
MPI_Recv(&botleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1)
{
MPI_Recv(&botleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1)
{
MPI_Send(&ttopright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
}
//Send topleft, receive botright
send_to = rank - ncols - 1;
receive_from = rank + ncols + 1;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1)
{
MPI_Send(&ttopleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1)
{
MPI_Recv(&botright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1)
{
MPI_Recv(&botright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1)
{
MPI_Send(&ttopleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
}
//Send botleft, receive topright
send_to = rank + ncols - 1;
receive_from = rank - ncols + 1;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1)
{
MPI_Send(&tbotleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1)
{
MPI_Recv(&topright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1)
{
MPI_Recv(&topright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1)
{
MPI_Send(&tbotleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
}
//Send botright, receive topleft
send_to = rank + ncols + 1;
receive_from = rank - ncols - 1;
if (rank%2==0)
{
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1)
{
MPI_Send(&tbotright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1)
{
MPI_Recv(&topleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
}
else if (rank%2==1)
{
if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1)
{
MPI_Recv(&topleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1)
{
MPI_Send(&tbotright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD);
}
}
info2[0] = topleft;
info2[1] = topright;
info2[2] = botleft;
info2[3] = botright;
}
// if (rank == 1){
// print_matrix(rsize, 1, top);
// print_matrix(rsize, csize, section);
// print_matrix(rsize, 1, bot);
// printf("\n");
// }
// printf("wr=%d,iteration=%d,maxval=%d, 11\n", rank, k,(csize-1)*rsize-1+rsize);
/////////// CELL UPDATES /////////////////
//count neighbor
for (i=0;i<csize;i++)
{
for (j=0; j<rsize; j++)
{
info[0] = i;
info[1] = j;
neighbors[i*rsize+j] = count_neighbors(info, info2, section,
top, bot, left, right);
// printf("%i",neighbors[i*rsize+j]);
}
// printf("\n");
}
//update cells
current_count = 0;
for (i=0;i<csize;i++)
{
for (j=0; j<rsize; j++)
{
//cell currently alive
if (section[i*rsize+j] == 0)
{
//2 or 3 neighbors lives, else die
if (neighbors[i*rsize+j] < 2 ||
neighbors[i*rsize+j] > 3)
{
section[i*rsize+j] = 255;
}
}
else
{
//Exactly 3 neighbors spawns new life
if (neighbors[i*rsize+j] == 3)
{
section[i*rsize+j] = 0;
}
}
}
}
}
MPI_Barrier(MPI_COMM_WORLD);
sleep(0.5);
//free malloc stuff
if( field_a != NULL ) free( field_a );
if( field_b != NULL ) free( field_b );
free(section);
free(neighbors);
free(top);
free(bot);
free(left);
free(right);
MPI_Finalize();
exit (0);
}
int main(int argc, char **argv)
{
MPI_Datatype newtype;
int i, ndims, array_of_gsizes[3], array_of_distribs[3];
int order, nprocs, j, len;
int array_of_dargs[3], array_of_psizes[3];
int *readbuf, *writebuf, mynod, *tmpbuf, array_size;
MPI_Count bufcount;
char *filename;
int errs = 0, toterrs;
MPI_File fh;
MPI_Status status;
MPI_Request request;
MPI_Info info = MPI_INFO_NULL;
int errcode;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
/* process 0 broadcasts the file name to other processes */
if (!mynod) {
filename = "testfile";
len = strlen(filename);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *)malloc(len + 1);
MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD);
}
/* create the distributed array filetype */
ndims = 3;
order = MPI_ORDER_C;
array_of_gsizes[0] = 32;
array_of_gsizes[1] = 32;
array_of_gsizes[2] = 32;
array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK;
array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK;
array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG;
array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG;
for (i = 0; i < ndims; i++) array_of_psizes[i] = 0;
MPI_Dims_create(nprocs, ndims, array_of_psizes);
MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes,
array_of_distribs, array_of_dargs,
array_of_psizes, order, MPI_INT, &newtype);
MPI_Type_commit(&newtype);
/* initialize writebuf */
MPI_Type_size_x(newtype, &bufcount);
bufcount = bufcount / sizeof(int);
writebuf = (int *)malloc(bufcount * sizeof(int));
for (i = 0; i < bufcount; i++) writebuf[i] = 1;
array_size = array_of_gsizes[0] * array_of_gsizes[1] * array_of_gsizes[2];
tmpbuf = (int *) calloc(array_size, sizeof(int));
MPI_Irecv(tmpbuf, 1, newtype, mynod, 10, MPI_COMM_WORLD, &request);
MPI_Send(writebuf, bufcount, MPI_INT, mynod, 10, MPI_COMM_WORLD);
MPI_Wait(&request, &status);
j = 0;
for (i = 0; i < array_size; i++)
if (tmpbuf[i]) {
writebuf[j] = i;
j++;
}
free(tmpbuf);
if (j != bufcount) {
fprintf(stderr, "Error in initializing writebuf on process %d\n",
mynod);
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* end of initialization */
/* write the array to the file */
errcode = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
errcode = MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_set_view");
errcode = MPI_File_iwrite_all(fh, writebuf, bufcount, MPI_INT, &request);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_iwrite_all");
MPI_Wait(&request, &status);
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
if (!mynod) {
/* wkl suggests potential for false " No Errors" if both read
* and write use the same file view */
/* solution: rank 0 reads entire file and checks write values */
errcode = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, info,
&fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
readbuf = (int *)malloc(array_size * sizeof(int));
errcode = MPI_File_read(fh, readbuf, array_size, MPI_INT, &status);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read");
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
for (i = 0; i < array_size; i++)
if (readbuf[i] != i) {
errs++;
fprintf(stderr, "Error: write integer %d but read %d\n",
i, readbuf[i]);
break;
}
free(readbuf);
}
MPI_Barrier(MPI_COMM_WORLD);
/* now read it back */
readbuf = (int *)malloc(bufcount * sizeof(int));
errcode = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open");
errcode = MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_set_view");
errcode = MPI_File_iread_all(fh, readbuf, bufcount, MPI_INT, &request);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_iread_all");
MPI_Wait(&request, &status);
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close");
/* check the data read */
for (i = 0; i < bufcount; i++) {
if (readbuf[i] != writebuf[i]) {
errs++;
fprintf(stderr, "Process %d, readbuf %d, writebuf %d, i %d\n",
mynod, readbuf[i], writebuf[i], i);
}
}
MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (mynod == 0) {
if (toterrs > 0) {
fprintf(stderr, "Found %d errors\n", toterrs);
}
else {
fprintf(stdout, " No Errors\n");
}
}
MPI_Type_free(&newtype);
free(readbuf);
free(writebuf);
if (mynod) free(filename);
MPI_Finalize();
return 0;
}
//------------------------------------------------------------------------------
// Function to output non-magnetic atomic positions to disk
//------------------------------------------------------------------------------
void atoms_non_magnetic(){
//------------------------------------------------------------
// Determine non magnetic atoms to be outputted to coord list
//------------------------------------------------------------
// array of atom numbers to be outputted
std::vector<uint64_t> atom_list(0);
// get output bounds
const double minB[3] = {atoms_output_min[0] * cs::system_dimensions[0],
atoms_output_min[1] * cs::system_dimensions[1],
atoms_output_min[2] * cs::system_dimensions[2]};
const double maxB[3] = {atoms_output_max[0] * cs::system_dimensions[0],
atoms_output_max[1] * cs::system_dimensions[1],
atoms_output_max[2] * cs::system_dimensions[2]};
// Determine non magnetic atoms to be outputted to coord list
for (uint64_t atom = 0; atom < cs::non_magnetic_atoms_array.size(); atom++){
const double cc[3] = {cs::non_magnetic_atoms_array[atom].x, cs::non_magnetic_atoms_array[atom].y, cs::non_magnetic_atoms_array[atom].z};
// check atom within output bounds
if ( (cc[0] >= minB[0]) && (cc[0] <= maxB[0]) ){
if ( (cc[1] >= minB[1]) && (cc[1] <= maxB[1]) ){
if ( (cc[2] >= minB[2]) && (cc[2] <= maxB[2]) ){
atom_list.push_back(atom); //non-magnetic atoms
}
}
}
}
//------------------------------------------------
// Create temporary buffers for atom information
//------------------------------------------------
uint64_t num_local_atoms = atom_list.size();
uint64_t num_total_atoms = 0; // number of atoms across all processors
#ifdef MPICF
// calculate number of atoms to be output on all processors
MPI_Allreduce(&num_local_atoms, &num_total_atoms, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);
#else
num_total_atoms = num_local_atoms;
#endif
std::vector<int> atom_type_buffer(num_local_atoms);
for(unsigned int atom = 0; atom < num_local_atoms; atom++) atom_type_buffer[atom] = cs::non_magnetic_atoms_array[ atom_list[atom] ].mat;
std::vector<int> atom_category_buffer(num_local_atoms);
for(unsigned int atom = 0; atom < num_local_atoms; atom++) atom_category_buffer[atom] = cs::non_magnetic_atoms_array[ atom_list[atom] ].cat;
std::vector<double> atom_coord_buffer(3*num_local_atoms);
for(unsigned int atom = 0; atom < num_local_atoms; atom++){
const uint64_t atom_id = atom_list[atom]; // get atom array index
atom_coord_buffer[3*atom + 0] = cs::non_magnetic_atoms_array[atom_id].x;
atom_coord_buffer[3*atom + 1] = cs::non_magnetic_atoms_array[atom_id].y;
atom_coord_buffer[3*atom + 2] = cs::non_magnetic_atoms_array[atom_id].z;
}
//------------------------------------------
// Output Meta Data from root process
//------------------------------------------
// set number of files
// const int files = config::internal::num_io_groups; // unused variable
if(config::internal::mode != legacy && vmpi::my_rank == 0){
config::internal::write_non_magnetic_meta(num_total_atoms);
}
//------------------------------------------
// Output coordinate data
//------------------------------------------
// Determine output filename
std::stringstream file_sstr;
// set simple file name for single file output
if(config::internal::num_io_groups == 1) file_sstr << "non-magnetic-atoms.data";
// otherwise set indexed files
else file_sstr << "non-magnetic-atoms-" << std::setfill('0') << std::setw(6) << config::internal::io_group_id << ".data";
// convert string stream to string
std::string filename = file_sstr.str();
// Calculate number of bytes to be written to disk
const double data_size = double(num_total_atoms) * 1.0e-9 * (3.0*double(sizeof(double) + 2.0*double(sizeof(int)) ) );
// Output informative messages of actual data size to be outputed to disk (in binary mode)
zlog << zTs() << "Total non-magnetic data filesize: " << 1000.0 * data_size << " MB" << std::endl;
// Output informative message to log file on root process
zlog << zTs() << "Outputting non-magnetic atomic coordinate file to disk ";
// Variable for calculating output bandwidth
double io_time = 1.0e-12;
//-----------------------------------------------------
// Parallel mode output
//-----------------------------------------------------
#ifdef MPICF
// Determine io mode and call appropriate function for data
switch(config::internal::mode){
// legacy
case config::internal::legacy:
break;
case config::internal::mpi_io:{
vutil::vtimer_t timer; // instantiate timer
MPI_File fh; // MPI file handle
MPI_Status status; // MPI io status
// convert filename to character string for output
char *cfilename = (char*)filename.c_str();
// Open file on all processors
MPI_File_open(MPI_COMM_WORLD, cfilename, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fh);
// write number of atoms on root process
if(vmpi::my_rank == 0) MPI_File_write(fh, &num_total_atoms, 1, MPI_UINT64_T, &status);
// Calculate local byte offsets since MPI-IO is simple and doesn't update the file handle pointer after I/O
MPI_Offset type_offset = config::internal::linear_offset + sizeof(uint64_t);
MPI_Offset category_offset = config::internal::linear_offset + num_total_atoms * sizeof(int) + sizeof(uint64_t);
MPI_Offset data_offset = config::internal::buffer_offset + 2 * num_total_atoms * sizeof(int) + sizeof(uint64_t);
timer.start(); // start timer
// Write data to disk
MPI_File_write_at_all(fh, type_offset, &atom_type_buffer[0], atom_type_buffer.size(), MPI_INT, &status);
MPI_File_write_at_all(fh, category_offset, &atom_category_buffer[0], atom_category_buffer.size(), MPI_INT, &status);
MPI_File_write_at_all(fh, data_offset, &atom_coord_buffer[0], atom_coord_buffer.size(), MPI_DOUBLE, &status);
timer.stop(); // Stop timer
// Calculate elapsed time
io_time = timer.elapsed_time();
// Close file
MPI_File_close(&fh);
break;
}
case config::internal::fpprocess:
io_time = write_coord_data(filename, atom_coord_buffer, atom_type_buffer, atom_category_buffer);
break;
case config::internal::fpnode:{
// Gather data from all processors in io group
std::vector<int> collated_atom_type_buffer(0);
collate_int_data(atom_type_buffer, collated_atom_type_buffer);
std::vector<int> collated_atom_category_buffer(0);
collate_int_data(atom_category_buffer, collated_atom_category_buffer);
std::vector<double> collated_atom_coord_buffer(0);
collate_double_data(atom_coord_buffer, collated_atom_coord_buffer);
// output data on master io processes
if(config::internal::io_group_master) io_time = write_coord_data(filename, collated_atom_coord_buffer, collated_atom_type_buffer, collated_atom_category_buffer);
// find longest time in all io nodes
double max_io_time = 0.0;
// calculate actual bandwidth on root process
MPI_Reduce(&io_time, &max_io_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
io_time = max_io_time;
break;
}
}
#else
//-----------------------------------------------------
// Serial mode output (ignores most io directives)
//-----------------------------------------------------
// if new output (not legacy) then output non magnetic atoms
if(config::internal::mode != config::internal::legacy) io_time = write_coord_data(filename, atom_coord_buffer, atom_type_buffer, atom_category_buffer);
#endif
// Output bandwidth to log file
zlog << data_size/io_time << " GB/s in " << io_time << " s" << std::endl;
return;
}
int main(int argc, char** argv){
int irank, nrank;
MPI_Init (&argc, &argv);
MPI_Comm_size (MCW, &nrank);
MPI_Comm_rank (MCW, &irank);
double t1,t2;
if(irank==0) t1 = MPI_Wtime();
int nx, ny;
int px, py;
/* コピペゾーン */
// (1) init dims
int dims[2] = {0,0};
MPI_Dims_create(nrank,2,dims);
ny = (NY-1)/dims[0];
nx = (NX-1)/dims[1];
// (2) init cart
int periods[2] = {0,0}; // 非周期境界
MPI_Comm cart;
MPI_Cart_create(MCW, 2, dims, periods, 0, &cart);
int c[2]; /* 座標 */
MPI_Cart_coords(cart, irank, 2, c);
py = c[0]; // c[2]は大きい順なのでyがc[0]
px = c[1];
double h = 1.0/NX;
double dt = 0.1*h*h;
double dth2 = dt/h/h;
int i,j,k;
int height = ny+2, width = nx+2;
double (*u)[width];
u = (double(*)[width])malloc(height*width*sizeof(double));
u = (double(*)[width])(&u[1][1]);
double (*un)[width];
un = (double(*)[width])malloc(height*width*sizeof(double));
un = (double(*)[width])(&un[1][1]);
for (j=-1;j<ny+1;j++)
for (i=-1;i<nx+1;i++){
u[j][i] = 0.0;
}
// (y=0)
if (py==0)
for (i=-1;i<nx+1;i++){
u[-1][i] = 1.0;
}
// (x=0)
if (px==0)
for (j=0;j<ny+1;j++){
u[j][-1] = 0.5;
}
MPI_Datatype vedge;
MPI_Type_vector(ny, 1, nx+2, MPI_DOUBLE, &vedge);
MPI_Type_commit(&vedge);
int north, south, east, west;
MPI_Cart_shift(cart,0,1,&south,&north);
MPI_Cart_shift(cart,1,1,&west,&east);
/* loop start */
for (k=0; k<2000; k++){
for (j=0; j<ny; j++){
for (i=0; i<nx; i++)
un[j][i] = u[j][i] + ( -4*u[j][i] + u[j][i+1] + u[j][i-1] + u[j+1][i] + u[j-1][i] )*dth2;
}
for (j=0; j<ny; j++){
for (i=0; i<nx; i++)
u[j][i] = un[j][i];
}
MPI_Sendrecv(&u[ny-1][0], nx, MPI_DOUBLE, north, 0,
&u[-1][0], nx, MPI_DOUBLE, south, 0,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Sendrecv(&u[0][0], nx, MPI_DOUBLE, south, 0,
&u[ny][0], nx, MPI_DOUBLE, north, 0,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Sendrecv(&u[0][nx-1], 1, vedge, east, 0,
&u[0][-1], 1, vedge, west, 0,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Sendrecv(&u[0][0], 1, vedge, west, 0,
&u[0][nx], 1, vedge, east, 0,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
} // end loop(k)
/* setview */
MPI_File udata;
MPI_File_open(cart, "u.data", MPI_MODE_WRONLY | MPI_MODE_CREATE,
MPI_INFO_NULL, &udata);
MPI_File_set_size(udata,0);
int size[2] = {NY+1, LW*(NX+1)+1}, subsize[2], start[2];
// ... py,pxはcartesian座標
subsize[0] = ny;
subsize[1] = LW*nx;
start[0] = py*ny+1;
start[1] = LW*(px*nx+1);
if (py == 0){ subsize[0]++; start[0]=0; } /* 南端↓ */
if (py == dims[0]-1) subsize[0]++; /* 北端↑ */
if (px == 0){ subsize[1]+=LW; start[1]=0; } /* 西端← */
if (px == dims[1]-1) subsize[1]+=LW+1; /* 東端→ */
MPI_Datatype ftype;
MPI_Type_create_subarray(2, size, subsize, start,
MPI_ORDER_C, MPI_CHAR, &ftype);
MPI_Type_commit(&ftype);
MPI_File_set_view(udata, 0, MPI_CHAR, ftype, "native",
MPI_INFO_NULL);
/* output */
MPI_Status st;
char *wbuf = (char*)malloc((LW*(nx+2)+2)*sizeof(char));
int jstart=0,istart=0, jend=ny, iend=nx;
if(py==0) jstart = -1;
if(py==dims[0]-1) jend = ny+1;
if(px==0) istart = -1;
if(px==dims[1]-1) iend = nx+1;
for(j=jstart; j<jend; j++){
for(i=istart,k=0; i<iend; i++,k+=LW){
sprintf( wbuf+k, " %.15E %.15E %21.15E\n",
(i+1 + px*nx)*h, (j+1 + py*ny)*h, u[j][i] );
}
if( px == dims[1]-1 ) // 東端→
sprintf(wbuf+(k++),"\n");
MPI_File_write(udata,wbuf,k,MPI_CHAR,&st);
}
MPI_File_close(&udata);
if(irank==0){
t2 = MPI_Wtime();
printf("%g\n",t2-t1);
}
MPI_Finalize ();
return 0;
}
int main(int argc, char **argv)
{
int *buf, i, rank, nprocs, len, sum;
int global_sum;
int errs=0, toterrs, errcode;
char *filename;
MPI_File fh;
MPI_Status status;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
double wr_stime, wr_etime, wr_time, wr_sumtime;
double rd_stime, rd_etime, rd_time, rd_sumtime;
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: shared_fp -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(COUNT * sizeof(int));
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
for (i=0; i<COUNT; i++) buf[i] = COUNT*rank + i;
errcode = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_open");
}
wr_stime = MPI_Wtime();
errcode = MPI_File_write_ordered(fh, buf, COUNT, MPI_INT, &status);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_write_shared");
}
wr_etime = MPI_Wtime();
for (i=0; i<COUNT; i++) buf[i] = 0;
MPI_Barrier(MPI_COMM_WORLD);
rd_stime = MPI_Wtime();
errcode = MPI_File_seek_shared(fh, 0, MPI_SEEK_SET);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_seek_shared");
}
errcode = MPI_File_read_ordered(fh, buf, COUNT, MPI_INT, &status);
if (errcode != MPI_SUCCESS) {
handle_error(errcode, "MPI_File_read_shared");
}
rd_etime = MPI_Wtime();
MPI_File_close(&fh);
sum = 0;
for (i=0; i<COUNT; i++) sum += buf[i];
MPI_Allreduce(&sum, &global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
wr_time = wr_etime - wr_stime;
rd_time = rd_etime - rd_stime;
MPI_Allreduce(&wr_time, &wr_sumtime, 1,
MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&rd_time, &rd_sumtime, 1,
MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
if (global_sum != (((COUNT*nprocs - 1)*(COUNT*nprocs))/2)) {
errs++;
fprintf(stderr, "Error: sum %d, global_sum %d, %d\n",
sum, global_sum,(((COUNT*nprocs - 1)*(COUNT*nprocs))/2));
}
free(buf);
free(filename);
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (rank == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
#ifdef TIMING
fprintf( stderr, "nprocs: %d bytes: %d write: %f read %f\n",
nprocs, COUNT*sizeof(int), wr_sumtime, rd_sumtime);
#endif
}
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
MPI_File fh;
MPI_Status status;
MPI_Offset size;
long long *buf, i;
char *filename;
int j, mynod, nprocs, len, flag, err;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (nprocs != 1) {
fprintf(stderr, "Run this program on one process only\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: large -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+1);
strcpy(filename, *argv);
fprintf(stderr, "This program creates an 4 Gbyte file. Don't run it if you don't have that much disk space!\n");
buf = (long long *) malloc(SIZE * sizeof(long long));
if (!buf) {
fprintf(stderr, "not enough memory to allocate buffer\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
for (i=0; i<NTIMES; i++) {
for (j=0; j<SIZE; j++)
buf[j] = i*SIZE + j;
err = MPI_File_write(fh, buf, SIZE, MPI_DOUBLE, &status);
/* MPI_DOUBLE because not all MPI implementations define
MPI_LONG_LONG_INT, even though the C compiler supports long long. */
if (err != MPI_SUCCESS) {
fprintf(stderr, "MPI_File_write returned error\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
MPI_File_get_size(fh, &size);
fprintf(stderr, "file size = %lld bytes\n", size);
MPI_File_seek(fh, 0, MPI_SEEK_SET);
for (j=0; j<SIZE; j++) buf[j] = -1;
flag = 0;
for (i=0; i<NTIMES; i++) {
err = MPI_File_read(fh, buf, SIZE, MPI_DOUBLE, &status);
/* MPI_DOUBLE because not all MPI implementations define
MPI_LONG_LONG_INT, even though the C compiler supports long long. */
if (err != MPI_SUCCESS) {
fprintf(stderr, "MPI_File_write returned error\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (j=0; j<SIZE; j++)
if (buf[j] != i*SIZE + j) {
fprintf(stderr, "error: buf %d is %lld, should be %lld \n", j, buf[j],
i*SIZE + j);
flag = 1;
}
}
if (!flag) fprintf(stderr, "Data read back is correct\n");
MPI_File_close(&fh);
free(buf);
free(filename);
MPI_Finalize();
return 0;
}
void
qpb_write_spinor(qpb_spinor_field spinor_field, char fname[])
{
MPI_Datatype mpi_dtype_spinor_float, filetype;
MPI_Type_contiguous(2*NC*NS, MPI_FLOAT, &mpi_dtype_spinor_float);
MPI_Type_commit(&mpi_dtype_spinor_float);
int starts[ND], l_dim[ND], g_dim[ND];
for(int i=0; i<ND; i++)
{
starts[i] = problem_params.coords[i]*problem_params.l_dim[i];
l_dim[i] = problem_params.l_dim[i];
g_dim[i] = problem_params.g_dim[i];
};
int ierr = MPI_Type_create_subarray(ND, g_dim, l_dim, starts, MPI_ORDER_C,
mpi_dtype_spinor_float, &filetype);
MPI_Type_commit(&filetype);
MPI_File fhandle;
ierr = MPI_File_open(MPI_COMM_WORLD, fname, MPI_MODE_WRONLY | MPI_MODE_CREATE,
MPI_INFO_NULL, &fhandle);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_open() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
ierr = MPI_File_set_view(fhandle, 0, mpi_dtype_spinor_float, filetype,
"native", MPI_INFO_NULL);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_set_view() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
void *buffer = qpb_alloc(problem_params.l_vol*sizeof(qpb_spinor_float));
for(int v=0; v<problem_params.l_vol; v++)
for(int sp=0; sp<NC*NS; sp++)
{
((float *) buffer)[v*NC*NS*2 + sp*2] = spinor_field.bulk[v][sp].re;
((float *) buffer)[v*NC*NS*2 + sp*2 + 1] = spinor_field.bulk[v][sp].im;
}
if(!qpb_is_bigendian())
qpb_byte_swap_float(buffer, problem_params.l_vol*NC*NS*2);
MPI_Status status;
ierr = MPI_File_write_all(fhandle, buffer, problem_params.l_vol,
mpi_dtype_spinor_float, &status);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_read() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
ierr = MPI_File_close(&fhandle);
if(ierr != MPI_SUCCESS)
{
if(am_master)
{
fprintf(stderr, "%s: MPI_File_close() returned in error\n", fname);
exit(QPB_FILE_ERROR);
}
}
free(buffer);
return;
}
int main(int argc, char **argv) {
int size, rank, rc, root = 0, nameLength = 20;
MPI_Status status;
MPI_File configFile = malloc(sizeof configFile);
MPI_Info info;
char *configFileName = "./configFile.txt";
createLogFile();
MPI_Init(&argc, &argv);
initLogFile();
MPI_Info_create(&info);
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
printf("%d/%d started.\n", rank+1, size);
char buf[nameLength + 1];
rc = MPI_File_open(MPI_COMM_WORLD, configFileName,
MPI_MODE_RDONLY, info, &configFile);
printf("%d/%d achieved the file_open result: %d.\n", rank+1, size, rc);
// set the individual pointer to our position in the config file
// master is the master of elevators
rc = MPI_File_seek(configFile, rank * nameLength, MPI_SEEK_SET);
rc = MPI_File_read(configFile, buf, nameLength, MPI_CHAR, &status);
buf[nameLength] = '\0';
int len = nameLength - 1;
while ((len >= 0) && (buf[len] == ' ')) {
buf[len] = '\0';
len--;
}
rc = MPI_File_close(&configFile);
char *ourname = buf;
char str[50 + nameLength];
printf(strcat (strcat (strcpy (str, "%d/%d has the name '"), ourname), "'.\n"), rank+1, size);
//read in the file to be counted, line by line.
FILE * fp;
fp = fopen(argv[1], "r");
if (fp == NULL) {
printf("%d/%d did not find a document to count! Switching to assignment part 1.\n", rank+1, size);
assgn = 1;
} else {
printf("%d/%d found a document to count. Switching to assignment part 2.\n", rank+1, size);
assgn = 2;
/*
// general idea:
char * line = NULL;
size_t lineLen = 0;
ssize_t read;
dlist* list = NULL;
dlist_create(10, &list);
printf("size of list is %d\n", list->size);
for (int c = 0; c < 15; c++) {
char buf[6]; // long enough for test + num, e.g. test1
sprintf(buf, "test%d", c);
dlist_append(&list, buf);
}
char* thestring;
printf("capacity is %d\n", list->capacity);
for (int c = 0; c < 15; c++) {
dlist_get(&list, c, &thestring);
printf("string is: %s\n", thestring);
}
while ((read = getline(&line, &lineLen, fp)) != -1) {
printf("Retrieved line of length %zu :\n", read);
printf("%s", line);
}
if (line)
free(line);
*/
char * line = NULL;
size_t lineLen = 0;
ssize_t read;
dlist_create(10, &list);
// go through the lines in the file...
int i = 0;
while ((read = getline(&line, &lineLen, fp)) != -1) {
i++;
// ... and map them to the individual worker threads
if (i % (size-1) == rank-1) {
dlist_append(&list, line);
}
}
if (line)
free(line);
}
// we check whether or not we are the root process.
if (rank == root) {
master(rank);
}
else {
worker(rank, ourname);
}
closeLogFile();
printf("%d/%d ended.\n", rank+1, size);
MPI_Finalize();
}
void printio_(int *fparam, int* piostep)
#endif
{
//printf("format param is %d, iostep is %d\n", (int)*fparam, *piostep);
int formatparam = *fparam;
int iostep = *piostep;
double overall_max, overall_min, overall_avg, overall_sum;
double io_time_max = 0.0;
double file_io_max = 0.0;
if( formatparam == 2 || formatparam == 3 || formatparam == 4 || formatparam == 5)
{
MPI_Comm_size(MPI_COMM_WORLD, &mysize);
MPI_Allreduce( &overall_time, &overall_min, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
MPI_Allreduce( &overall_time, &overall_max, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce( &overall_time, &overall_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
overall_avg = overall_sum / mysize;
}
else if(formatparam == 6 || formatparam == -6 || formatparam == 8 || formatparam == 18)
{
if(mySpecies == 1)
{
MPI_Allreduce( &overall_time, &overall_min, 1, MPI_DOUBLE, MPI_MIN, localcomm);
MPI_Allreduce( &overall_time, &overall_max, 1, MPI_DOUBLE, MPI_MAX, localcomm);
MPI_Allreduce( &overall_time, &overall_sum, 1, MPI_DOUBLE, MPI_SUM, localcomm);
overall_avg = overall_sum / localsize;
MPI_Allreduce( &file_io_time, &file_io_max, 1, MPI_DOUBLE, MPI_MAX, localcomm);
MPI_Allreduce( &io_time, &io_time_max, 1, MPI_DOUBLE, MPI_MAX, localcomm);
}
else if(mySpecies == 2)
{
overall_time = 0;
overall_min = 0;
overall_max = 0;
overall_avg = 0;
}
}
int temp_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &temp_rank);
if(temp_rank == 0) {
printf("**************************************\n");
printf("I/O time (io_step=%d) stats: overall avg = %lf sec, min = %lf sec, max = %lf sec "
"(io_max = %lf sec, file_io_max = %lf sec, wtick=%lf sec),"
"checkpoint file path is %s, machine is %s, io_option = %d, num_groups = %d "
"(DEBUG_FLAG=%d, COMPUTE_TRACE_FLAG=%d).\n",
io_step, overall_avg, overall_min, overall_max, io_time_max, file_io_max, MPI_Wtick(),
path, mach_name, formatparam, numGroups,
DEBUG_FLAG, COMPUTE_TRACE_FLAG);
printf("**************************************\n");
}
MPI_Barrier(MPI_COMM_WORLD);
// return if IO trace flag not set, otherwise write timing trace of each i/o op
if(IOTRACE_FLAG != 1)
return;
char tracefname[128];
memset((void*)tracefname, 0, 128);
sprintf(tracefname, "iotrace-t%.5d.dat", iostep);
// write the actual file
if (1) {
MPI_File timefile;
int rc;
rc = MPI_File_open(MPI_COMM_WORLD, tracefname,
MPI_MODE_CREATE | MPI_MODE_WRONLY , MPI_INFO_NULL, &timefile);
char mytime[128];
sprintf(mytime, "\n%10d %10.3lf %10.3lf %10.3lf %10.3lf ",
temp_rank, overall_time, overall_avg, overall_min, overall_max);
long long offsets = temp_rank * 56 ;
MPI_Status write_data_status;
MPI_File_write_at_all_begin(timefile,
offsets,
mytime,
56,
MPI_CHAR);
MPI_File_write_at_all_end(timefile,
mytime,
&write_data_status);
MPI_File_close( & timefile );
}
}
int main (int argc, char *argv[])
{
int proc_num, my_rank;
MPI_Status status;
MPI_Request request;
MPI_File fh;
int i, j, k, iter;
int **mymat, *allmat;
double compute_time, compute_time_start, compute_time_end;
double io_time, io_time_start, io_time_end;
double total_time_start, total_time_end;
double sum, temp, diff, bb, allbb;
double e = 0.00001;
double *x, *myx;
// whether to use prefetch or not ,see below
int compare_mode = 0;
// Init MPI
MPI_Init(&argc, &argv);
// record start time of program
total_time_start = MPI_Wtime();
// get the number of procs and rank in the comm
MPI_Comm_size(MPI_COMM_WORLD, &proc_num);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
if(argc < 4) {
printf("Usage: jacobi mat.bin dim mat_num [enable prefetch]\n");
return -1;
}
else if(argc == 5) {
// compare mode
// 0 means no prefetch, 1 means prefetch
compare_mode = atoi(argv[4]);
if(my_rank == 0 ) {
if(compare_mode == 1)
printf("Using prefetching\n");
else
printf("No prefetching\n");
}
}
// broadcast prefetch mode
MPI_Bcast( &compare_mode, 1, MPI_INT, 0, MPI_COMM_WORLD);
// n is dimension of the input matrix
int n = atoi(argv[2]);
// each proc get myrows rows of the matrix
int myrows = n / proc_num;
// CACHE Allocation
int *prefetch_cache = malloc( myrows * (n+1) * sizeof(int));
// allmat is 1-D array to store matrix
allmat = (int*) malloc( myrows * (n+1) * sizeof(int));
// mymat makes it a 2-D array
mymat = (int**) malloc(myrows * sizeof(int *));
for (i = 0; i < myrows; i++) {
mymat[i]= &allmat[i * (n+1)];
}
// x stores global result, myx stores local result
x = (double*) malloc( n * sizeof(double) );
myx = (double*) malloc( myrows * sizeof(double));
// how many matrices to solve in total
int mat_num = atoi(argv[3]);
#if DEBUG
printf("I'm proc%d, n=%d, myrows=%d, mat_num=%d\n", my_rank, n, myrows, mat_num);
if(my_rank == -1) {
int dowait = 1;
while(dowait) {
;
}
}
#endif
// File opening
MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if(fh==NULL) {
printf("File not exist\n");
return -1;
}
io_time = 0.0;
compute_time = 0.0;
double start, finish;
// each round read entire matrix and prefetch next round
for(k = 0; k < mat_num; k++) {
MPI_Barrier(MPI_COMM_WORLD);
// I/O time
io_time_start = MPI_Wtime();
if(compare_mode == 1) {
// use prefetch
if(k == 0) {
start = MPI_Wtime();
// first time read, no pattern information known so just normal read
MPI_File_read_at( fh, (myrows * my_rank + k * n) * (n+1) * sizeof(int)
, allmat, myrows * (n+1), MPI_INT, &status );
finish = MPI_Wtime();
if(my_rank ==0) printf("First read time %lf\n",finish - start);
// According to previous read, predict the next read
// use non-blocking read so computation could be performed at the same time
MPI_File_iread_at( fh, (myrows * my_rank + (k+1) * n) * (n+1) * sizeof(int)
, prefetch_cache, myrows * (n+1), MPI_INT, &request );
}
else {
start = MPI_Wtime();
// wait for previous iread(prefetched the predicted next access) completed
MPI_Wait(&request, &status);
finish = MPI_Wtime();
if(my_rank ==0) printf("Wait time %lf\n",finish - start);
start = MPI_Wtime();
// copy prefetched data from cache to target
memcpy(allmat, prefetch_cache, myrows * (n+1) * sizeof(int));
finish = MPI_Wtime();
if(my_rank ==0) printf("Memcpy time %lf\n",finish - start);
// next read is predicted so perform prefetch
if(k != mat_num -1) {
MPI_File_iread_at( fh, (myrows * my_rank + (k+1) * n) * (n+1) * sizeof(int)
, prefetch_cache, myrows * (n+1), MPI_INT, &request );
}
}
}
else {
// normal read
MPI_File_read_at( fh, (myrows * my_rank + k * n) * (n+1) * sizeof(int)
, allmat, myrows * (n+1), MPI_INT, &status );
}
MPI_Barrier(MPI_COMM_WORLD);
if(my_rank == 0) {
io_time_end = MPI_Wtime();
printf("I/O time of %d round: %lf\n",k , io_time_end - io_time_start);
io_time += io_time_end - io_time_start;
}
#if PRINTMAT
// print matrix
printf("rank %d:\n",my_rank);
for(i=0; i<myrows; i++) {
for(j=0; j<n+1; j++) {
printf(" %4d", mymat[i][j]);
}
printf("\n");
}
#endif
// set local and global x to zero for each iteration
memset( myx, 0, sizeof(myx[0]) * myrows );
memset( x, 0, sizeof(x[0]) * myrows );
compute_time_start = MPI_Wtime();
// start iteration of computation till converge
iter=0;
do {
bb=0.0;
// all proc get all x
MPI_Allgather(myx, myrows, MPI_DOUBLE, x, myrows, MPI_DOUBLE, MPI_COMM_WORLD);
for(i = 0; i < myrows; i++) {
sum=0.0;
for(j = 0; j < n; j++) {
if(j!=i+myrows*my_rank) {
sum=sum+(double)mymat[i][j]*x[j];
}
}
temp=( (double)mymat[i][n]-sum ) / (double)mymat[i][i+myrows*my_rank];
diff=fabs(x[i]-temp);
if(diff>bb) {
bb=diff;
}
myx[i]=temp;
}
// each process get same bb value so all can go out of loop
MPI_Allreduce( &bb, &allbb, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD );
iter++;
// if(my_rank == 0)
// printf("iter = %d, bb = %lf\n",iter, allbb);
} while(allbb>=e);
// gather final x for print of each matrix
MPI_Allgather(myx, myrows, MPI_DOUBLE, x, myrows, MPI_DOUBLE, MPI_COMM_WORLD);
if(my_rank ==0 ) {
// record end time of computation
compute_time_end = MPI_Wtime();
printf("Compute time of %d round: %lf\n",k , compute_time_end - compute_time_start);
compute_time += compute_time_end - compute_time_start;
// append result to file
print_x(iter, n, x);
}
}//k
MPI_File_close( &fh );
if(my_rank == 0) {
printf("Total I/O time: %lf bandwidth: %.2lf MB/s\n", io_time, n*(n+1)*4.0/(io_time*1024*1024));
printf("Total compute time: %lf\n", compute_time);
}
total_time_end = MPI_Wtime();
if(my_rank == 0)
printf("Total time: %lf\n",total_time_end - total_time_start);
// free allocated memory
free(allmat);
free(mymat);
free(myx);
free(x);
free(prefetch_cache);
MPI_Finalize();
return 0;
}
/*
*
* Main functions
*
*/
int main( int argc, char* argv[] ) {
double start, end;
double IO_millis = 0, comm_millis = 0, comp_millis = 0, sync_millis = 0;
int rank, size;
MPI_Init( &argc, &argv );
MPI_Comm_size( MPI_COMM_WORLD, &size );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
int N;
char* in_file;
char* out_file;
char phase = EVEN_PHASE;
N = atoi( argv[1] );
in_file = argv[2];
out_file = argv[3];
// Currently, the parameters needed in the project have been set up.
// N : the # of integers to be sorted
// in_file : the path of the input file which contains all integers to be sorted
// out_file : the path of the output file where the sorted sequence is printed
int red_process_num = 0;
if ( size > N ) {
red_process_num = size - N;
size = N;
}
int data_bufsize = ( N % size > 0 ) ? ( N + size - ( N % size ) ) : N;
int* data_buf = malloc( ( size + red_process_num ) * ( data_bufsize / size ) * sizeof( int ) );
MPI_File f;
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
start = MPI_Wtime();
// MPI_File_open (MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_File *fh)
MPI_File_open( MPI_COMM_WORLD, in_file, MPI_MODE_RDONLY, MPI_INFO_NULL, &f );
// MPI_File_read (MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)
MPI_File_read( f, data_buf, N, MPI_INT, MPI_STATUS_IGNORE );
end = MPI_Wtime();
IO_millis += (end - start) * 1000;
int i;
for (i = 0; i < data_bufsize - N; i++) data_buf[ N + i ] = INT_MAX;
int local_bufsize = data_bufsize / size;
int* local_buf = malloc( local_bufsize * sizeof( int ) );
if ( rank < size )
for (i = 0; i < local_bufsize; i++) local_buf[i] = data_buf[ rank * local_bufsize + i ];
if ( rank != ROOT ) free( data_buf );
int msg_buf;
int msg_recved, msg_sent;
int local_buf_pos = rank * local_bufsize; // it means local_buf[0] == data_buf[local_buf_pos]
int swapped[2] = { 1, 1 };
int sync_swapped[2] = { 1, 1 };
int swap_temp;
while ( sync_swapped[0] > 0 || sync_swapped[1] > 0 ) {
msg_recved = 0;
msg_sent = 0;
swapped[phase] = 0;
if ( rank < size ) {
// Send: even-phase when pos is odd; odd-phase when pos is even
if ( rank > 0 && local_buf_pos % 2 != phase ) {
start = MPI_Wtime();
MPI_Send( &( local_buf[0] ), 1, MPI_INT, rank-1, 0, MPI_COMM_WORLD );
end = MPI_Wtime();
comm_millis += (end - start) * 1000;
msg_sent = 1;
}
// Recv: even-phase when pos+bufsize-1 is even, odd-phase when pos+bufsize-1 is odd
if ( rank < size-1 && ( local_buf_pos + local_bufsize - 1 ) % 2 == phase ) {
start = MPI_Wtime();
MPI_Recv( &msg_buf, 1, MPI_INT, rank+1, MPI_ANY_TAG, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
end = MPI_Wtime();
comm_millis += (end - start) * 1000;
msg_recved = 1;
}
}
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
if ( rank < size ) {
int j;
// Compare:
// even-phase starts at local_buf_pos+1/local_buf_pos+2 (even/odd)
// odd-phase starts at local_buf_pos/local_buf_pos+1 (even/odd)
start = MPI_Wtime();
for (j = phase + 1; j < N; j+=2) {
if ( j >= local_buf_pos + local_bufsize ) break;
if ( j <= local_buf_pos ) continue;
if ( compare( local_buf[ j - local_buf_pos - 1 ], local_buf[ j - local_buf_pos ] ) == 1 ) {
swapped[phase] = 1;
swap_temp = local_buf[ j - local_buf_pos - 1 ];
local_buf[ j - local_buf_pos - 1 ] = local_buf[ j - local_buf_pos ];
local_buf[ j - local_buf_pos ] = swap_temp;
}
}
end = MPI_Wtime();
comp_millis += (end - start) * 1000;
}
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
if ( rank < size ) {
if ( msg_recved == 1 ) {
start = MPI_Wtime();
if ( compare( local_buf[ local_bufsize-1 ], msg_buf ) == 1 ) {
swapped[phase] = 1;
swap_temp = local_buf[ local_bufsize-1 ];
local_buf[ local_bufsize-1 ] = msg_buf;
msg_buf = swap_temp;
}
end = MPI_Wtime();
comp_millis += (end - start) * 1000;
start = MPI_Wtime();
MPI_Send( &msg_buf, 1, MPI_INT, rank+1, 0, MPI_COMM_WORLD );
end = MPI_Wtime();
comm_millis += (end - start) * 1000;
}
}
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
if ( rank < size ) {
if ( msg_sent == 1 ) {
start = MPI_Wtime();
MPI_Recv( &msg_buf, 1, MPI_INT, rank-1, MPI_ANY_TAG, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
end = MPI_Wtime();
comm_millis += (end - start) * 1000;
start = MPI_Wtime();
if ( msg_buf != local_buf[0] ) swapped[phase] = 1;
local_buf[0] = msg_buf;
end = MPI_Wtime();
comp_millis += (end - start) * 1000;
}
}
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
start = MPI_Wtime();
// MPI_Allreduce (send_buf, recv_buf, count, data_type, op, comm)
MPI_Allreduce( &( swapped[phase] ), &( sync_swapped[phase] ), 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
end = MPI_Wtime();
comm_millis += (end - start) * 1000;
toggle_phase( &phase );
}
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
start = MPI_Wtime();
// MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm)
MPI_Gather( local_buf, local_bufsize, MPI_INT, data_buf, local_bufsize, MPI_INT, ROOT, MPI_COMM_WORLD );
end = MPI_Wtime();
comm_millis += (end - start) * 1000;
if ( rank == ROOT ) {
start = MPI_Wtime();
FILE* f = fopen( out_file, "wb" );
fwrite( data_buf, sizeof( int ), N, f );
end = MPI_Wtime();
IO_millis += (end - start) * 1000;
}
start = MPI_Wtime();
MPI_Barrier( MPI_COMM_WORLD );
end = MPI_Wtime();
sync_millis += (end - start) * 1000;
printf( "{\n\t\"id\": %d,\n\t\"i/o\": %.3f,\n\t\"comm\": %.3f,\n\t\"sync\": %.3f,\n\t\"comp\": %.3f\n}\n",
rank, IO_millis, comm_millis, sync_millis, comp_millis );
MPI_Barrier( MPI_COMM_WORLD );
MPI_Finalize();
return 0;
}
int main(int argc, char * argv[])
{
int i;
int ret;
MPI_File file;
MPI_Status status;
char * filename = "tokufs:/hellobasicmpiio";
MPI_Init(&argc, &argv);
ret = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &file);
printf("open ret = %d\n", ret);
if (ret != MPI_SUCCESS) {
char * estr = malloc(1000);
int l;
MPI_Error_string(ret, estr, &l);
printf("%s\n", estr);
free(estr);
}
assert(ret == MPI_SUCCESS);
/* Generate the usual 0-99 repeating buffer */
char write_buf[BUF_SIZE];
for (i = 0; i < BUF_SIZE; i++) {
write_buf[i] = i % 100;
}
/* Write to the file as 10 seperate chunks with explicit offsets. */
int num_chunks = 10;
assert(BUF_SIZE % num_chunks == 0);
int write_size = BUF_SIZE / num_chunks;
for (i = 0; i < 10; i++) {
ret = MPI_File_write_at(file, i * write_size,
write_buf + i * write_size,
write_size, MPI_CHAR, &status);
assert(ret == MPI_SUCCESS);
}
/* Close the file and open it again */
ret = MPI_File_close(&file);
assert(ret == MPI_SUCCESS);
ret = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_EXCL | MPI_MODE_RDWR,
MPI_INFO_NULL, &file);
assert(ret == MPI_SUCCESS);
/* Read back the file in a similar fashion, into a new buffer. */
char read_buf[BUF_SIZE];
int read_size = write_size;
for (i = 0; i < 10; i++) {
ret = MPI_File_read_at(file, i * read_size,
read_buf + i * read_size,
read_size, MPI_CHAR, &status);
assert(ret == MPI_SUCCESS);
}
/* Verify the read buf is the same as the write buf */
for (i = 0; i < BUF_SIZE; i++) {
if (read_buf[i] != write_buf[i]) {
printf("%s:%s: buf[%d]: expected %d, got %d\n",
__FILE__, __FUNCTION__,
i, write_buf[i], read_buf[i]);
}
assert(read_buf[i] == write_buf[i]);
}
ret = MPI_File_close(&file);
assert(ret == 0);
ret = MPI_Finalize();
assert(ret == 0);
return 0;
}
int main(int argc, char **argv)
{
int *buf, i, rank, nints, len;
char *filename, *tmp;
int errs = 0, toterrs, errcode;
MPI_File fh;
MPI_Status status;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: simple -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
MPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE);
nints = SIZE/sizeof(int);
for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
/* each process opens a separate file called filename.'myrank' */
tmp = (char *) malloc(len+10);
strcpy(tmp, filename);
sprintf(filename, "%s.%d", tmp, rank);
errcode = MPI_File_open(MPI_COMM_SELF, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open(1)");
errcode = MPI_File_write(fh, buf, nints, MPI_INT, &status);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_write");
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_clode (1)");
/* reopen the file and read the data back */
for (i=0; i<nints; i++) buf[i] = 0;
errcode = MPI_File_open(MPI_COMM_SELF, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_open(2)");
errcode = MPI_File_read(fh, buf, nints, MPI_INT, &status);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_read");
errcode = MPI_File_close(&fh);
if (errcode != MPI_SUCCESS) handle_error(errcode, "MPI_File_close(2)");
/* check if the data read is correct */
for (i=0; i<nints; i++) {
if (buf[i] != (rank*100000 + i)) {
errs++;
fprintf(stderr, "Process %d: error, read %d, should be %d\n",
rank, buf[i], rank*100000+i);
}
}
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (rank == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
free(buf);
free(filename);
free(tmp);
MPI_Finalize();
return 0;
}
void stamp_matrix(Lattice *grid, double *matrix, string filename) {
#ifdef HAVE_MPI
// Set variables for mpi output
char *data_as_txt;
int count;
MPI_File file;
MPI_Status status;
// each number is represented by charspernum chars
const int charspernum = 14;
MPI_Datatype num_as_string;
MPI_Type_contiguous(charspernum, MPI_CHAR, &num_as_string);
MPI_Type_commit(&num_as_string);
// create a type describing our piece of the array
int globalsizes[2] = {grid->global_dim_y - 2 * grid->periods[0] * grid->halo_y, grid->global_dim_x - 2 * grid->periods[1] * grid->halo_x};
int localsizes [2] = {grid->inner_end_y - grid->inner_start_y, grid->inner_end_x - grid->inner_start_x};
int starts[2] = {grid->inner_start_y, grid->inner_start_x};
int order = MPI_ORDER_C;
MPI_Datatype localarray;
MPI_Type_create_subarray(2, globalsizes, localsizes, starts, order, num_as_string, &localarray);
MPI_Type_commit(&localarray);
// output real matrix
//conversion
data_as_txt = new char[(grid->inner_end_x - grid->inner_start_x) * (grid->inner_end_y - grid->inner_start_y) * charspernum];
count = 0;
for (int j = grid->inner_start_y - grid->start_y; j < grid->inner_end_y - grid->start_y; j++) {
for (int k = grid->inner_start_x - grid->start_x; k < grid->inner_end_x - grid->start_x - 1; k++) {
sprintf(&data_as_txt[count * charspernum], "%+.5e ", matrix[j * grid->dim_x + k]);
count++;
}
if(grid->mpi_coords[1] == grid->mpi_dims[1] - 1) {
sprintf(&data_as_txt[count * charspernum], "%+.5e\n ", matrix[j * grid->dim_x + (grid->inner_end_x - grid->start_x) - 1]);
count++;
}
else {
sprintf(&data_as_txt[count * charspernum], "%+.5e ", matrix[j * grid->dim_x + (grid->inner_end_x - grid->start_x) - 1]);
count++;
}
}
// open the file, and set the view
MPI_File_open(grid->cartcomm, const_cast<char*>(filename.c_str()),
MPI_MODE_CREATE | MPI_MODE_WRONLY,
MPI_INFO_NULL, &file);
MPI_File_set_view(file, 0, MPI_CHAR, localarray, (char *)"native", MPI_INFO_NULL);
MPI_File_write_all(file, data_as_txt, (grid->inner_end_x - grid->inner_start_x) * (grid->inner_end_y - grid->inner_start_y), num_as_string, &status);
MPI_File_close(&file);
delete [] data_as_txt;
#else
print_matrix(filename.c_str(), &(matrix[grid->global_dim_x * (grid->inner_start_y - grid->start_y) + grid->inner_start_x - grid->start_x]), grid->global_dim_x,
grid->global_dim_x - 2 * grid->periods[1]*grid->halo_x, grid->global_dim_y - 2 * grid->periods[0]*grid->halo_y);
#endif
return;
}
int
main(int argc, char* argv[])
{
int i, rank, npes, bug=0;
int buf[ng];
MPI_File thefile;
MPI_Status status;
MPI_Datatype filetype;
MPI_Comm new_comm;
MPI_Offset offset=0;
MPI_Info info=MPI_INFO_NULL;
int gsize[D],distrib[D],dargs[D],psize[D];
int dims[D],periods[D],reorder;
double t1,t2,mbs;
double to1,to2,tc1,tc2;
double et,eto,etc;
double max_mbs,min_mbs,avg_mbs;
double max_et,min_et,avg_et;
double max_eto,min_eto,avg_eto;
double max_etc,min_etc,avg_etc;
char process_name[MPI_MAX_PROCESSOR_NAME + 1];
char rr_blank[] = {" "};
char rr_empty[] = {"???????"};
int count;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &npes);
if ( rank == 0 )
{
if ( argc < 2 )
{
printf(" ERROR: no filename given\n");
bug++;
}
if ( npes == np )
{
printf(" file name: %s\n",argv[1]);
printf(" total number of PE's: %3d\n",np);
printf(" number of PE's in x direction: %4d\n",npx);
printf(" number of PE's in y direction: %4d\n",npy);
printf(" number of PE's in z direction: %4d\n",npz);
printf(" global grid size: %dx%dx%d 4 byte integers (total %lld)\n",X,Y,Z,(unsigned long)X*Y*Z);
printf(" local grid size: %dx%dx%d 4 byte integers (total %d)\n",nx,ny,nz,ng);
}
else
{
printf(" ERROR: total number of PE's must be %d\n",np);
printf(" actual number of PE's was %d\n",npes);
bug++;
}
if ( bug )
{
MPI_Abort(MPI_COMM_WORLD,-1);
}
}
if ( MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS)
{
sprintf(process_name, rr_empty);
}
else
{
if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count);
process_name[MAX_RR_NAME] = '\0';
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Info_create(&info);
/* allow multiple writers to write to the file concurrently */
/*MPI_Info_set(info,"panfs_concurrent_write","1");*/
/* use data aggregation */
/*MPI_Info_set(info,"romio_cb_write","enable"); */
/*MPI_Info_set(info,"romio_cb_write","disable");*/
/*MPI_Info_set(info,"romio_cb_read","enable"); */
/*MPI_Info_set(info,"romio_cb_read","disable");*/
/* use one aggregator/writer per node */
/*MPI_Info_set(info,"cb_config_list","*:1");*/
/* aggregators/writers per allocation: use this or the above (both work) */
/*i = ((npes-1)/8) + 1;
sprintf(awpa,"%d",i);
MPI_Info_set (info,"cb_nodes",awpa);*/
for ( i=0; i<ng; i++ ) buf[i] = rank*10000 + (i+1)%1024;
for ( i=0; i<D; i++ )
{
periods[i] = 1; /* true */
}
reorder = 1; /* true */
dims[0] = npx;
dims[1] = npy;
dims[2] = npz;
MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm);
for ( i=0; i<D; i++ )
{
distrib[i] = MPI_DISTRIBUTE_BLOCK;
dargs[i] = MPI_DISTRIBUTE_DFLT_DARG;
/* psize[i] = 0; */
}
gsize[0] = X;
gsize[1] = Y;
gsize[2] = Z;
psize[0] = npx;
psize[1] = npy;
psize[2] = npz;
/*
MPI_Dims_create(npes, D, psize);
printf("psize %d %d %d\n",psize[0],psize[1],psize[2]);
*/
MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT, &filetype);
/*MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_C, MPI_INT, &filetype); don't do this */
MPI_Type_commit(&filetype);
to1 = MPI_Wtime();
MPI_File_open(new_comm, argv[1], MPI_MODE_WRONLY | MPI_MODE_CREATE, info, &thefile);
to2 = MPI_Wtime();
MPI_File_set_size(thefile, offset);
MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL);
t1 = MPI_Wtime();
for ( i=0; i<LOOP; i++)
{
MPI_File_write_all(thefile, buf, ng, MPI_INT, &status);
}
t2 = MPI_Wtime();
tc1 = MPI_Wtime();
MPI_File_close(&thefile);
tc2 = MPI_Wtime();
et = (t2 - t1)/LOOP;
eto = (to2 - to1)/LOOP;
etc = (tc2 - tc1)/LOOP;
mbs = (((double)(LOOP*X*Y*Z)*sizeof(int)))/(1000000.0*(t2-t1));
/*printf(" %s[%3d] ET %8.2f %8.2f %8.2f %8.1f mbs\n", process_name, rank, t1, t2, t2-t1, mbs);*/
MPI_Barrier(MPI_COMM_WORLD);
MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
fflush(stdout);
if ( rank == 0 )
{
mbs = avg_mbs/npes;
printf("\n average write rate: %9.1f mbs\n", mbs);
printf(" minimum write rate: %9.1f mbs\n", min_mbs);
printf(" maximum write rate: %9.1f mbs\n\n", max_mbs);
avg_eto = avg_eto/npes;
avg_et = avg_et/npes;
avg_etc = avg_etc/npes;
printf(" open time: %9.3f min %9.3f avg %9.3f max\n",min_eto,avg_eto,max_eto);
printf(" write time: %9.3f min %9.3f avg %9.3f max\n",min_et,avg_et,max_et);
printf(" close time: %9.3f min %9.3f avg %9.3f max\n\n",min_etc,avg_etc,max_etc);
fflush(stdout);
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int i, rank, len, err;
int errs = 0;
char *filename, *tmp;
MPI_File fh;
char string[MPI_MAX_ERROR_STRING];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#if VERBOSE
if (!rank) {
fprintf(stderr, "Tests if errors are reported correctly...\n");
fprintf(stderr, "Should say \"Invalid displacement argument\"\n\n");
}
#endif
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: simple -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len + 10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len + 10, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len + 10);
MPI_Bcast(filename, len + 10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
/* each process opens a separate file called filename.'myrank' */
tmp = (char *) malloc(len + 10);
strcpy(tmp, filename);
sprintf(filename, "%s.%d", tmp, rank);
MPI_CHECK(MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE + MPI_MODE_RDWR,
MPI_INFO_NULL, &fh));
err = MPI_File_set_view(fh, -1, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL);
/* disp is deliberately passed as -1 */
/* This test is designed for ROMIO specifically and tests for a
* specific error message */
if (err != MPI_SUCCESS) {
MPI_Error_string(err, string, &len);
if (!rank) {
#if VERBOSE
fprintf(stderr, "%s\n", string);
#else
/* check for the word "displacement" in the message.
* This allows other formatting of the message */
if (strstr(string, "displacement") == 0) {
fprintf(stderr, "Unexpected error message %s\n", string);
errs++;
}
#endif
}
} else {
errs++;
fprintf(stderr, "File set view did not return an error\n");
}
MPI_File_close(&fh);
free(filename);
free(tmp);
if (!rank) {
if (errs == 0) {
printf(" No Errors\n");
} else {
printf(" Found %d errors\n", errs);
}
}
MPI_Finalize();
return 0;
}
uint readVectorFromEnsight( latticeMesh* mesh, scalar*** field, char* fname ) {
unsigned int status = 0;
// Open file
char name[200];
sprintf(name, "lattice.%s_%d", fname, timeToIndex(mesh->time.current));
MPI_File file;
MPI_File_open( MPI_COMM_WORLD, name, MPI_MODE_RDONLY, MPI_INFO_NULL, &file );
// Allocate space
*field = matrixDoubleAlloc(mesh->mesh.nPoints, 3, 0);
float *auxField = (float*)malloc( mesh->mesh.nPoints * sizeof(float) );
// Set Offset
MPI_Offset offset = 240*sizeof(char) + sizeof(int);
uint i,j;
for(i = 0 ; i < mesh->parallel.pid ; i++ ) {
offset += 3*mesh->parallel.nodesPerPatch[i] * sizeof(float);
offset += 160*sizeof(char) + sizeof(int);
}
MPI_File_seek(file, offset, MPI_SEEK_SET);
// Read Array
MPI_Status st;
for( j = 0 ; j < 3 ; j++) {
MPI_File_read(file, auxField, mesh->mesh.nPoints, MPI_FLOAT, &st);
for( i = 0 ; i < mesh->mesh.nPoints ; i++ ) {
field[0][i][j] = (scalar)auxField[i];
}
MPI_Barrier(MPI_COMM_WORLD);
}
MPI_File_close(&file);
free(auxField);
if( (int)st._ucount/sizeof(float) == mesh->mesh.nPoints ) {
status = 1;
}
return status;
}
int main(int argc, char **argv)
{
MPI_File fh;
int rank, len, err, i;
int errs = 0, toterrs;
char *filename;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: excl -fname filename\n\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len + 10);
strcpy(filename, *argv);
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(filename, len + 10, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len + 10);
MPI_Bcast(filename, len + 10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
if (!rank)
MPI_File_delete(filename, MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
/* this open should succeed */
err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (err != MPI_SUCCESS) {
handle_error(err, "MPI_File_open");
errs++;
fprintf(stderr, "Process %d: open failed when it should have succeeded\n", rank);
} else
MPI_File_close(&fh);
MPI_Barrier(MPI_COMM_WORLD);
/* this open should fail */
err = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (err == MPI_SUCCESS) {
errs++;
fprintf(stderr, "Process %d: open succeeded when it should have failed\n", rank);
MPI_File_close(&fh);
}
MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (rank == 0) {
if (toterrs > 0) {
fprintf(stderr, "Found %d errors\n", toterrs);
} else {
fprintf(stdout, " No Errors\n");
}
}
free(filename);
MPI_Finalize();
return 0;
}