void FORTRAN_API mpi_file_write_all_(MPI_Fint *fh,void *buf,int *count, MPI_Datatype *datatype,MPI_Status *status, int *ierr ){ MPI_File fh_c; fh_c = MPI_File_f2c(*fh); *ierr = MPI_File_write_all(fh_c,buf,*count,*datatype,status); }
void seissol::checkpoint::mpio::Wavefield::write(double time, int timestepWaveField) { EPIK_TRACER("CheckPoint_write"); SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Writing check point."; // Write the header writeHeader(time, timestepWaveField); // Save data EPIK_USER_REG(r_write_wavefield, "checkpoint_write_wavefield"); SCOREP_USER_REGION_DEFINE(r_write_wavefield); EPIK_USER_START(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); checkMPIErr(setDataView(file())); checkMPIErr(MPI_File_write_all(file(), dofs(), numDofs(), MPI_DOUBLE, MPI_STATUS_IGNORE)); EPIK_USER_END(r_write_wavefield); SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Writing check point. Done."; }
void mpi_file_write_all_(MPI_Fint *fh,void *buf,int *count, MPI_Fint *datatype,MPI_Status *status, int *ierr ){ MPI_File fh_c; MPI_Datatype datatype_c; fh_c = MPI_File_f2c(*fh); datatype_c = MPI_Type_f2c(*datatype); *ierr = MPI_File_write_all(fh_c,buf,*count,datatype_c,status); }
PetscErrorCode MPIU_File_write_all(MPI_File fd,void *data,PetscMPIInt cnt,MPI_Datatype dtype,MPI_Status *status) { PetscErrorCode ierr; PetscDataType pdtype; PetscFunctionBegin; ierr = PetscMPIDataTypeToPetscDataType(dtype,&pdtype);CHKERRQ(ierr); ierr = PetscByteSwap(data,pdtype,cnt);CHKERRQ(ierr); ierr = MPI_File_write_all(fd,data,cnt,dtype,status);CHKERRQ(ierr); ierr = PetscByteSwap(data,pdtype,cnt);CHKERRQ(ierr); PetscFunctionReturn(0); }
JNIEXPORT void JNICALL Java_mpi_File_writeAll( JNIEnv *env, jobject jthis, jlong fh, jobject buf, jboolean db, jint off, jint count, jlong jType, jint bType, jlongArray stat) { MPI_Datatype type = (MPI_Datatype)jType; void *ptr; ompi_java_buffer_t *item; ompi_java_getReadPtr(&ptr, &item, env, buf, db, off, count, type, bType); MPI_Status status; int rc = MPI_File_write_all((MPI_File)fh, ptr, count, type, &status); ompi_java_exceptionCheck(env, rc); ompi_java_releaseReadPtr(ptr, item, buf, db); ompi_java_status_set(env, stat, &status); }
void seissol::checkpoint::mpio::Wavefield::write(const void* header, size_t headerSize) { SCOREP_USER_REGION("CheckPoint_write", SCOREP_USER_REGION_TYPE_FUNCTION); logInfo(rank()) << "Checkpoint backend: Writing."; // Write the header writeHeader(header, headerSize); // Save data SCOREP_USER_REGION_DEFINE(r_write_wavefield); SCOREP_USER_REGION_BEGIN(r_write_wavefield, "checkpoint_write_wavefield", SCOREP_USER_REGION_TYPE_COMMON); checkMPIErr(setDataView(file())); unsigned int totalIter = totalIterations(); unsigned int iter = iterations(); unsigned int count = dofsPerIteration(); if (m_useLargeBuffer) { totalIter = (totalIter + sizeof(real) - 1) / sizeof(real); iter = (iter + sizeof(real) - 1) / sizeof(real); count *= sizeof(real); } unsigned long offset = 0; for (unsigned int i = 0; i < totalIter; i++) { if (i == iter-1) // Last iteration count = numDofs() - (iter-1) * count; checkMPIErr(MPI_File_write_all(file(), const_cast<real*>(&dofs()[offset]), count, MPI_DOUBLE, MPI_STATUS_IGNORE)); if (i < iter-1) offset += count; // otherwise we just continue writing the last chunk over and over else if (i != totalIter-1) checkMPIErr(MPI_File_seek(file(), -count * sizeof(real), MPI_SEEK_CUR)); } SCOREP_USER_REGION_END(r_write_wavefield); // Finalize the checkpoint finalizeCheckpoint(); logInfo(rank()) << "Checkpoint backend: Writing. Done."; }
int savematrix_rows(MPI_File *fh, float *data, int numrows, int rank, int numtasks, int m, int n) { MPI_Datatype darray; MPI_Status status; int gsizes[2] = {m, n}; int distribs[2] = {MPI_DISTRIBUTE_BLOCK, MPI_DISTRIBUTE_BLOCK}; int dargs[2] = {MPI_DISTRIBUTE_DFLT_DARG, MPI_DISTRIBUTE_DFLT_DARG}; int psizes[2] = {4, 1}; MPI_Type_create_darray(numtasks, rank, 2, gsizes, distribs, dargs, psizes, MPI_ORDER_C, MPI_FLOAT, &darray); MPI_Type_commit(&darray); MPI_File_set_view(*fh, 0, MPI_FLOAT, darray, "native", MPI_INFO_NULL); MPI_File_write_all(*fh, data, numrows*n, MPI_FLOAT, &status); MPI_Type_free(&darray); return 0; }
int main(int argc, char **argv) { int ierr, rank, size; MPI_Offset offset; MPI_File file; MPI_Status status; int npts=200; int start; int locnpts; float *data; MPI_Datatype viewtype; ierr = MPI_Init(&argc, &argv); ierr|= MPI_Comm_size(MPI_COMM_WORLD, &size); ierr|= MPI_Comm_rank(MPI_COMM_WORLD, &rank); locnpts = npts/size; start = locnpts * rank; if (rank == size-1) locnpts = (npts-start); data = malloc(locnpts * sizeof(float)); for (int i=0; i<locnpts; i++) data[i] = sin((start+i)*1.0*8*atan(1.)/npts); MPI_File_open(MPI_COMM_WORLD, "sine.dat", MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &file); /* Other MPI-IO stuff */ MPI_File_write_all(file, data, locnpts, MPI_FLOAT, &status); MPI_File_close(&file); free(data); MPI_Finalize(); return 0; }
void MPIIO_WriteData(simulation_data *sim, char *Filename) { int dimuids[3]={sim->global_dims[2], sim->global_dims[1], sim->global_dims[0]}; int f, rc, ustart[3], ucount[3]; MPI_Offset disp = 0; long offset; MPI_File filehandle; MPI_Datatype filetype; rc = MPI_File_open(sim->comm_cart, Filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &filehandle); ustart[2] = sim->grid.Ncolumns * sim->coords[0]; ustart[1] = sim->grid.Nrows * sim->coords[1]; ustart[0] = 0; ucount[2] = sim->grid.Ncolumns; ucount[1] = sim->grid.Nrows; ucount[0] = sim->grid.Nlevels; // Create the subarray representing the local block MPI_Type_create_subarray(3, dimuids, ucount, ustart, MPI_ORDER_C, MPI_FLOAT, &filetype); MPI_Type_commit(&filetype); for(f=0; f < NFIELDS; f++) { MPI_File_set_view(filehandle, disp, MPI_FLOAT, filetype, "native", MPI_INFO_NULL); MPI_File_write_all(filehandle, sim->grid.data[f], ucount[0]*ucount[1]*ucount[2], MPI_FLOAT, MPI_STATUS_IGNORE); disp += sim->global_dims[2] * sim->global_dims[1] * sim->global_dims[0] * sizeof(float); } MPI_File_close(&filehandle); MPI_Type_free(&filetype); }
int main(int argc, char *argv[]) { int iarrayOfSizes[2], iarrayOfSubsizes[2], iarrayOfStarts[2], ilocal_size; int nproc[2], periods[2], icoord[2]; int m, n, i, j, wsize, wrank, crank, ndims, lrows, lcols, grow, gcol, err; MPI_Datatype filetype; MPI_File fh; MPI_Comm cartcomm; MPI_Info info0, info3; double t, topen, twrite, tclose, wrate; double *local_array; char nstripesStr[12], stripeUnitStr[12]; int nstripes = -1; int stripeUnit = -1; MPI_Offset headerSize = 0; MPI_Init(0,0); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); /* Get global array size */ m = n = 128; /* Set default size */ /* ioda [ n ] [ m ] [ nstripes ] [ stripeunit ] [ headersize ] */ if (argc > 0) { if (argc > 1) m = atoi(argv[1]); if (argc > 2) n = atoi(argv[2]); if (argc > 3) nstripes = atoi(argv[3]); if (argc > 4) stripeUnit = atoi(argv[4]); if (argc > 5) headerSize = atoi(argv[5]); if (argc > 6) { if (wrank == 0) fprintf(stderr,"Unrecognized argument %s\n", argv[6]); MPI_Abort(MPI_COMM_WORLD,1); } } if (wrank == 0) printf("Matrix is [%d,%d]; file dir = %s\n", m, n, MYSCRATCHDIR ); /* The default number of stripes = totalsize/1M */ if (nstripes < 0) { nstripes = n * m * sizeof(double) / (1024*1024); if (nstripes < 1) nstripes = 1; } if (wrank == 0) printf("nstripes = %d, stripeUnit = %d, header size = %d\n", nstripes, stripeUnit, (int)headerSize); /* Use topology routines to get decomposition and coordinates */ MPI_Comm_size(MPI_COMM_WORLD, &wsize); nproc[0] = 0; nproc[1] = 0; ndims = 2; MPI_Dims_create(wsize, ndims, nproc); periods[0] = 0; periods[1] = 0; MPI_Cart_create(MPI_COMM_WORLD, ndims, nproc, periods, 1, &cartcomm); MPI_Comm_rank(cartcomm, &crank); MPI_Cart_coords(cartcomm, crank, ndims, icoord); iarrayOfSizes[0] = m; iarrayOfSizes[1] = n; iarrayOfSubsizes[0] = m/nproc[0]; iarrayOfSubsizes[1] = n/nproc[1]; iarrayOfStarts[0] = icoord[0] * iarrayOfSubsizes[0]; iarrayOfStarts[1] = icoord[1] * iarrayOfSubsizes[1]; /* Initialize my block of the data */ ilocal_size = iarrayOfSubsizes[0] * iarrayOfSubsizes[1]; lrows = iarrayOfSubsizes[0]; lcols = iarrayOfSubsizes[1]; local_array = (double *)malloc(lrows*lcols*sizeof(double)); gcol = iarrayOfStarts[1]; grow = iarrayOfStarts[0]; for (i=0; i<lrows; i++) { for (j=0; j<lcols; j++) { local_array[j*lrows+i] = (grow+i) + (gcol+j)*m; } } /* Fortran order simply means the data is stored by columns */ MPI_Type_create_subarray(ndims, iarrayOfSizes, iarrayOfSubsizes, iarrayOfStarts, MPI_ORDER_FORTRAN, MPI_DOUBLE, &filetype); MPI_Type_commit(&filetype); info0 = MPI_INFO_NULL; info3 = MPI_INFO_NULL; if (nstripes > 0 || stripeUnit > 0) { MPI_Info_create(&info0); if (nstripes > 0) { snprintf(nstripesStr, sizeof(nstripesStr), "%d", nstripes); MPI_Info_set(info0, "striping_factor", nstripesStr); MPI_Info_set(info0, "cb_nodes", nstripesStr); } if (stripeUnit > 0) { snprintf(stripeUnitStr, sizeof(stripeUnitStr), "%d", stripeUnit); MPI_Info_set(info0, "striping_unit", stripeUnitStr); } MPI_Info_dup(info0, &info3); MPI_Info_set(info3, "romio_no_indep_rw", "true"); /* Other hints to consider: direct_io=true The default cb_buffer_size is 16777216 , but is overridden by the striping unit, which is smaller by default. */ } /* level - 3 */ MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-3.out", MPI_MODE_CREATE | MPI_MODE_RDWR, info3, &fh); topen = MPI_Wtime() - t; if (err != MPI_SUCCESS) myAbort(err, "open testfile-3.out"); if (headerSize > 0) { /* Simulate writing a header */ if (wrank == 0) { char *header; header = (char *)calloc(1,(size_t)headerSize); MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); free(header); } MPI_Barrier(cartcomm); } MPI_File_set_view(fh, headerSize, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL); MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); err = MPI_File_write_all(fh, local_array, ilocal_size, MPI_DOUBLE, MPI_STATUS_IGNORE); twrite = MPI_Wtime() - t; if (err != MPI_SUCCESS) myAbort(err, "collective write"); err = MPI_File_close(&fh); tclose = MPI_Wtime() - t; /* tclose is the time for the write(s) + the close, in case the implementation delays (some of) the writes until the close */ if (err != MPI_SUCCESS) myAbort(err, "close testfile-3.out"); MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if (twrite > 0) wrate = (double)m * (double)n * sizeof(double)/twrite; if (wrank == 0) printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen, twrite, tclose, wrate); /* level - 0 */ MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-0.out", MPI_MODE_CREATE | MPI_MODE_RDWR, info0, &fh); topen = MPI_Wtime() - t; if (err != MPI_SUCCESS) myAbort(err, "open testfile-0.out"); if (headerSize > 0) { /* Simulate writing a header */ if (wrank == 0) { char *header; header = (char *)calloc(1,(size_t)headerSize); MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); free(header); } MPI_Barrier(cartcomm); } MPI_Barrier(MPI_COMM_WORLD); t = MPI_Wtime(); gcol = iarrayOfStarts[1]; grow = iarrayOfStarts[0]; for (j=0; j<lcols; j++) { MPI_Offset offset = headerSize + ((MPI_Offset)(grow) + (MPI_Offset)(gcol+j)*m) * sizeof(double); err = MPI_File_write_at(fh, offset, local_array+j*lrows, lrows, MPI_DOUBLE, MPI_STATUS_IGNORE); if (err != MPI_SUCCESS) myAbort(err, "write at"); } twrite = MPI_Wtime() - t; err = MPI_File_close(&fh); tclose = MPI_Wtime() - t; /* tclose is the time for the write(s) + the close, in case the implementation delays (some of) the writes until the close */ if (err != MPI_SUCCESS) myAbort(err, "close testfile-0"); MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if (twrite > 0) wrate = (double)m * (double)n * sizeof(double)/twrite; if (wrank == 0) printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen, twrite, tclose, wrate); if (info0 != MPI_INFO_NULL) { MPI_Info_free(&info0); MPI_Info_free(&info3); } free(local_array); MPI_Finalize(); return 0; }
int main(int argc, char* argv[]) { int i, rank, npes, bug=0; int buf[ng]; MPI_File thefile; MPI_Status status; MPI_Datatype filetype; MPI_Comm new_comm; MPI_Offset offset=0; MPI_Info info=MPI_INFO_NULL; int gsize[D],distrib[D],dargs[D],psize[D]; int dims[D],periods[D],reorder; double t1,t2,mbs; double to1,to2,tc1,tc2; double et,eto,etc; double max_mbs,min_mbs,avg_mbs; double max_et,min_et,avg_et; double max_eto,min_eto,avg_eto; double max_etc,min_etc,avg_etc; char process_name[MPI_MAX_PROCESSOR_NAME + 1]; char rr_blank[] = {" "}; char rr_empty[] = {"???????"}; int count; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &npes); if ( rank == 0 ) { if ( argc < 2 ) { printf(" ERROR: no filename given\n"); bug++; } if ( npes == np ) { printf(" file name: %s\n",argv[1]); printf(" total number of PE's: %3d\n",np); printf(" number of PE's in x direction: %4d\n",npx); printf(" number of PE's in y direction: %4d\n",npy); printf(" number of PE's in z direction: %4d\n",npz); printf(" global grid size: %dx%dx%d 4 byte integers (total %lld)\n",X,Y,Z,(unsigned long)X*Y*Z); printf(" local grid size: %dx%dx%d 4 byte integers (total %d)\n",nx,ny,nz,ng); } else { printf(" ERROR: total number of PE's must be %d\n",np); printf(" actual number of PE's was %d\n",npes); bug++; } if ( bug ) { MPI_Abort(MPI_COMM_WORLD,-1); } } if ( MPI_Get_processor_name(process_name, &count) != MPI_SUCCESS) { sprintf(process_name, rr_empty); } else { if (count < MAX_RR_NAME) strncat(&process_name[count],rr_blank,MAX_RR_NAME-count); process_name[MAX_RR_NAME] = '\0'; } MPI_Barrier(MPI_COMM_WORLD); MPI_Info_create(&info); /* allow multiple writers to write to the file concurrently */ /*MPI_Info_set(info,"panfs_concurrent_write","1");*/ /* use data aggregation */ /*MPI_Info_set(info,"romio_cb_write","enable"); */ /*MPI_Info_set(info,"romio_cb_write","disable");*/ /*MPI_Info_set(info,"romio_cb_read","enable"); */ /*MPI_Info_set(info,"romio_cb_read","disable");*/ /* use one aggregator/writer per node */ /*MPI_Info_set(info,"cb_config_list","*:1");*/ /* aggregators/writers per allocation: use this or the above (both work) */ /*i = ((npes-1)/8) + 1; sprintf(awpa,"%d",i); MPI_Info_set (info,"cb_nodes",awpa);*/ for ( i=0; i<ng; i++ ) buf[i] = rank*10000 + (i+1)%1024; for ( i=0; i<D; i++ ) { periods[i] = 1; /* true */ } reorder = 1; /* true */ dims[0] = npx; dims[1] = npy; dims[2] = npz; MPI_Cart_create(MPI_COMM_WORLD, D, dims, periods, reorder, &new_comm); for ( i=0; i<D; i++ ) { distrib[i] = MPI_DISTRIBUTE_BLOCK; dargs[i] = MPI_DISTRIBUTE_DFLT_DARG; /* psize[i] = 0; */ } gsize[0] = X; gsize[1] = Y; gsize[2] = Z; psize[0] = npx; psize[1] = npy; psize[2] = npz; /* MPI_Dims_create(npes, D, psize); printf("psize %d %d %d\n",psize[0],psize[1],psize[2]); */ MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_FORTRAN, MPI_INT, &filetype); /*MPI_Type_create_darray(npes, rank, D, gsize, distrib, dargs, psize, MPI_ORDER_C, MPI_INT, &filetype); don't do this */ MPI_Type_commit(&filetype); to1 = MPI_Wtime(); MPI_File_open(new_comm, argv[1], MPI_MODE_WRONLY | MPI_MODE_CREATE, info, &thefile); to2 = MPI_Wtime(); MPI_File_set_size(thefile, offset); MPI_File_set_view(thefile, offset, MPI_INT, filetype, "native", MPI_INFO_NULL); t1 = MPI_Wtime(); for ( i=0; i<LOOP; i++) { MPI_File_write_all(thefile, buf, ng, MPI_INT, &status); } t2 = MPI_Wtime(); tc1 = MPI_Wtime(); MPI_File_close(&thefile); tc2 = MPI_Wtime(); et = (t2 - t1)/LOOP; eto = (to2 - to1)/LOOP; etc = (tc2 - tc1)/LOOP; mbs = (((double)(LOOP*X*Y*Z)*sizeof(int)))/(1000000.0*(t2-t1)); /*printf(" %s[%3d] ET %8.2f %8.2f %8.2f %8.1f mbs\n", process_name, rank, t1, t2, t2-t1, mbs);*/ MPI_Barrier(MPI_COMM_WORLD); MPI_Reduce(&mbs, &avg_mbs, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&mbs, &min_mbs, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); MPI_Reduce(&mbs, &max_mbs, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); MPI_Reduce(&et, &avg_et, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&et, &min_et, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); MPI_Reduce(&et, &max_et, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); MPI_Reduce(&eto, &avg_eto, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&eto, &min_eto, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); MPI_Reduce(&eto, &max_eto, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); MPI_Reduce(&etc, &avg_etc, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&etc, &min_etc, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); MPI_Reduce(&etc, &max_etc, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); fflush(stdout); if ( rank == 0 ) { mbs = avg_mbs/npes; printf("\n average write rate: %9.1f mbs\n", mbs); printf(" minimum write rate: %9.1f mbs\n", min_mbs); printf(" maximum write rate: %9.1f mbs\n\n", max_mbs); avg_eto = avg_eto/npes; avg_et = avg_et/npes; avg_etc = avg_etc/npes; printf(" open time: %9.3f min %9.3f avg %9.3f max\n",min_eto,avg_eto,max_eto); printf(" write time: %9.3f min %9.3f avg %9.3f max\n",min_et,avg_et,max_et); printf(" close time: %9.3f min %9.3f avg %9.3f max\n\n",min_etc,avg_etc,max_etc); fflush(stdout); } MPI_Finalize(); return 0; }
void step4(inst i, int r, int s) { inst instance = i; int rank = r; int size = s; // Creation of the 2D torus we will then use MPI_Comm comm; int dim[2] = {instance.p, instance.q}; int period[2] = {1, 1}; int reorder = 0; int coord[2]; MPI_Cart_create(MPI_COMM_WORLD, 2, dim, period, reorder, &comm); MPI_Cart_coords(comm, rank, 2, coord); grid global_grid; char type = 0; MPI_File input_file; // We start by reading the header of the file MPI_File_open(comm, instance.input_path, MPI_MODE_RDONLY, MPI_INFO_NULL, &input_file); MPI_File_read_all(input_file, &type, 1, MPI_CHAR, MPI_STATUS_IGNORE); if(type == 1) { if (rank == 0) fprintf(stderr, "Error: type 1 files are not supported in step 4\n"); MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); exit(EXIT_FAILURE); } // we needed to swap the next 2 lines MPI_File_read_all(input_file, &(global_grid.n), 1, MPI_UINT64_T, MPI_STATUS_IGNORE); MPI_File_read_all(input_file, &(global_grid.m), 1, MPI_UINT64_T, MPI_STATUS_IGNORE); #ifdef DEBUG if(rank == 0) printf("n, m = %zu %zu\n", global_grid.n, global_grid.m); #endif if(!(global_grid.n % instance.p == 0 && global_grid.m % instance.q == 0)) { if(rank == 0) fprintf(stderr, "Error: please choose the grid parameters so they divide the grid of the cellular automaton. For example %zu %zu, but you need to move from %d procs to %zu\n", instance.p + (global_grid.n % instance.p), instance.q + (global_grid.m % instance.q), size, (instance.p + (global_grid.n % instance.p))*(instance.q + (global_grid.m % instance.q))); MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); exit(EXIT_FAILURE); } size_t local_nrows = global_grid.n/instance.p; size_t local_ncols = global_grid.m/instance.q; // Now we create the data structures. int blocks[2] = {1, 2}; MPI_Datatype types[2] = {MPI_BYTE, MPI_DOUBLE}; MPI_Aint a_size = sizeof(cell2); MPI_Aint a_disp[3] = {offsetof(cell2, type), offsetof(cell2, u), offsetof(cell2, s)}; MPI_Aint p_size = 17; MPI_Aint p_disp[3] = {0, 1, 9}; MPI_Datatype p_tmp, a_tmp, p_cell, a_cell; // Aligned struct, memory representation MPI_Type_create_struct(2, blocks, a_disp, types, &a_tmp); MPI_Type_create_resized(a_tmp, 0, a_size, &a_cell); MPI_Type_commit(&a_cell); // Packed struct, file-based representation MPI_Type_create_struct(2, blocks, p_disp, types, &p_tmp); MPI_Type_create_resized(p_tmp, 0, p_size, &p_cell); MPI_Type_commit(&p_cell); // Now, we create our matrix MPI_Datatype matrix; int sizes[2] = {global_grid.n, global_grid.m}; int subsizes[2] = {local_nrows, local_ncols}; int starts[2] = {0, 0}; MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, p_cell, &matrix); MPI_Type_commit(&matrix); // We extend this matrix MPI_Datatype ematrix; int e_subsizes[2] = {2 + subsizes[0], 2 + subsizes[1]}; int e_start[2] = {1, 1}; MPI_Type_create_subarray(2, e_subsizes, subsizes, e_start, MPI_ORDER_C, a_cell, &ematrix); MPI_Type_commit(&ematrix); // The next 3 types are for the export of the grid MPI_Datatype d_type; MPI_Type_create_resized(MPI_DOUBLE, 0, sizeof(cell2), &d_type); MPI_Type_commit(&d_type); MPI_Datatype d_matrix; MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, MPI_DOUBLE, &d_matrix); MPI_Type_commit(&d_matrix); MPI_Datatype d_rmatrix; // to go from the extended matrix with ghost zones to the other one MPI_Type_create_subarray(2, e_subsizes, subsizes, e_start, MPI_ORDER_C, d_type, &d_rmatrix); MPI_Type_commit(&d_rmatrix); // Set file view for each element MPI_Offset grid_start; MPI_File_get_position(input_file, &grid_start); MPI_File_set_view(input_file, grid_start + global_grid.m*local_nrows*p_size*coord[0] + local_ncols*p_size*coord[1], p_cell, matrix, "native", MPI_INFO_NULL); // allocate the cell array we will use cell2 **cells; cells = malloc(2*sizeof(cell2 *)); double *sensors; cells[1] = calloc((2+local_nrows)*(2+local_ncols),sizeof(cell2)); cells[0] = calloc((2+local_nrows)*(2+local_ncols),sizeof(cell2)); sensors = calloc(local_nrows*local_ncols, sizeof(double)); MPI_File_read_all(input_file, cells[0], 1, ematrix, MPI_STATUS_IGNORE); MPI_File_close(&input_file); #ifdef DEBUG for(size_t i = 1; i < 1+local_nrows; i++) for(size_t j = 1; j < 1+local_ncols; j++) fprintf(stderr, "%d - %d %f\n", rank, cells[0][i*(2+local_ncols)+j].type, cells[0][i*(2+local_ncols)+j].u); #endif MPI_Datatype l_row; // local row MPI_Type_contiguous(local_ncols, d_type, &l_row); MPI_Type_commit(&l_row); MPI_Datatype l_col; // local column. A bit trickier, we need a type_vector. MPI_Type_vector(local_nrows, 1, local_ncols+2, d_type, &l_col); MPI_Type_commit(&l_col); int top, bot, left, right; double sqspeed = 0; int curr = 0, next = 0; char *alldump = malloc(256); for(int s = 0; s < instance.iteration; s++) { // We will update cell[next], and use the data of cell[curr] curr = s % 2; next = (s+1) % 2; // We copy the edges of the grid. // We first need the ranks of the neighbours MPI_Cart_shift(comm, 0, 1, &top, &bot); MPI_Cart_shift(comm, 1, 1, &left, &right); // Then we need to update the edges of our local grid // Update top and bottom rows MPI_Sendrecv(&(cells[curr][1*(local_ncols+2)+1].u), 1, l_row, top, 0, &(cells[curr][(local_ncols+2)*(local_nrows+1)+1].u), 1, l_row, bot, 0, comm, MPI_STATUS_IGNORE); MPI_Sendrecv(&(cells[curr][(local_ncols+2)*(local_nrows)+1].u), 1, l_row, bot, 0, &(cells[curr][1].u), 1, l_row, top, 0, comm, MPI_STATUS_IGNORE); // Update left and right MPI_Sendrecv(&(cells[curr][1*(local_ncols+2)+1].u), 1, l_col, left, 0, &(cells[curr][1*(local_ncols+2)+local_ncols+1].u), 1, l_col, right, 0, comm, MPI_STATUS_IGNORE); MPI_Sendrecv(&(cells[curr][1*(local_ncols+2)+local_ncols].u), 1, l_col, right, 0, &(cells[curr][1*(local_ncols+2)].u), 1, l_col, left, 0, comm, MPI_STATUS_IGNORE); // We compute the update of the grid for(size_t i = 1; i < 1+local_nrows; i++) { for(size_t j = 1; j < 1+local_ncols; j++) { if(instance.step < 2 || cells[next][j+i*(2+local_ncols)].type != 1) { // If walls we do not do anything sqspeed = cells[0][j+i*(2+local_ncols)].s * cells[0][j+i*(2+local_ncols)].s; cells[next][j+i*(2+local_ncols)].u = cells[curr][j+i*(2+local_ncols)].u + (cells[curr][j+i*(2+local_ncols)].v * instance.dt); cells[next][j+i*(2+local_ncols)].v = cells[curr][j+i*(2+local_ncols)].v + sqspeed * (cells[curr][j+(i+1)*(2+local_ncols)].u + cells[curr][j+(i-1)*(2+local_ncols)].u + cells[curr][(j+1) + i*(2+local_ncols)].u + cells[curr][(j-1) + i*(2+local_ncols)].u - (4 * cells[curr][j+i*(2+local_ncols)].u)) * instance.dt; if(instance.step == 3 && cells[next][j+i*(2+local_ncols)].type == 2) { // Case of sensors sensors[(j-1)+(i-1)*local_ncols] += cells[next][j+i*(2+local_ncols)].u * cells[next][j+i*(2+local_ncols)].u; } } } } if(instance.alldump != NULL && s % instance.frequency == 0) { MPI_File dump_file; sprintf(alldump, instance.alldump, (s / instance.frequency)); MPI_File_open(comm, alldump, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &dump_file); MPI_File_set_view(dump_file, global_grid.m*local_nrows*sizeof(double)*coord[0] + local_ncols*sizeof(double)*coord[1], MPI_DOUBLE, d_matrix, "native", MPI_INFO_NULL); MPI_File_write_all(dump_file, &(cells[curr][0].u), 1, d_rmatrix, MPI_STATUS_IGNORE); MPI_File_close(&dump_file); } } if(instance.lastdump != NULL) { // bon, comment on fait ça ? peut être qu'en faisant un resize ça marche ? MPI_File last_file; MPI_File_open(comm, instance.lastdump, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &last_file); MPI_File_set_view(last_file, global_grid.m*local_nrows*sizeof(double)*coord[0] + local_ncols*sizeof(double)*coord[1], MPI_DOUBLE, d_matrix, "native", MPI_INFO_NULL); // déjà, il y a un grid_strat en trop, d_type ou MPI_DOUBLE ? MPI_File_write_all(last_file, &(cells[next][0].u), 1, d_rmatrix, MPI_STATUS_IGNORE); MPI_File_close(&last_file); } if(instance.step == 3 && instance.sensors != NULL) { MPI_File sensor_file; MPI_File_open(comm, instance.sensors, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &sensor_file); MPI_Datatype string; MPI_Type_contiguous(1024, MPI_CHAR, &string); MPI_Type_commit(&string); char text[1024]; for(size_t i = 1; i < 1+local_nrows; i++) { for(size_t j = 1; j < 1+local_ncols; j++) { if(instance.step == 3 && cells[next][j+i*(2+local_ncols)].type == 2) { memset(text,0,sizeof(text)); sprintf(text, "%zu %zu %f\r\n", (i-1)+coord[0]*local_nrows, (j-1)+coord[1]*local_ncols, sensors[(j-1)+(i-1)*local_ncols]); MPI_File_write(sensor_file, text, 1, string, MPI_STATUS_IGNORE); } } } MPI_Type_free(&string); MPI_File_close(&sensor_file); } // Some cleaning free(cells); free(alldump); MPI_Type_free(&a_cell); MPI_Type_free(&p_cell); MPI_Type_free(&matrix); MPI_Type_free(&ematrix); MPI_Type_free(&d_type); MPI_Type_free(&d_matrix); MPI_Type_free(&d_rmatrix); MPI_Type_free(&l_row); MPI_Type_free(&l_col); }
int main(int argc, char *argv[]) { int my_id, nprocs; int mpi_dims[4]; int period[4] = {0, 0, 0, 0}; int coords[4]; int dimsf[4] = {nbands, gpts, gpts, gpts}; int count[4]; int offset[4]; int ndims = 4; double t0, t1; #ifdef PAPI PAPI_dmem_info_t dmem; double mem1, mem2, mem1_max, mem2_max, mem1_ave, mem2_ave; int papi_err; #endif double *my_data; MPI_Comm cart_comm; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); assert(argc == 5); for (int i=1; i < argc; i++) mpi_dims[i-1] = atoi(argv[i]); assert(mpi_dims[0] * mpi_dims[1] * mpi_dims[2] * mpi_dims[3] == nprocs); MPI_Cart_create(MPI_COMM_WORLD, 4, mpi_dims, period, 0, &cart_comm); MPI_Comm_rank(cart_comm, &my_id); MPI_Cart_coords(cart_comm, my_id, 4, coords); assert(nbands % mpi_dims[0] == 0); for (int i=1; i < 4; i++) assert(gpts % mpi_dims[i] == 0); int total_size = nbands*gpts*gpts*gpts; count[0] = nbands / mpi_dims[0]; offset[0] = coords[0] * count[0]; int data_size = count[0]; for (int i=1; i < 4; i++) { count[i] = gpts/mpi_dims[i]; offset[i] = coords[i] * count[i]; data_size *= count[i]; } my_data = (double *) malloc(data_size * sizeof(double)); for (int i=0; i < data_size; i++) my_data[i] = my_id; MPI_Info info; MPI_File fp; MPI_Datatype filetype; // MPI_Info_set(info, "cb_nodes", "64"); MPI_Barrier(MPI_COMM_WORLD); #ifdef PAPI papi_err = PAPI_get_dmem_info(&dmem); if (papi_err != PAPI_OK) printf("PAPI_ERR\n"); mem1 = (double)dmem.size / 1024.0; MPI_Reduce(&mem1, &mem1_max, 1, MPI_DOUBLE, MPI_MAX, 0, cart_comm); MPI_Reduce(&mem1, &mem1_ave, 1, MPI_DOUBLE, MPI_SUM, 0, cart_comm); mem1_ave /= nprocs; #endif t0 = MPI_Wtime(); MPI_File_open(MPI_COMM_WORLD, "test.dat", MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &fp); MPI_Type_create_subarray(ndims, dimsf, count, offset, MPI_ORDER_C, MPI_DOUBLE, &filetype); MPI_Type_commit(&filetype); MPI_File_set_view(fp, 0, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL); MPI_File_write_all(fp, my_data, data_size, MPI_DOUBLE, MPI_STATUS_IGNORE); MPI_Type_free(&filetype); MPI_File_close(&fp); MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); #ifdef PAPI papi_err = PAPI_get_dmem_info(&dmem); if (papi_err != PAPI_OK) printf("PAPI_ERR\n"); mem2 = (double)dmem.size/ 1024.0; MPI_Reduce(&mem2, &mem2_max, 1, MPI_DOUBLE, MPI_MAX, 0, cart_comm); MPI_Reduce(&mem2, &mem2_ave, 1, MPI_DOUBLE, MPI_SUM, 0, cart_comm); mem2_ave /= nprocs; #endif if (my_id == 0) { printf("IO time %f (%f) MB %f s\n", total_size * 8/(1024.0*1024.0), data_size * 8/(1024.0*1024.0), t1-t0); #ifdef PAPI printf("Memory usage max (ave): %f (%f) %f (%f) \n", mem1_max, mem1_ave, mem2_max, mem2_ave); #endif } MPI_Finalize(); }
int main (int argc, char **argv) { MPI_Request request; MPI_File fh; MPI_Datatype ftype; MPI_Offset offset; MPI_Status status; int rank, wsize, fsize, i; char file_name[128]; int buf[BUF_SIZE * TEST_OPS]; int count; MPI_Init (&argc, &argv); MPI_Comm_size (MPI_COMM_WORLD, &wsize); MPI_Comm_rank (MPI_COMM_WORLD, &rank); strcpy (file_name, argv[0]); strcat (file_name, ".tmp"); MPI_File_open (MPI_COMM_WORLD, file_name, MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fh); fsize = wsize * BUF_SIZE * TEST_OPS; MPI_File_preallocate (fh, fsize); memset (buf, 0, BUF_SIZE * TEST_OPS); offset = 0; count = BLOCK_SIZE; for (i = 0; i < TEST_OPS; i++) { offset = i * BLOCK_SIZE + (rank * BLOCK_SIZE * TEST_OPS); MPI_File_seek (fh, offset, MPI_SEEK_SET); MPI_File_write (fh, buf, count, MPI_INT, &status); MPI_File_seek (fh, offset, MPI_SEEK_SET); MPI_File_read (fh, buf, count, MPI_INT, &status); } for (i = 0; i < TEST_OPS; i++) { offset = i * BLOCK_SIZE + (rank * BLOCK_SIZE * TEST_OPS); MPI_File_write_at (fh, offset, buf, count, MPI_INT, &status); MPI_File_read_at (fh, offset, buf, count, MPI_INT, &status); } MPI_Type_vector (fsize / BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE * wsize, MPI_INT, &ftype); MPI_Type_commit (&ftype); offset = rank * BLOCK_SIZE * TEST_OPS; count = BLOCK_SIZE * TEST_OPS; MPI_File_set_view (fh, offset, MPI_INT, ftype, "native", MPI_INFO_NULL); MPI_File_write_all (fh, buf, count, MPI_INT, &status); MPI_File_read_all (fh, buf, count, MPI_INT, &status); MPI_File_close (&fh); MPI_Finalize (); }
int main (int argc, char *argv[]) { double all_start, all_end, all_total; double comm_start, comm_end, comm_total; double ior_start, ior_end, ior_total, iow_start, iow_end, iow_total, io_total; double compute_start, compute_end, compute_total; all_total = 0; comm_total = 0; io_total = 0; ior_total = 0; iow_total = 0; compute_total = 0; int i, j, k; // Initial MPI environment /* rank: the ID of each process size: # total available process */ int rank, size; MPI_Init(&argc, &argv); all_start = MPI_Wtime(); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Argument exception handle if(argc < 4) { if (rank == ROOT) { fprintf(stderr, "Insufficient args\n"); fprintf(stderr, "Usage: %s N input_file", argv[0]); } MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return 0; } // Argument assigned const int N = atoi(argv[1]); const char *input = argv[2]; const char *output = argv[3]; // Part 1: Determine arguments of the I/O, and do MPI I/O /* num_per_node: # number stored in each process rank_first_add: the first process which need to add MAX_INT num_first_add: # number that the first process which need to add read_file: indicate the process if it need to read file *node_arr: local number array of each process */ MPI_File ifh; MPI_Status istatus; int num_per_node, rank_first_add, num_first_add, read_file, *node_arr; read_file=0; compute_start = MPI_Wtime(); if(N<size){ // N < #process num_per_node = 1; num_first_add = 1; rank_first_add = N; if(rank<rank_first_add) read_file = 1; } else{ // N >= #process if(N%size){ // If N can't be divided into the # process num_per_node = (N/size) + 1; rank_first_add = N/num_per_node; // # element to be add in the first rank num_first_add = num_per_node - (N%num_per_node); if(rank<=rank_first_add) read_file = 1; } else{ // If N can be divided into # process num_per_node = N/size; rank_first_add = size; // no need to add num_first_add = 0; read_file = 1; } } compute_end = MPI_Wtime(); compute_total += (compute_end - compute_start); node_arr = (int*) malloc(num_per_node * sizeof(int)); // store the N/P numbers in each node ior_start = MPI_Wtime(); MPI_File_open(MPI_COMM_WORLD, input, MPI_MODE_RDONLY, MPI_INFO_NULL, &ifh); if(read_file) MPI_File_set_view(ifh, rank*num_per_node*sizeof(int), MPI_INT, MPI_INT, "native", MPI_INFO_NULL); else MPI_File_set_view(ifh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL); if(read_file && rank!=rank_first_add) MPI_File_read_all(ifh, node_arr, num_per_node, MPI_INT, &istatus); else if(read_file && rank==rank_first_add){ MPI_File_read_all(ifh, node_arr, (num_per_node-num_first_add), MPI_INT, &istatus); for(i=(num_per_node-num_first_add); i<num_per_node; ++i) node_arr[i] = MAX_INT; } else{ MPI_File_read_all(ifh, node_arr, 0, MPI_INT, &istatus); for(i=0; i<num_per_node; ++i) node_arr[i] = MAX_INT; } MPI_File_close(&ifh); ior_end = MPI_Wtime(); ior_total = (ior_end - ior_start); io_total += ior_total; MPI_Barrier(MPI_COMM_WORLD); // Part 2: Start odd-even sort algorithm MPI_Status status; int *next_arr, *merge_arr, *ori_arr; next_arr = (int*) malloc(num_per_node * sizeof(int)); compute_start = MPI_Wtime(); Mergesort(num_per_node, node_arr); compute_end = MPI_Wtime(); compute_total += (compute_end - compute_start); MPI_Barrier(MPI_COMM_WORLD); for(i=0; i<size; ++i){ if(i%2==0){ // Even-phase if(rank%2){ // Odd-rank process: # 1, 3, 5...(Sender) => P -> P-1 comm_start = MPI_Wtime(); MPI_Send(node_arr, num_per_node, MPI_INT, rank-1, 8, MPI_COMM_WORLD); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); comm_start = MPI_Wtime(); MPI_Recv(node_arr, num_per_node, MPI_INT, rank-1, 8, MPI_COMM_WORLD, &status); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); } else{ // Even-rank process: # 0, 2, 4...(Receiver) => P-1 <- P if(rank!=size-1){ comm_start = MPI_Wtime(); MPI_Recv(next_arr, num_per_node, MPI_INT, rank+1, 8, MPI_COMM_WORLD, &status); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); compute_start = MPI_Wtime(); merge_arr = (int*) malloc(num_per_node * 2 * sizeof(int)); for(j=0; j<num_per_node; ++j){ merge_arr[j] = node_arr[j]; merge_arr[j+num_per_node] = next_arr[j]; } Mergesort(num_per_node*2, merge_arr); for(j=0; j<num_per_node; ++j){ node_arr[j] = merge_arr[j]; next_arr[j] = merge_arr[j+num_per_node]; } free(merge_arr); compute_end = MPI_Wtime(); compute_total += (compute_end - compute_start); comm_start = MPI_Wtime(); MPI_Send(next_arr, num_per_node, MPI_INT, rank+1, 8, MPI_COMM_WORLD); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); } } } else{ // Odd-phase if(rank%2==0){ // Even-rank process: # 0, 2, 4... (Sender) => Q -> Q-1 if(rank!=0){ comm_start = MPI_Wtime(); MPI_Send(node_arr, num_per_node, MPI_INT, rank-1, 8, MPI_COMM_WORLD); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); comm_start = MPI_Wtime(); MPI_Recv(node_arr, num_per_node, MPI_INT, rank-1, 8, MPI_COMM_WORLD, &status); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); } } else{ // Odd-rank process: # 1, 3, 5... (Receiver) => Q-1 <- Q if(rank!=size-1){ comm_start = MPI_Wtime(); MPI_Recv(next_arr, num_per_node, MPI_INT, rank+1, 8, MPI_COMM_WORLD, &status); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); compute_start = MPI_Wtime(); merge_arr = (int*) malloc(num_per_node * 2 * sizeof(int)); for(j=0; j<num_per_node; ++j){ merge_arr[j] = node_arr[j]; merge_arr[j+num_per_node] = next_arr[j]; } Mergesort(num_per_node*2, merge_arr); for(j=0; j<num_per_node; ++j){ node_arr[j] = merge_arr[j]; next_arr[j] = merge_arr[j+num_per_node]; } free(merge_arr); compute_end = MPI_Wtime(); compute_total += (compute_end - compute_start); comm_start = MPI_Wtime(); MPI_Send(next_arr, num_per_node, MPI_INT, rank+1, 8, MPI_COMM_WORLD); comm_end = MPI_Wtime(); comm_total += (comm_end - comm_start); } } } } MPI_Barrier(MPI_COMM_WORLD); free(next_arr); MPI_File ofh; MPI_Status ostatus; iow_start = MPI_Wtime(); MPI_File_open(MPI_COMM_WORLD, output, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &ofh); if(read_file) MPI_File_set_view(ofh, rank*num_per_node*sizeof(int), MPI_INT, MPI_INT, "native", MPI_INFO_NULL); else MPI_File_set_view(ofh, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL); if(read_file && rank!=rank_first_add) MPI_File_write_all(ofh, node_arr, num_per_node, MPI_INT, &ostatus); else if(read_file && rank==rank_first_add) MPI_File_write_all(ofh, node_arr, (num_per_node-num_first_add), MPI_INT, &ostatus); else MPI_File_write_all(ofh, node_arr, 0, MPI_INT, &ostatus); MPI_File_close(&ofh); iow_end = MPI_Wtime(); iow_total = (iow_end - iow_start); io_total += iow_total; all_end = MPI_Wtime(); all_total = (all_end - all_start); printf("Rank:%d (All:%lf, I:%lf, W:%lf, I/O:%lf, Comm:%lf, Compute:%lf)\n", rank, all_total, ior_total, iow_total, io_total, comm_total, compute_total); MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { MPI_Datatype newtype; int i, ndims, array_of_gsizes[3], array_of_distribs[3]; int order, nprocs, len, flag, err; int array_of_dargs[3], array_of_psizes[3]; int *readbuf, *writebuf, bufcount, mynod; char filename[1024]; MPI_File fh; MPI_Status status; MPI_Aint size_with_aint; MPI_Offset size_with_offset; MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &mynod); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); /* process 0 takes the file name as a command-line argument and broadcasts it to other processes */ if (!mynod) { i = 1; while ((i < argc) && strcmp("-fname", *argv)) { i++; argv++; } if (i >= argc) { printf("\n*# Usage: large_array -fname filename\n\n"); MPI_Abort(MPI_COMM_WORLD, 1); } argv++; len = strlen(*argv); strcpy(filename, *argv); MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD); printf("This program creates a 4 Gbyte file. Don't run it if you don't have that much disk space!\n"); } else { MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD); } /* create the distributed array filetype */ ndims = 3; order = MPI_ORDER_C; array_of_gsizes[0] = 1024; array_of_gsizes[1] = 1024; array_of_gsizes[2] = 4*1024/sizeof(int); array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK; array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK; array_of_distribs[2] = MPI_DISTRIBUTE_BLOCK; array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG; array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG; array_of_dargs[2] = MPI_DISTRIBUTE_DFLT_DARG; for (i=0; i<ndims; i++) array_of_psizes[i] = 0; MPI_Dims_create(nprocs, ndims, array_of_psizes); /* check if MPI_Aint is large enough for size of global array. if not, complain. */ size_with_aint = sizeof(int); for (i=0; i<ndims; i++) size_with_aint *= array_of_gsizes[i]; size_with_offset = sizeof(int); for (i=0; i<ndims; i++) size_with_offset *= array_of_gsizes[i]; if (size_with_aint != size_with_offset) { printf("Can't use an array of this size unless the MPI implementation defines a 64-bit MPI_Aint\n"); MPI_Abort(MPI_COMM_WORLD, 1); } MPI_Type_create_darray(nprocs, mynod, ndims, array_of_gsizes, array_of_distribs, array_of_dargs, array_of_psizes, order, MPI_INT, &newtype); MPI_Type_commit(&newtype); /* initialize writebuf */ MPI_Type_size(newtype, &bufcount); bufcount = bufcount/sizeof(int); writebuf = (int *) malloc(bufcount * sizeof(int)); if (!writebuf) printf("Process %d, not enough memory for writebuf\n", mynod); for (i=0; i<bufcount; i++) writebuf[i] = mynod*1024 + i; /* write the array to the file */ MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL); MPI_File_write_all(fh, writebuf, bufcount, MPI_INT, &status); MPI_File_close(&fh); free(writebuf); /* now read it back */ readbuf = (int *) calloc(bufcount, sizeof(int)); if (!readbuf) printf("Process %d, not enough memory for readbuf\n", mynod); MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh); MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL); MPI_File_read_all(fh, readbuf, bufcount, MPI_INT, &status); MPI_File_close(&fh); /* check the data read */ flag = 0; for (i=0; i<bufcount; i++) if (readbuf[i] != mynod*1024 + i) { printf("Process %d, readbuf=%d, writebuf=%d\n", mynod, readbuf[i], mynod*1024 + i); flag = 1; } if (!flag) printf("Process %d: data read back is correct\n", mynod); MPI_Type_free(&newtype); free(readbuf); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { err = MPI_File_delete(filename, MPI_INFO_NULL); if (err == MPI_SUCCESS) printf("file deleted\n"); } MPI_Finalize(); return 0; }
void parallel_readwrite(char *file_name, void *dump_buffer, int type_of_file, int is_write, long long offset) { #if MPI && DO_PARALLEL_WRITE MPI_File fh; MPI_Status status; MPI_Datatype mpi_elementary_type, mpi_file_type; int file_open_error, file_write_error ; int error_string_length; char error_string[BUFSIZ]; MPI_Offset file_size; int count; void *mpi_buffer; size_t mpi_buffer_size; int mode; MPI_Offset mpi_offset; MPI_Barrier(MPI_COMM_WORLD); if (is_write) { mode = MPI_MODE_CREATE | MPI_MODE_WRONLY | MPI_MODE_APPEND; } else { mode = MPI_MODE_RDONLY; } file_open_error = MPI_File_open(MPI_COMM_WORLD, file_name, mode, MPI_INFO_NULL, &fh); if (file_open_error != MPI_SUCCESS) { MPI_Error_string(file_open_error, error_string, &error_string_length); fprintf(stderr, "parallel_readwrite(): error opening file: %3d: %s\n", mpi_rank, error_string); MPI_Abort(MPI_COMM_WORLD, file_open_error); /* It is still OK to abort, because we have failed to open the file. */ } else { // if (i_am_the_master) // chmod(file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (offset < 0L) { if(is_write) { MPI_File_get_position(fh, &mpi_offset); offset = mpi_offset; } else { offset = 0L; } } MPI_Barrier(MPI_COMM_WORLD); //differentiate data type and buffers involved based on file type if( DUMP_FILE == type_of_file ) { mpi_elementary_type = MPI_DUMP_TYPE; mpi_file_type = dump_file_type; mpi_buffer = (void*)dump_buffer; mpi_buffer_size = dump_buffer_size; } else if( GDUMP_FILE == type_of_file){ mpi_elementary_type = MPI_GDUMP_TYPE; mpi_file_type = gdump_file_type; mpi_buffer = (void*)gdump_buffer; mpi_buffer_size = gdump_buffer_size; } else if( GDUMP2_FILE == type_of_file){ mpi_elementary_type = MPI_GDUMP2_TYPE; mpi_file_type = gdump2_file_type; mpi_buffer = (void*)gdump2_buffer; mpi_buffer_size = gdump2_buffer_size; } else if( RDUMP_FILE == type_of_file){ mpi_elementary_type = MPI_RDUMP_TYPE; mpi_file_type = rdump_file_type; mpi_buffer = (void*)rdump_buffer; mpi_buffer_size = rdump_buffer_size; } else if( FDUMP_FILE == type_of_file){ mpi_elementary_type = MPI_FDUMP_TYPE; mpi_file_type = fdump_file_type; mpi_buffer = (void*)fdump_buffer; mpi_buffer_size = fdump_buffer_size; } else { if(i_am_the_master) fprintf(stderr, "Unknown file type %d\n", type_of_file); MPI_File_close(&fh); MPI_Finalize(); exit(2); } MPI_File_set_view(fh, offset, mpi_elementary_type, mpi_file_type, "native", MPI_INFO_NULL); if (is_write) { file_write_error = MPI_File_write_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type, &status); } else { file_write_error = MPI_File_read_all(fh, mpi_buffer, mpi_buffer_size, mpi_elementary_type, &status); } if (file_write_error != MPI_SUCCESS) { MPI_Error_string(file_write_error, error_string, &error_string_length); fprintf(stderr, "parallel_readwrite(): error %s file: %3d: %s\n", (is_write)?("writing"):("reading"), mpi_rank, error_string); MPI_File_close(&fh); //if (i_am_the_master) MPI_File_delete(file_name, MPI_INFO_NULL); MPI_Finalize(); exit(1); } // MPI_Get_count(&status, MPI_FLOAT, &count); // MPI_File_get_size(fh, &file_size); // if(1) { // printf("%3d: wrote %d floats, expected to write %lld floats\n", mpi_rank, count, (long long int)dump_buffer_size); // printf("%3d: file size is %lld bytes, header-related offset is %lld\n", mpi_rank, file_size, offset); // } MPI_File_close(&fh); } #endif }
void qpb_write_spinor(qpb_spinor_field spinor_field, char fname[]) { MPI_Datatype mpi_dtype_spinor_float, filetype; MPI_Type_contiguous(2*NC*NS, MPI_FLOAT, &mpi_dtype_spinor_float); MPI_Type_commit(&mpi_dtype_spinor_float); int starts[ND], l_dim[ND], g_dim[ND]; for(int i=0; i<ND; i++) { starts[i] = problem_params.coords[i]*problem_params.l_dim[i]; l_dim[i] = problem_params.l_dim[i]; g_dim[i] = problem_params.g_dim[i]; }; int ierr = MPI_Type_create_subarray(ND, g_dim, l_dim, starts, MPI_ORDER_C, mpi_dtype_spinor_float, &filetype); MPI_Type_commit(&filetype); MPI_File fhandle; ierr = MPI_File_open(MPI_COMM_WORLD, fname, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &fhandle); if(ierr != MPI_SUCCESS) { if(am_master) { fprintf(stderr, "%s: MPI_File_open() returned in error\n", fname); exit(QPB_FILE_ERROR); } } ierr = MPI_File_set_view(fhandle, 0, mpi_dtype_spinor_float, filetype, "native", MPI_INFO_NULL); if(ierr != MPI_SUCCESS) { if(am_master) { fprintf(stderr, "%s: MPI_File_set_view() returned in error\n", fname); exit(QPB_FILE_ERROR); } } void *buffer = qpb_alloc(problem_params.l_vol*sizeof(qpb_spinor_float)); for(int v=0; v<problem_params.l_vol; v++) for(int sp=0; sp<NC*NS; sp++) { ((float *) buffer)[v*NC*NS*2 + sp*2] = spinor_field.bulk[v][sp].re; ((float *) buffer)[v*NC*NS*2 + sp*2 + 1] = spinor_field.bulk[v][sp].im; } if(!qpb_is_bigendian()) qpb_byte_swap_float(buffer, problem_params.l_vol*NC*NS*2); MPI_Status status; ierr = MPI_File_write_all(fhandle, buffer, problem_params.l_vol, mpi_dtype_spinor_float, &status); if(ierr != MPI_SUCCESS) { if(am_master) { fprintf(stderr, "%s: MPI_File_read() returned in error\n", fname); exit(QPB_FILE_ERROR); } } ierr = MPI_File_close(&fhandle); if(ierr != MPI_SUCCESS) { if(am_master) { fprintf(stderr, "%s: MPI_File_close() returned in error\n", fname); exit(QPB_FILE_ERROR); } } free(buffer); return; }
int main(int argc, char** argv) { int my_rank, p; int i, dest; mpz_t currentPrime; unsigned long int product; sscanf(argv[1], "%lu", &product); int secondFactor = 0; int bcastStatus; int equals; /** GMP library variables **/ mpz_t nextPrimeNumber; mpz_t testFactor; mpz_init(nextPrimeNumber); mpz_init_set_str (nextPrimeNumber, argv[1], 10); mpz_init(testFactor); mpz_init_set_ui(currentPrime, 2); mpz_nextprime(nextPrimeNumber, nextPrimeNumber); mpz_t testProduct; mpz_init(testProduct); /** MPI Initialization **/ MPI_Request finalValue; MPI_File out; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &p); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Status status; /** Get Ready to receive a factor if another process finds one */ MPI_Irecv(&secondFactor, 1, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &finalValue); /** Prepare initial offset for each process **/ for (i=0 ; i < my_rank ; i++) { mpz_nextprime(currentPrime, currentPrime); } /** Start Timing **/ double start = MPI_Wtime(), diff; while (!secondFactor) { /** Check if another process has found the factors **/ MPI_Test (&finalValue, &bcastStatus, &status); if(bcastStatus) { /** Somebody else has found the factors, we are done **/ MPI_Wait(&finalValue, &status); break; } /** Skip P primes before checking again **/ for (i=0 ; i < p ; i++) { mpz_nextprime(currentPrime, currentPrime); } /** Brute force check if the current working prime is a factor of the input number **/ for (mpz_set_ui(testFactor , 2) ; mpz_get_ui(testFactor) <= mpz_get_ui(currentPrime); mpz_nextprime(testFactor, testFactor)) { /** Check if another process has found the factors **/ MPI_Test (&finalValue, &bcastStatus, &status); if(bcastStatus) { MPI_Wait(&finalValue, &status); break; } mpz_mul_ui(testProduct, currentPrime, mpz_get_ui(testFactor)); equals = mpz_cmp_ui(testProduct, product); if (equals == 0){ /** We've found the factor, find the second number, secnd it to the other processes **/ secondFactor = mpz_get_ui(testFactor); printf("done by process %d, factors are %lu and %d \n", my_rank, mpz_get_ui(currentPrime), secondFactor); fflush(stdout); for (dest = 0 ; dest < p ; dest++) { if (dest != my_rank) { MPI_Send(&secondFactor, 1, MPI_UNSIGNED_LONG, dest, 0, MPI_COMM_WORLD); } } } } } diff = MPI_Wtime() - start; /** End Timing **/ /** Prepare file contents **/ char fileName[200], fileContents[200]; sprintf(fileName, "time_%lu", product); sprintf(fileContents, "%d\t%f\n", my_rank, diff); /** Write File **/ MPI_File_open( MPI_COMM_WORLD, fileName, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &out ); MPI_File_seek(out, my_rank*strlen ( fileContents ) , MPI_SEEK_SET); MPI_File_write_all(out , &fileContents, strlen ( fileContents ), MPI_CHAR, &status ); MPI_File_close(&out); /** Fin **/ MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return(0); }
int main( int argc, char *argv[] ) { int opt; extern char *optarg; extern int optind; int is_output_timing=0, is_print_usage = 0; int _debug=0, use_gen_file = 0, use_actsto = 0, use_normalsto=0; char *token; MPI_Offset disp, offset, file_size; MPI_Datatype etype, ftype, buftype; int errs = 0; int size, rank, i, count; char *fname = NULL; double *buf; MPI_File fh; MPI_Comm comm; MPI_Status status; int64_t nitem = 0; int fsize = 0, type_size; double stime, etime, iotime, comptime, elapsed_time; double max_iotime, max_comptime; double max, min, sum=0.0, global_sum; MPI_Init( &argc, &argv ); comm = MPI_COMM_WORLD; MPI_Comm_size( comm, &size ); MPI_Comm_rank( comm, &rank ); while ( (opt=getopt(argc,argv,"i:s:godhxt"))!= EOF) { switch (opt) { case 'i': fname = optarg; break; case 'o': is_output_timing = 1; break; case 'g': use_gen_file = 1; break; case 'd': _debug = 1; break; case 'h': is_print_usage = 1; break; case 's': token = strtok(optarg, ":"); //if (rank == 0) printf("token=%s\n", token); if(token == NULL) { if (rank == 0) printf("1: Wrong file size format!\n"); MPI_Finalize(); exit(1); } fsize = atoi(token); token = strtok(NULL, ":"); //if (rank == 0) printf("token=%s\n", token); if(token == NULL) { if (rank == 0) printf("2: Wrong file size format!\n"); MPI_Finalize(); exit(1); } if(*token != 'm' && *token != 'g') { if (rank == 0) printf("3: Wrong file size format!\n"); MPI_Finalize(); exit(1); } if (rank ==0) printf("fsize = %d (%s)\n", fsize, (*token=='m'?"MB":"GB")); if (fsize == 0) nitem = 0; else { MPI_Type_size(MPI_DOUBLE, &type_size); nitem = fsize*1024; /* KB */ nitem = nitem*1024; /* MB */ if(*token == 'g') { //if(rank == 0) printf("data in GB\n"); nitem = nitem*1024; /* GB */ } nitem = nitem/type_size; //printf("nitem=%lld\n", nitem); nitem = nitem/size; /* size means comm size */ } if (rank == 0) printf("nitem = %d\n", nitem); break; case 'x': use_actsto = 1; break; case 't': use_normalsto = 1; break; default: is_print_usage = 1; break; } } if (fname == NULL || is_print_usage == 1 || nitem == 0) { if (rank == 0) usage(argv[0]); MPI_Finalize(); exit(1); } int sizeof_mpi_offset; sizeof_mpi_offset = (int)(sizeof(MPI_Offset)); // 8 //if (rank == 0) printf ("size_of_mpi_offset=%d\n", sizeof_mpi_offset); if(use_normalsto == 1 && use_actsto == 1) { if(rank == 0) printf("Can't test both: either normalsto or actsto\n"); MPI_Finalize(); exit(1); } #if 0 if(use_actsto == 1) { if (size != 1) { if(rank == 0) printf("active storage should be run with only 1 process!!!\n"); MPI_Finalize(); exit(1); } } #endif /* initialize random seed: */ srand(time(NULL)); if(use_gen_file == 1) { int t, result; MPI_File_open( comm, fname, MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fh ); /* Set the file view */ disp = rank * nitem * type_size; printf("%d: disp = %lld\n", rank, disp); etype = MPI_DOUBLE; ftype = MPI_DOUBLE; result = MPI_File_set_view(fh, disp, etype, ftype, "native", MPI_INFO_NULL); if(result != MPI_SUCCESS) sample_error(result, "MPI_File_set_view"); buf = (double *)malloc( nitem * sizeof(double) ); if (buf == NULL) { if(rank == 0) printf("malloc() failed\n"); MPI_Finalize(); exit(1); } buf[0] = rand()%4096; if(rank==0) printf("%lf\n", buf[0]); max = min = sum = buf[0]; for(i=1; i<nitem; i++) { t = rand()%4096; if (t>max) max = t; if (t<min) min = t; sum += t; buf[i] = t; if (i<10 && rank == 0) printf("%lf\n", buf[i]); } if(rank == 0) { printf("MPI_Type_size(MPI_DOUBLE)=%d\n", type_size); printf ("max=%lf, min=%lf, sum=%lf\n", max, min, sum); } stime = MPI_Wtime(); /* Write to file */ MPI_File_write_all( fh, buf, nitem, MPI_DOUBLE, &status ); etime = MPI_Wtime(); iotime = etime - stime; printf("%d: iotime (write) = %10.4f\n", rank, iotime); MPI_Get_count( &status, MPI_DOUBLE, &count ); //printf("count = %lld\n", count); if (count != nitem) { fprintf( stderr, "%d: Wrong count (%lld) on write\n", rank, count ); fflush(stderr); /* exit */ MPI_Finalize(); exit(1); } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); if(rank == 0) printf("File is written\n\n"); } double *tmp = (double *)malloc( nitem * sizeof(double) ); memset (tmp, 0, nitem*sizeof(double)); if(use_normalsto == 1) { MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh ); /* Read nothing (check status) */ memset( &status, 0xff, sizeof(MPI_Status) ); offset = rank * nitem * type_size; /* start I/O */ stime = MPI_Wtime(); MPI_File_read_at(fh, offset, tmp, nitem, MPI_DOUBLE, &status); etime = MPI_Wtime(); /* end I/O */ iotime = etime - stime; if(_debug==1) printf("%d: iotime = %10.4f\n", rank, iotime); MPI_Reduce(&iotime, &max_iotime, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); sum = 0.0; /* reset sum */ /* start computation */ stime = MPI_Wtime(); for(i=0; i<nitem; i++) { sum += tmp[i]; } MPI_Reduce(&sum, &global_sum, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); etime = MPI_Wtime(); /* end computation */ comptime = etime - stime; if(_debug==1) printf("%d: comptime = %10.4f\n", rank, comptime); MPI_Reduce(&comptime, &max_comptime, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); if(rank == 0) { elapsed_time = max_comptime + max_iotime; printf("<<Result (SUM) with normal read>>\n" "SUM = %10.4f \n" "Computation time = %10.4f sec\n" "I/O time = %10.4f sec\n" "total time = %10.4f sec\n\n", global_sum, max_comptime, max_iotime, elapsed_time); } MPI_File_close(&fh); } #if 0 if(use_actsto == 1) { #if 0 /* MPI_MAX */ MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh ); stime = MPI_Wtime(); MPI_File_read_at_ex( fh, offset, tmp, nitem, MPI_DOUBLE, MPI_MAX, &status ); etime = MPI_Wtime(); elapsed_time = etime-stime; printf ("<<Result with active storage>>\n" "max=%lf (in %10.4f sec)\n", tmp[0], elapsed_time); MPI_File_close(&fh); /* MPI_MIN */ MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh ); stime = MPI_Wtime(); MPI_File_read_at_ex( fh, offset, tmp, nitem, MPI_DOUBLE, MPI_MIN, &status ); etime = MPI_Wtime(); elapsed_time = etime - stime; printf ("min=%lf (in %10.4f sec)\n", tmp[0], elapsed_time); MPI_File_close(&fh); #endif /* MPI_SUM */ MPI_File_open( comm, fname, MPI_MODE_RDWR, MPI_INFO_NULL, &fh ); memset(&status, 0xff, sizeof(MPI_Status)); offset = rank * nitem * type_size; stime = MPI_Wtime(); MPI_File_read_at_ex( fh, offset, tmp, nitem, MPI_DOUBLE, MPI_SUM, &status ); etime = MPI_Wtime(); elapsed_time = etime - stime; printf ("<<Result with active storage>>\n" "sum=%lf (in %10.4f sec)\n", tmp[0], elapsed_time); MPI_File_close( &fh ); } #endif MPI_Barrier(MPI_COMM_WORLD); if (use_gen_file == 1) free( buf ); free( tmp ); MPI_Finalize(); return errs; }
void stamp_matrix(Lattice *grid, double *matrix, string filename) { #ifdef HAVE_MPI // Set variables for mpi output char *data_as_txt; int count; MPI_File file; MPI_Status status; // each number is represented by charspernum chars const int charspernum = 14; MPI_Datatype num_as_string; MPI_Type_contiguous(charspernum, MPI_CHAR, &num_as_string); MPI_Type_commit(&num_as_string); // create a type describing our piece of the array int globalsizes[2] = {grid->global_dim_y - 2 * grid->periods[0] * grid->halo_y, grid->global_dim_x - 2 * grid->periods[1] * grid->halo_x}; int localsizes [2] = {grid->inner_end_y - grid->inner_start_y, grid->inner_end_x - grid->inner_start_x}; int starts[2] = {grid->inner_start_y, grid->inner_start_x}; int order = MPI_ORDER_C; MPI_Datatype localarray; MPI_Type_create_subarray(2, globalsizes, localsizes, starts, order, num_as_string, &localarray); MPI_Type_commit(&localarray); // output real matrix //conversion data_as_txt = new char[(grid->inner_end_x - grid->inner_start_x) * (grid->inner_end_y - grid->inner_start_y) * charspernum]; count = 0; for (int j = grid->inner_start_y - grid->start_y; j < grid->inner_end_y - grid->start_y; j++) { for (int k = grid->inner_start_x - grid->start_x; k < grid->inner_end_x - grid->start_x - 1; k++) { sprintf(&data_as_txt[count * charspernum], "%+.5e ", matrix[j * grid->dim_x + k]); count++; } if(grid->mpi_coords[1] == grid->mpi_dims[1] - 1) { sprintf(&data_as_txt[count * charspernum], "%+.5e\n ", matrix[j * grid->dim_x + (grid->inner_end_x - grid->start_x) - 1]); count++; } else { sprintf(&data_as_txt[count * charspernum], "%+.5e ", matrix[j * grid->dim_x + (grid->inner_end_x - grid->start_x) - 1]); count++; } } // open the file, and set the view MPI_File_open(grid->cartcomm, const_cast<char*>(filename.c_str()), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file); MPI_File_set_view(file, 0, MPI_CHAR, localarray, (char *)"native", MPI_INFO_NULL); MPI_File_write_all(file, data_as_txt, (grid->inner_end_x - grid->inner_start_x) * (grid->inner_end_y - grid->inner_start_y), num_as_string, &status); MPI_File_close(&file); delete [] data_as_txt; #else print_matrix(filename.c_str(), &(matrix[grid->global_dim_x * (grid->inner_start_y - grid->start_y) + grid->inner_start_x - grid->start_x]), grid->global_dim_x, grid->global_dim_x - 2 * grid->periods[1]*grid->halo_x, grid->global_dim_y - 2 * grid->periods[0]*grid->halo_y); #endif return; }
int main (int argc, char **argv) { struct arguments arguments; /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ argp_parse (&argp, argc, argv, 0, 0, &arguments); int run_type; run_type = 0; //default is serial if (sscanf (arguments.args[0], "%i", &run_type)!=1) {} int iterations; iterations = 0; //default is serial if (sscanf (arguments.args[1], "%i", &iterations)!=1) {} int count_when; count_when = 1000; if (sscanf (arguments.args[2], "%i", &count_when)!=1) {} char print_list[200]; //used for input list if (sscanf (arguments.args[3], "%s", &print_list)!=1) {} // printf("Print list = %s\n", print_list); //Extract animation list from arguments char char_array[20][12] = { NULL }; //seperated input list int animation_list[20][2] = { NULL }; //integer input list start,range char *tok = strtok(print_list, ","); //counters int i,j,k,x,y,ii,jj; ii = 0; jj = 0; //Loop over tokens parsing our commas int tok_len = 0; while (tok != NULL) { //first loop parses out commas tok_len = strlen(tok); for (jj=0;jj<tok_len;jj++) { char_array[ii][jj] = tok[jj]; } // printf("Tok = %s\n", char_array[ii]); tok = strtok(NULL, ","); ii++; } //looking for a range input, convert to ints int stop; for (ii=0;ii<20;ii++) { //convert first number to int tok = strtok(char_array[ii], "-"); if (tok != NULL) { animation_list[ii][0] = atoi(tok); tok = strtok(NULL, ","); } //look for second number, add to range if (tok != NULL) { stop = atoi(tok); animation_list[ii][1] = stop - animation_list[ii][0]; } // if (rank == 0) // { // printf("Animation_list = %i, %i\n", // animation_list[ii][0], animation_list[ii][1]); // } } //should an animation be generated //prints a bunch of .pgm files, have to hand //make the gif... int animation; animation = arguments.animation; //verbose? int verbose; verbose = arguments.verbose; // printf("VERBOSE = %i",verbose); if (verbose>0 && verbose<=10) { verbose = 1; } // Initialize the MPI environment MPI_Init(NULL, NULL); // Get the number of processes int world_size; MPI_Comm_size(MPI_COMM_WORLD, &world_size); // Get the rank of the process int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Get the name of the processor char processor_name[MPI_MAX_PROCESSOR_NAME]; int name_len; MPI_Get_processor_name(processor_name, &name_len); //Print run information, exit on bad command line input if (rank == 0 && verbose == 1) { printf("Verbose=%i, RunType=%i, Iterations=%i, CountWhen=%i, Animation=%i\n", verbose,run_type,iterations,count_when, animation); } if (world_size>1 && run_type ==0) { printf("Runtype and processors count not consistant\n"); MPI_Finalize(); exit(0); } if (world_size==1 && run_type>0) { printf("Runtype and processors count not consistant\n"); MPI_Finalize(); exit(0); } if (count_when <= 0) { if (rank == 0) { printf("Invalid count interval, positive integers only\n"); } MPI_Finalize(); exit(0); } //serial if (world_size == 1 && run_type == 0) { ncols=1; nrows=1; } //Blocked else if (world_size>1 && run_type == 1) { ncols = 1; nrows = world_size; my_col = 0; my_row = rank; } //Checker else if (world_size>1 && run_type == 2) { ncols = (int)sqrt(world_size); nrows = (int)sqrt(world_size); my_row = rank/nrows; my_col = rank-my_row*nrows; if (ncols*nrows!=world_size) { if (rank == 0) { printf("Number of processors must be square, Exiting\n"); } MPI_Finalize(); exit(0); } } // if (verbose == 1) // { // printf("WR,row,col=%i,%i,%i\n",rank,my_row,my_col); // } //////////////////////READ IN INITIAL PGM//////////////////////////////// if(!readpgm("life.pgm")) { // printf("WR=%d,HERE2\n",rank); if( rank==0 ) { pprintf( "An error occured while reading the pgm file\n" ); } MPI_Finalize(); return 1; } // Count the life forms. Note that we count from [1,1] - [height+1,width+1]; // we need to ignore the ghost row! i = 0; for(y=1; y<local_height+1; y++ ) { for(x=1; x<local_width+1; x++ ) { if( field_a[ y * field_width + x ] ) { i++; } } } // pprintf( "%i local buggies\n", i ); int total; MPI_Allreduce( &i, &total, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); if( rank==0 && verbose == 1 ) { pprintf( "%i total buggies\n", total ); } // printf("WR=%d, Row=%d, Col=%d\n",rank,my_row,my_col); //Row and column size per processor int rsize, csize; rsize = local_width; csize = local_height; if (rank == 0 && verbose == 1) { printf("rsize,csize,NP = %d, %d, %d\n",rsize,csize,world_size); } //Create new derived datatype for writing to files MPI_Datatype submatrix; int array_of_gsizes[2]; int array_of_distribs[2]; int array_of_dargs[2]; int array_of_psize[2]; if (run_type == 1) { if (rank == 0) { printf("g0,g1 = %i,%i\n", local_height*ncols, local_width); printf("p0,p1 = %i,%i\n", nrows, ncols); } array_of_gsizes[0] = local_height*ncols; array_of_gsizes[1] = local_width; array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK; array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK; array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG; array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG; array_of_psize[0] = nrows; array_of_psize[1] = ncols; // int order = MPI_ORDER_C; //size,rank,ndims,array_gsizes,array_distribs,array_args,array_psizes //order,oldtype,*newtype MPI_Type_create_darray(world_size, rank, 2, array_of_gsizes, array_of_distribs, array_of_dargs, array_of_psize, MPI_ORDER_C, MPI_UNSIGNED_CHAR, &submatrix); MPI_Type_commit(&submatrix); } else if (run_type == 2) { if (rank == 0) { printf("g0,g1 = %i,%i\n", local_height*ncols, local_width*nrows); printf("p0,p1 = %i,%i\n", nrows, ncols); } array_of_gsizes[0] = local_height*ncols; array_of_gsizes[1] = local_width*nrows; array_of_distribs[0] = MPI_DISTRIBUTE_BLOCK; array_of_distribs[1] = MPI_DISTRIBUTE_BLOCK; array_of_dargs[0] = MPI_DISTRIBUTE_DFLT_DARG; array_of_dargs[1] = MPI_DISTRIBUTE_DFLT_DARG; array_of_psize[0] = nrows; array_of_psize[1] = ncols; // int order = MPI_ORDER_C; //size,rank,ndims,array_gsizes,array_distribs,array_args,array_psizes //order,oldtype,*newtype MPI_Type_create_darray(world_size, rank, 2, array_of_gsizes, array_of_distribs, array_of_dargs, array_of_psize, MPI_ORDER_C, MPI_UNSIGNED_CHAR, &submatrix); MPI_Type_commit(&submatrix); } MPI_Barrier(MPI_COMM_WORLD); //////////////////ALLOCATE ARRAYS, CREATE DATATYPES///////////////////// //Create new column derived datatype MPI_Datatype column; //count, blocklength, stride, oldtype, *newtype MPI_Type_hvector(csize, 1, sizeof(unsigned char), MPI_UNSIGNED_CHAR, &column); MPI_Type_commit(&column); //Create new row derived datatype MPI_Datatype row; //count, blocklength, stride, oldtype, *newtype MPI_Type_hvector(rsize, 1, sizeof(unsigned char), MPI_UNSIGNED_CHAR, &row); MPI_Type_commit(&row); //allocate arrays and corner storage unsigned char *section; unsigned char *neighbors; //to use unsigned char *top; unsigned char *bot; unsigned char *left; unsigned char *right; //to send unsigned char *ttop; unsigned char *tbot; unsigned char *tleft; unsigned char *tright; //MALLOC!! section = (unsigned char*)malloc(rsize*csize*sizeof(unsigned char)); neighbors = (unsigned char*)malloc(rsize*csize*sizeof(unsigned char)); top = (unsigned char*)malloc(rsize*sizeof(unsigned char)); bot = (unsigned char*)malloc(rsize*sizeof(unsigned char)); left = (unsigned char*)malloc(csize*sizeof(unsigned char)); right = (unsigned char*)malloc(csize*sizeof(unsigned char)); ttop = (unsigned char*)malloc(rsize*sizeof(unsigned char)); tbot = (unsigned char*)malloc(rsize*sizeof(unsigned char)); tleft = (unsigned char*)malloc(csize*sizeof(unsigned char)); tright = (unsigned char*)malloc(csize*sizeof(unsigned char)); //corners unsigned char topleft,topright,botleft,botright; //used in calculations unsigned char ttopleft,ttopright,tbotleft,tbotright; topleft = 255; topright = 255; botleft = 255; botright = 255; //used for animation, each process will put there own result in and then //each will send to process 1 which will add them up unsigned char* full_matrix; unsigned char* full_matrix_buffer; if (animation == 1) { int msize1 = rsize*ncols*csize*nrows; full_matrix = (unsigned char*)malloc(msize1*sizeof(unsigned char)); full_matrix_buffer = (unsigned char*)malloc(msize1*sizeof(unsigned char)); for (i=0; i<msize1; i++) { full_matrix[i] = 0; full_matrix_buffer[i] = 0; } } // printf("Rsize,Lsize,Fsize=%i %i %i,Csize,Lsize,Fsize=%i %i %i\n",rsize,local_width,field_width,csize,local_height,field_height); //Serial initialize vars int count = 0; if (world_size == 1 && run_type == 0) { for (i=0;i<csize;i++) { for (j=0;j<rsize;j++) { section[i*rsize + j] = 255; if (field_a[(i+1)*(2+rsize) + j + 1]) { section[i*rsize + j] = 0; count += 1; } else { section[i*rsize + j] = 255; } top[j] = 255; bot[j] = 255; ttop[j] = 255; tbot[j] = 255; } right[i] = 255; left[i] = 255; tright[i] = 255; tleft[i] = 255; } // printf("COUNT 4 = %d\n", count); } //Blocked/Checkered initializing variables else if (world_size > 1 && (run_type == 1 || run_type == 2)) { //initialize for (i=0;i<csize;i++) { for (j=0;j<rsize;j++) { section[i*rsize + j] = 255; if (field_a[(i+1)*(2+rsize) + j + 1]) { section[i*rsize + j] = 0; count += 1; } else { section[i*rsize + j] = 255; } top[j] = 255; bot[j] = 255; ttop[j] = 255; tbot[j] = 255; } right[i] = 255; left[i] = 255; tright[i] = 255; tleft[i] = 255; } // MPI_Allreduce( &count, &total, 1, MPI_UNSIGNED_CHAR, MPI_SUM, MPI_COMM_WORLD ); // if (rank == 0) // { // printf("COUNT 4 = %d\n", total); // } } //header/footer for mpio writes char header1[15]; header1[0] = 0x50; header1[1] = 0x35; header1[2] = 0x0a; header1[3] = 0x35; header1[4] = 0x31; header1[5] = 0x32; header1[6] = 0x20; header1[7] = 0x35; header1[8] = 0x31; header1[9] = 0x32; header1[10] = 0x0a; header1[11] = 0x32; header1[12] = 0x35; header1[13] = 0x35; header1[14] = 0x0a; char footer; footer = 0x0a; //make a frame or not? int create_frame = 0; //send to int send_to; int receive_from; int info[5]; info[2] = rank; info[3] = rsize; info[4] = csize; unsigned char info2[4]; info2[0] = topleft; info2[1] = topright; info2[2] = botleft; info2[3] = botright; int current_count; int location; //Gameplay for (k=0;k<iterations;k++) { //Count buggies if (k%count_when==0) { if (verbose == 1) { current_count = rsize*csize-count_buggies(rsize,csize,section); MPI_Allreduce( ¤t_count, &total, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); if (rank == 0) { printf("Iteration=%5d, Count=%6d\n", k,total); } ////corner debug // printf("WR,tl,tr,bl,br = %d %d %d %d %d\n", rank, topleft, topright, botleft, botright); } } //Write to file serially for comparison //If animation is requested if (animation == 1 && run_type == 0) { //Put smaller matrix part into larger matrix for (i=0; i<csize; i++) { for (j=0; j<rsize; j++) { location = (my_row*csize*rsize*ncols + my_col*rsize + i*rsize*ncols + j); full_matrix_buffer[location] = section[i*rsize+j]; } // if (rank == 0) // { // printf("Location = %d\n", location); // } } //Gather matrix MPI_Reduce(full_matrix_buffer, full_matrix, rsize*ncols*csize*nrows, MPI_UNSIGNED_CHAR, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && run_type == 0) { write_matrix_to_pgm(k, rsize*ncols, csize*nrows, full_matrix); } } //mpio write pgm else if (animation == 1 && (run_type == 1 || run_type == 2)) { //default is no frame create_frame = 0; for (ii=0;ii<20;ii++) { for (jj=0;jj<animation_list[ii][1]+1;jj++) { // if (rank == 0) // { // printf("a,ii,j,k= %i,%i,%i,%i, Frame? = %i\n", // animation_list[ii][0],ii,jj,k,(animation_list[ii][0]+jj-k)==0); // } if ((animation_list[ii][0] + jj - k) == 0) { create_frame = 1; break; } } } if (create_frame == 1) { //dynamic filename with leading zeroes for easy conversion to gif char buffer[128]; snprintf(buffer, sizeof(char)*128, "Animation/frame%04d.pgm", k); /* open the file, and set the view */ MPI_File file; MPI_File_open(MPI_COMM_WORLD, buffer, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &file); MPI_File_set_view(file, 0, MPI_UNSIGNED_CHAR, MPI_UNSIGNED_CHAR, "native", MPI_INFO_NULL); //write header MPI_File_write(file, &header1, 15, MPI_CHAR, MPI_STATUS_IGNORE); //write matrix MPI_File_set_view(file, 15, MPI_UNSIGNED_CHAR, submatrix, "native", MPI_INFO_NULL); MPI_File_write_all(file, section, rsize*csize, MPI_UNSIGNED_CHAR, MPI_STATUS_IGNORE); //write footer (trailing newline) MPI_File_set_view(file, 15+rsize*ncols*csize*nrows, MPI_UNSIGNED_CHAR, MPI_UNSIGNED_CHAR, "native", MPI_INFO_NULL); MPI_File_write(file, &footer, 1, MPI_CHAR, MPI_STATUS_IGNORE); } } // BLOCKED COMMUNITATION // if (run_type == 1) { //change bot (send top) to account for middle area //alternate to avoid locking send_to = rank - 1; receive_from = rank + 1; //figure out what to send //top and bottom for (i=0;i<rsize;i++) { ttop[i] = section[i]; tbot[i] = section[rsize*(csize-1)+i]; } //left n right for (i=0;i<csize;i++) { tleft[i] = section[0 + rsize*i]; tright[i] = section[rsize-1 + rsize*i]; } //send top, receive bot if (rank%2==0) { if (send_to<world_size && send_to>=0) { MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from >= 0) { MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from >= 0) { MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0) { MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD); } } //change top to account for middle area //alternate to avoid locking send_to = rank + 1; receive_from = rank - 1; //send bot, receive top if (rank%2==0) { // printf("%d, %d, %d\n", rank, send_to, receive_from); if (send_to<world_size && send_to>=0) { MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from >= 0) { MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { // printf("%d, %d, %d\n", rank, send_to, receive_from); if (receive_from<world_size && receive_from >= 0) { //*data,count,type,from,tag,comm,mpi_status MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0) { //*data,count,type,to,tag,comm MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD); } } } // CHECKERED COMMUNITATION // else if (run_type == 2) { //figure out what to send //top and bottom for (i=0;i<rsize;i++) { ttop[i] = section[i]; tbot[i] = section[rsize*(csize-1)+i]; } //left n right for (i=0;i<csize;i++) { tleft[i] = section[0 + rsize*i]; tright[i] = section[rsize-1 + rsize*i]; } //corners ttopleft = tleft[0]; tbotleft = tleft[csize-1]; ttopright = tright[0]; tbotright = tright[csize-1]; //Send top, receive bot send_to = rank - nrows; receive_from = rank + nrows; if (rank%2==0) { if (send_to<world_size && send_to>=0) { MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0) { MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0) { MPI_Recv(bot, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0) { MPI_Send(ttop, 1, row, send_to, 0, MPI_COMM_WORLD); } } //Send bot, receive top send_to = rank + nrows; receive_from = rank - nrows; if (rank%2==0) { if (send_to<world_size && send_to>=0) { MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0) { MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0) { MPI_Recv(top, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0) { MPI_Send(tbot, 1, row, send_to, 0, MPI_COMM_WORLD); } } //Send left, receive right send_to = rank - 1; receive_from = rank + 1; if (rank%2==0) { if (send_to<world_size && send_to>=0 && send_to/nrows==my_row) { MPI_Send(tleft, 1, column, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row) { MPI_Recv(right, 1, column, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row) { MPI_Recv(right, 1, column, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0 && send_to/nrows==my_row) { MPI_Send(tleft, 1, column, send_to, 0, MPI_COMM_WORLD); } } //Send right, receive left send_to = rank + 1; receive_from = rank - 1; if (rank%2==0) { if (send_to<world_size && send_to>=0 && send_to/nrows==my_row) { MPI_Send(tright, 1, row, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row) { MPI_Recv(left, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row) { MPI_Recv(left, 1, row, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0 && send_to/nrows==my_row) { MPI_Send(tright, 1, row, send_to, 0, MPI_COMM_WORLD); } } //Send topright, receive botleft send_to = rank - ncols + 1; receive_from = rank + ncols - 1; if (rank%2==0) { if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1) { MPI_Send(&ttopright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1) { MPI_Recv(&botleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1) { MPI_Recv(&botleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1) { MPI_Send(&ttopright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } } //Send topleft, receive botright send_to = rank - ncols - 1; receive_from = rank + ncols + 1; if (rank%2==0) { if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1) { MPI_Send(&ttopleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1) { MPI_Recv(&botright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row+1) { MPI_Recv(&botright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0 && send_to/nrows==my_row-1) { MPI_Send(&ttopleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } } //Send botleft, receive topright send_to = rank + ncols - 1; receive_from = rank - ncols + 1; if (rank%2==0) { if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1) { MPI_Send(&tbotleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1) { MPI_Recv(&topright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1) { MPI_Recv(&topright, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1) { MPI_Send(&tbotleft, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } } //Send botright, receive topleft send_to = rank + ncols + 1; receive_from = rank - ncols - 1; if (rank%2==0) { if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1) { MPI_Send(&tbotright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1) { MPI_Recv(&topleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } } else if (rank%2==1) { if (receive_from<world_size && receive_from>=0 && receive_from/nrows==my_row-1) { MPI_Recv(&topleft, 1, MPI_UNSIGNED_CHAR, receive_from, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (send_to<world_size && send_to>=0 && send_to/nrows==my_row+1) { MPI_Send(&tbotright, 1, MPI_UNSIGNED_CHAR, send_to, 0, MPI_COMM_WORLD); } } info2[0] = topleft; info2[1] = topright; info2[2] = botleft; info2[3] = botright; } // if (rank == 1){ // print_matrix(rsize, 1, top); // print_matrix(rsize, csize, section); // print_matrix(rsize, 1, bot); // printf("\n"); // } // printf("wr=%d,iteration=%d,maxval=%d, 11\n", rank, k,(csize-1)*rsize-1+rsize); /////////// CELL UPDATES ///////////////// //count neighbor for (i=0;i<csize;i++) { for (j=0; j<rsize; j++) { info[0] = i; info[1] = j; neighbors[i*rsize+j] = count_neighbors(info, info2, section, top, bot, left, right); // printf("%i",neighbors[i*rsize+j]); } // printf("\n"); } //update cells current_count = 0; for (i=0;i<csize;i++) { for (j=0; j<rsize; j++) { //cell currently alive if (section[i*rsize+j] == 0) { //2 or 3 neighbors lives, else die if (neighbors[i*rsize+j] < 2 || neighbors[i*rsize+j] > 3) { section[i*rsize+j] = 255; } } else { //Exactly 3 neighbors spawns new life if (neighbors[i*rsize+j] == 3) { section[i*rsize+j] = 0; } } } } } MPI_Barrier(MPI_COMM_WORLD); sleep(0.5); //free malloc stuff if( field_a != NULL ) free( field_a ); if( field_b != NULL ) free( field_b ); free(section); free(neighbors); free(top); free(bot); free(left); free(right); MPI_Finalize(); exit (0); }
void do_collective_write() { MPI_Info info; MPI_Datatype contig; MPI_Comm sub_write_comm; MPI_File fh; char coll_path[PATH_LEN]; int sub_comm_size, sub_rank, sub_comm_color; int striping_factor_int; int disp; int rc; int *buf; ptimes[0].start = MPI_Wtime(); ptimes[1].start = MPI_Wtime(); // if (m_size == 1) { // sub_write_comm = MPI_COMM_WORLD; // sub_comm_color = 0; // } else { sub_comm_color = get_sub_collective_io_comm(&sub_write_comm); // } /* Construct a datatype for distributing the input data across all * processes. */ MPI_Type_contiguous(data_size / sizeof(int), MPI_INT, &contig); MPI_Type_commit(&contig); /* Set the stripe_count and stripe_size, that is, the striping_factor * and striping_unit. Both keys and values for MPI_Info_set must be * in the form of ascii strings. */ MPI_Info_create(&info); striping_factor_int = max_striping_factor / m_size; if (striping_factor_int == 0) striping_factor_int = 1; sprintf(striping_factor, "%d", striping_factor_int); MPI_Info_set(info, "striping_factor", striping_factor); MPI_Info_set(info, "striping_unit", striping_unit); // MPI_Info_set(info, "romio_cb_write", "enable"); // MPI_Info_set(info, "romio_cb_write", "disable"); /* Get path to the target file of the communicator */ MPI_Comm_size(sub_write_comm, &sub_comm_size); get_coll_io_path(coll_path, sub_comm_color); /* Delete the output file if it exists so that striping can be set * on the output file. */ // rc = MPI_File_delete(coll_path, info); /* Create write data*/ MPI_Comm_rank(sub_write_comm, &sub_rank); buf = create_io_data(sub_rank); /* if (sub_rank == 0) { */ /* rc = MPI_File_delete(coll_path, MPI_INFO_NULL); */ /* if (rc != MPI_SUCCESS) { */ /* gio_err("MPI_File_delete failed (%s:%s:%d)", __FILE__, __func__, __LINE__); */ /* } */ /* } */ ptimes[1].end = MPI_Wtime(); MPI_Barrier(MPI_COMM_WORLD); /* Open the file */ // gio_dbg("start ***********************"); ptimes[2].start = MPI_Wtime(); rc = MPI_File_open(sub_write_comm, coll_path, MPI_MODE_WRONLY | MPI_MODE_CREATE, info, &fh); ptimes[2].end = MPI_Wtime(); if (rc != MPI_SUCCESS) { gio_err("MPI_File_open failed (%s:%s:%d)", __FILE__, __func__, __LINE__); } // gio_dbg("start *********************** %d", sub_rank); ptimes[3].start = MPI_Wtime(); /* Set the file view for the output file. In this example, we will * use the same contiguous datatype as we used for reading the data * into local memory. A better example would be to write out just * part of the data, say 4 contiguous elements followed by a gap of * 4 elements, and repeated. */ disp = sub_rank * data_size; #ifdef GIO_LARGE_FILE int i; for (i = 0; i < sub_rank; i++) { MPI_File_seek(fh, data_size, MPI_SEEK_CUR); } #else MPI_File_set_view(fh, disp, contig, contig, "native", info); #endif if (rc != MPI_SUCCESS) { gio_err("MPI_File_set_view failed (%s:%s:%d)", __FILE__, __func__, __LINE__); } ptimes[3].end = MPI_Wtime(); // gio_dbg("end ***********************"); /* MPI Collective Write */ ptimes[4].start = MPI_Wtime(); rc = MPI_File_write_all(fh, buf, 1, contig, MPI_STATUS_IGNORE); if (rc != MPI_SUCCESS) { gio_err("MPI_File_set_view failed (%s:%s:%d)", __FILE__, __func__, __LINE__); } ptimes[4].end = MPI_Wtime(); /*Free data*/ free_io_data(buf); /* Close Files */ ptimes[5].start = MPI_Wtime(); MPI_File_close(&fh); ptimes[5].end = MPI_Wtime(); ptimes[0].end = MPI_Wtime(); print_results(); return; }
int main(int argc, char* argv[]) { int n, my_rank; int array_of_subsizes[NDIMS], array_of_starts[NDIMS], array_of_sizes[NDIMS]; int size = 4; int sqrtn; int ln; MPI_Datatype filetype, memtype; MPI_File fh; char hdr[128]; int header_bytes; unsigned char *cur; char name[128]; int resultlen; int ret; int i, j; /* Initialize MPI. */ MPI_Init(&argc, &argv); MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); /* Learn my rank and the total number of processors. */ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Comm_size(MPI_COMM_WORLD, &n); /* Speak! */ MPI_Get_processor_name(name, &resultlen); printf("process %d running on %s\n", my_rank, name); /* Set up our values. */ sqrtn = (int)sqrt(n); ln = size/sqrtn; printf("n = %d, sqrtn = %d, ln = %d storage = %d\n", n, sqrtn, ln, (ln + 2) * (ln + 2)); /* Allocation storage. */ if (!(cur = calloc((ln + 2) * (ln + 2), 1))) return ERR; /* Initialize data. */ for (i = 1; i < ln + 1; i++) for (j = 1; j < ln + 1; j++) cur[i * (ln + 2) + j] = my_rank; /* Create a subarray type for the file. */ array_of_sizes[0] = array_of_sizes[1] = size; array_of_subsizes[0] = array_of_subsizes[1] = ln; array_of_starts[0] = my_rank/sqrtn * ln; array_of_starts[1] = (my_rank % sqrtn) * ln; if ((ret = MPI_Type_create_subarray(NDIMS, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_BYTE, &filetype))) MPIERR(ret); if ((ret = MPI_Type_commit(&filetype))) MPIERR(ret); /* Create a subarray type for memory. */ array_of_sizes[0] = array_of_sizes[1] = ln + 2; array_of_subsizes[0] = array_of_subsizes[1] = ln; array_of_starts[0] = array_of_starts[1] = 1; if ((ret = MPI_Type_create_subarray(NDIMS, array_of_sizes, array_of_subsizes, array_of_starts, MPI_ORDER_C, MPI_BYTE, &memtype))) MPIERR(ret); if ((ret = MPI_Type_commit(&memtype))) MPIERR(ret); MPI_File_delete(FILE_NAME, MPI_INFO_NULL); if ((ret = MPI_File_open(MPI_COMM_WORLD, FILE_NAME, MPI_MODE_CREATE|MPI_MODE_RDWR, MPI_INFO_NULL, &fh))) MPIERR(ret); /* Create header info, and have process 0 write it to the file. */ sprintf(hdr, "P5\n%d %d\n255\n", size, size); header_bytes = strlen(hdr); if ((ret = MPI_File_write_all(fh, hdr, header_bytes, MPI_BYTE, MPI_STATUS_IGNORE))) MPIERR(ret); /* Set the file view to translate our memory data into the file's data layout. */ MPI_File_set_view(fh, header_bytes, MPI_BYTE, filetype, "native", MPI_INFO_NULL); /* Write the output. */ MPI_File_write(fh, cur, 1, memtype, MPI_STATUS_IGNORE); if ((ret = MPI_File_close(&fh))) MPIERR(ret); MPI_Finalize(); return 0; }
int test_file(char *filename, int mynod, int nprocs, char * cb_hosts, const char *msg, int verbose) { MPI_Datatype typevec, newtype, t[3]; int *buf, i, b[3], errcode, errors=0; MPI_File fh; MPI_Aint d[3]; MPI_Status status; int SIZE = (STARTING_SIZE/nprocs)*nprocs; MPI_Info info; if (mynod==0 && verbose) fprintf(stderr, "%s\n", msg); buf = (int *) malloc(SIZE*sizeof(int)); if (buf == NULL) { perror("test_file"); MPI_Abort(MPI_COMM_WORLD, -1); } if (cb_hosts != NULL ) { MPI_Info_create(&info); MPI_Info_set(info, "cb_config_list", cb_hosts); } else { info = MPI_INFO_NULL; } MPI_Type_vector(SIZE/nprocs, 1, nprocs, MPI_INT, &typevec); b[0] = b[1] = b[2] = 1; d[0] = 0; d[1] = mynod*sizeof(int); d[2] = SIZE*sizeof(int); t[0] = MPI_LB; t[1] = typevec; t[2] = MPI_UB; MPI_Type_struct(3, b, d, t, &newtype); MPI_Type_commit(&newtype); MPI_Type_free(&typevec); if (!mynod) { if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n"); MPI_File_delete(filename, info); } MPI_Barrier(MPI_COMM_WORLD); errcode = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); if (errcode != MPI_SUCCESS) { handle_error(errcode, "MPI_File_open"); } MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info); for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE); errcode = MPI_File_write_all(fh, buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) { handle_error(errcode, "nc mem - nc file: MPI_File_write_all"); } MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<SIZE; i++) buf[i] = -1; errcode = MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) { handle_error(errcode, "nc mem - nc file: MPI_File_read_at_all"); } /* the verification for N compute nodes is tricky. Say we have 3 * processors. * process 0 sees: 0 -1 -1 3 -1 -1 ... * process 1 sees: -1 34 -1 -1 37 -1 ... * process 2 sees: -1 -1 68 -1 -1 71 ... */ /* verify those leading -1s exist if they should */ for (i=0; i<mynod; i++ ) { if ( buf[i] != -1 ) { if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]); errors++; } } /* now the modulo games are hairy. processor 0 sees real data in the 0th, * 3rd, 6th... elements of the buffer (assuming nprocs==3 ). proc 1 sees * the data in 1st, 4th, 7th..., and proc 2 sees it in 2nd, 5th, 8th */ for(/* 'i' set in above loop */; i<SIZE; i++) { if ( ((i-mynod)%nprocs) && buf[i] != -1) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); errors++; } if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE) ) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod,i,SIZE)); errors++; } } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { if(verbose) fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n"); MPI_File_delete(filename, info); } MPI_Barrier(MPI_COMM_WORLD); MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod,i,SIZE); errcode = MPI_File_write_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "nc mem - c file: MPI_File_write_at_all"); MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<SIZE; i++) buf[i] = -1; errcode = MPI_File_read_at_all(fh, mynod*(SIZE/nprocs)*sizeof(int), buf, 1, newtype, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "nc mem - c file: MPI_File_read_at_all"); /* just like as above */ for (i=0; i<mynod; i++ ) { if ( buf[i] != -1 ) { if(verbose) fprintf(stderr, "Process %d: buf is %d, should be -1\n", mynod, buf[i]); errors++; } } for(/* i set in above loop */; i<SIZE; i++) { if ( ((i-mynod)%nprocs) && buf[i] != -1) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); errors++; } if ( !((i-mynod)%nprocs) && buf[i] != SEEDER(mynod,i,SIZE)) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod,i,SIZE) ); errors++; } } MPI_File_close(&fh); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { if(verbose) fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n"); MPI_File_delete(filename, info); } MPI_Barrier(MPI_COMM_WORLD); MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, info, &fh); MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", info); for (i=0; i<SIZE; i++) buf[i] = SEEDER(mynod, i, SIZE); errcode = MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "c mem - nc file: MPI_File_write_all"); MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<SIZE; i++) buf[i] = -1; errcode = MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status); if (errcode != MPI_SUCCESS) handle_error(errcode, "c mem - nc file: MPI_File_read_at_all"); /* same crazy checking */ for (i=0; i<SIZE; i++) { if (buf[i] != SEEDER(mynod, i, SIZE)) { if(verbose) fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], SEEDER(mynod, i, SIZE)); errors++; } } MPI_File_close(&fh); MPI_Type_free(&newtype); free(buf); if (info != MPI_INFO_NULL) MPI_Info_free(&info); return errors; }
int main(int argc,char* argv[]) { char* params = NULL; char gauge_name[qcd_MAX_STRING_LENGTH]; char param_name[qcd_MAX_STRING_LENGTH]; char out_name[qcd_MAX_STRING_LENGTH]; qcd_int_4 x_src[4],lx_src[4],i,nsmear,nsmearAPE; qcd_real_8 alpha ,alphaAPE,plaq; int params_len; qcd_geometry geo; qcd_gaugeField u, uAPE; qcd_gaugeField *u_ptr, *uAPE_ptr, *utmp_ptr; qcd_vector vec; qcd_uint_2 P[4]; qcd_uint_2 L[4]; qcd_real_8 theta[4]={M_PI,0.,0.,0.}; // boundary conditions int myid,numprocs, namelen; char processor_name[MPI_MAX_PROCESSOR_NAME]; //set up MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD,&numprocs); // num. of processes taking part in the calculation MPI_Comm_rank(MPI_COMM_WORLD,&myid); // each process gets its ID MPI_Get_processor_name(processor_name,&namelen); // //////////////////// READ INPUT FILE ///////////////////////////////////////////// if(argc!=2) { if(myid==0) fprintf(stderr,"No input file specified\n"); exit(EXIT_FAILURE); } strcpy(param_name,argv[1]); if(myid==0) { i=0; printf("opening input file %s\n",param_name); params=qcd_getParams(param_name,¶ms_len); if(params==NULL) { i=1; } } MPI_Bcast(&i,1,MPI_INT, 0, MPI_COMM_WORLD); if(i==1) exit(EXIT_FAILURE); MPI_Bcast(¶ms_len, 1, MPI_INT, 0, MPI_COMM_WORLD); if(myid!=0) params = (char*) malloc(params_len*sizeof(char)); MPI_Bcast(params, params_len, MPI_CHAR, 0, MPI_COMM_WORLD); sscanf(qcd_getParam("<processors_txyz>",params,params_len),"%hd %hd %hd %hd",&P[0], &P[1], &P[2], &P[3]); if(P[0] != 1) { fprintf(stderr, " Must use only 1 process along t-direction, exiting...\n"); exit(EXIT_FAILURE); } sscanf(qcd_getParam("<lattice_txyz>",params,params_len),"%hd %hd %hd %hd",&L[0], &L[1], &L[2], &L[3]); if(qcd_initGeometry(&geo,L,P, theta, myid, numprocs)) exit(EXIT_FAILURE); if(myid==0) printf(" Local lattice: %i x %i x %i x %i\n",geo.lL[0],geo.lL[1],geo.lL[2],geo.lL[3]); sscanf(qcd_getParam("<source_pos_txyz>",params,params_len),"%d %d %d %d",&x_src[0], &x_src[1], &x_src[2], &x_src[3]); if(myid==0) printf(" Got source coords: %d %d %d %d\n",x_src[0],x_src[1],x_src[2],x_src[3]); sscanf(qcd_getParam("<alpha_gauss>",params,params_len),"%lf", &alpha); if(myid==0) printf(" Got alpha_gauss: %lf\n",alpha); sscanf(qcd_getParam("<nsmear_gauss>",params,params_len),"%d",&nsmear); if(myid==0) printf(" Got nsmear_gauss: %d\n",nsmear); sscanf(qcd_getParam("<alpha_APE>",params,params_len),"%lf",&alphaAPE); if(myid==0) printf(" Got alpha_APE: %lf\n",alphaAPE); sscanf(qcd_getParam("<nsmear_APE>",params,params_len),"%d",&nsmearAPE); if(myid==0) printf(" Got nsmear_APE: %d\n",nsmearAPE); strcpy(gauge_name,qcd_getParam("<cfg_name>",params,params_len)); if(myid==0) printf(" Got conf name: %s\n",gauge_name); strcpy(out_name,qcd_getParam("<src_block_name>",params,params_len)); if(myid==0) printf(" Got out name: %s\n",out_name); free(params); /////////////////////////////////////////////////////////////////////////////////////////////////// if(nsmear != 0) { qcd_initGaugeField(&u,&geo); qcd_initGaugeField(&uAPE,&geo); if(qcd_getGaugeField(gauge_name,qcd_GF_LIME,&u)) { fprintf(stderr,"process %i: Error reading gauge field!\n",myid); exit(EXIT_FAILURE); } if(myid==0) printf("gauge-field loaded\n"); plaq = qcd_calculatePlaquette(&u); if(myid==0) printf("plaquette = %e\n",plaq); u_ptr = &u; uAPE_ptr = &uAPE; for(i=0; i<nsmearAPE; i++) { qcd_apeSmear3d(uAPE_ptr, u_ptr, alphaAPE); utmp_ptr=u_ptr; u_ptr=uAPE_ptr; uAPE_ptr=utmp_ptr; } utmp_ptr=u_ptr; u_ptr=uAPE_ptr; uAPE_ptr=utmp_ptr; //reverse the last swap. Also needed when nsmearAPE=0 qcd_destroyGaugeField(u_ptr); uAPE = *uAPE_ptr; if(myid==0) printf("gauge-field APE-smeared\n"); plaq = qcd_calculatePlaquette(&uAPE); if(myid==0) printf("plaquette = %e\n",plaq); //qcd_initPropagator(&source,&geo); } qcd_initVector(&vec,&geo); // which process has the source coords? for(i=0; i<4; i++) lx_src[i] = x_src[i]/geo.lL[i]; qcd_zeroVector(&vec); if( (lx_src[0]==geo.Pos[0]) && (lx_src[1]==geo.Pos[1]) && (lx_src[2]==geo.Pos[2]) && (lx_src[3]==geo.Pos[3]) ) { vec.D[qcd_LEXIC((x_src[0]%geo.lL[0]), (x_src[1]%geo.lL[1]), (x_src[2]%geo.lL[2]), (x_src[3]%geo.lL[3]), geo.lL)][0][0].re=1.; } for(i=0; i<nsmear; i++) { if(qcd_gaussIteration3d(&vec,&uAPE,alpha,x_src[0])) { fprintf(stderr,"process %i: Error while smearing!\n",geo.myid); exit(EXIT_FAILURE); } } qcd_real_8 *src = malloc(sizeof(qcd_real_8)*geo.lL[1]*geo.lL[2]*geo.lL[3]); if(src == NULL) { fprintf(stderr, "process %i: malloc returned NULL!\n",geo.myid); exit(EXIT_FAILURE); } if( lx_src[0]==geo.Pos[0] ) { for(int z=0; z<geo.lL[3]; z++) for(int y=0; y<geo.lL[2]; y++) for(int x=0; x<geo.lL[1]; x++) { int lv = qcd_LEXIC((x_src[0]%geo.lL[0]), x, y, z, geo.lL); double loc_norm = 0; for(int mu=0; mu<4; mu++) for(int c=0; c<3; c++) { loc_norm += qcd_NORMSQUARED(vec.D[lv][mu][c]); } src[x+geo.lL[1]*(y+z*geo.lL[2])] = loc_norm; } } qcd_destroyVector(&vec); /* * This assumes only 1 process in time * If not, anything may happen */ MPI_Datatype fileview; MPI_File fh; MPI_Status status; int globv3[] = {geo.L[3], geo.L[2], geo.L[1]}; int locv3[] = {geo.lL[3], geo.lL[2], geo.lL[1]}; int starts[] = {geo.Pos[3]*locv3[0], geo.Pos[2]*locv3[1], geo.Pos[1]*locv3[2]}; MPI_Type_create_subarray(3, globv3, locv3, starts, MPI_ORDER_C, MPI_DOUBLE, &fileview); MPI_Type_commit(&fileview); MPI_File_open(MPI_COMM_WORLD, out_name, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &fh); MPI_File_set_size(fh, 0); MPI_File_set_view(fh, 0, MPI_DOUBLE, fileview, "native", MPI_INFO_NULL); if(!qcd_isBigEndian()) { qcd_swap_8(src, (geo.lL[1]*geo.lL[2]*geo.lL[3])); } MPI_File_write_all(fh, src, (geo.lL[1]*geo.lL[2]*geo.lL[3]), MPI_DOUBLE, &status); MPI_File_close(&fh); free(src); if(nsmear != 0) qcd_destroyGaugeField(&uAPE); ////////////////////////////////////// CLEAN UP AND EXIT /////////////////////////////////////////// //qcd_destroyPropagator(&source); qcd_destroyGeometry(&geo); MPI_Finalize(); return(EXIT_SUCCESS); }//end main
int main(int argc, char **argv) { int info, i, j, pcol, Adim; double *D; int *DESCD; CSRdouble BT_i, B_j, Xsparse, Zsparse, Btsparse; /*BT_i.allocate(0,0,0); B_j.allocate(0,0,0); Xsparse.allocate(0,0,0); Zsparse.allocate(0,0,0); Btsparse.allocate(0,0,0);*/ //Initialise MPI and some MPI-variables info = MPI_Init ( &argc, &argv ); if ( info != 0 ) { printf ( "Error in MPI initialisation: %d\n",info ); return info; } position= ( int* ) calloc ( 2,sizeof ( int ) ); if ( position==NULL ) { printf ( "unable to allocate memory for processor position coordinate\n" ); return EXIT_FAILURE; } dims= ( int* ) calloc ( 2,sizeof ( int ) ); if ( dims==NULL ) { printf ( "unable to allocate memory for grid dimensions coordinate\n" ); return EXIT_FAILURE; } //BLACS is the interface used by PBLAS and ScaLAPACK on top of MPI blacs_pinfo_ ( &iam,&size ); //determine the number of processes involved info=MPI_Dims_create ( size, 2, dims ); //determine the best 2D cartesian grid with the number of processes if ( info != 0 ) { printf ( "Error in MPI creation of dimensions: %d\n",info ); return info; } //Until now the code can only work with square process grids //So we try to get the biggest square grid possible with the number of processes involved if (*dims != *(dims+1)) { while (*dims * *dims > size) *dims -=1; *(dims+1)= *dims; if (iam==0) printf("WARNING: %d processor(s) unused due to reformatting to a square process grid\n", size - (*dims * *dims)); size = *dims * *dims; //cout << "New size of process grid: " << size << endl; } blacs_get_ ( &i_negone,&i_zero,&ICTXT2D ); //Initialisation of the BLACS process grid, which is referenced as ICTXT2D blacs_gridinit_ ( &ICTXT2D,"R",dims, dims+1 ); if (iam < size) { //The rank (iam) of the process is mapped to a 2D grid: position= (process row, process column) blacs_pcoord_ ( &ICTXT2D,&iam,position, position+1 ); if ( *position ==-1 ) { printf ( "Error in proces grid\n" ); return -1; } //Filenames, dimensions of all matrices and other important variables are read in as global variables (see src/readinput.cpp) info=read_input ( *++argv ); if ( info!=0 ) { printf ( "Something went wrong when reading input file for processor %d\n",iam ); return -1; } //blacs_barrier is used to stop any process of going beyond this point before all processes have made it up to this point. blacs_barrier_ ( &ICTXT2D,"ALL" ); if ( * ( position+1 ) ==0 && *position==0 ) printf ( "Reading of input-file succesful\n" ); if ( * ( position+1 ) ==0 && *position==0 ) { printf("\nA linear mixed model with %d observations, %d genotypes, %d random effects and %d fixed effects\n", n,k,m,l); printf("was analyzed using %d (%d x %d) processors\n",size,*dims,*(dims+1)); } //Dimension of A (sparse matrix) is the number of fixed effects(m) + the sparse random effects (l) Adim=m+l; //Dimension of D (dense matrix) is the number of dense effects (k) Ddim=k; pcol= * ( position+1 ); //Define number of blocks needed to store a complete column/row of D Dblocks= Ddim%blocksize==0 ? Ddim/blocksize : Ddim/blocksize +1; //Define the number of rowblocks needed by the current process to store its part of the dense matrix D Drows= ( Dblocks - *position ) % *dims == 0 ? ( Dblocks- *position ) / *dims : ( Dblocks- *position ) / *dims +1; Drows= Drows<1? 1 : Drows; //Define the number of columnblocks needed by the current process to store its part of the dense matrix D Dcols= ( Dblocks - pcol ) % * ( dims+1 ) == 0 ? ( Dblocks- pcol ) / * ( dims+1 ) : ( Dblocks- pcol ) / * ( dims+1 ) +1; Dcols=Dcols<1? 1 : Dcols; //Define the local leading dimension of D (keeping in mind that matrices are always stored column-wise) lld_D=Drows*blocksize; //Initialise the descriptor of the dense distributed matrix DESCD= ( int* ) malloc ( DLEN_ * sizeof ( int ) ); if ( DESCD==NULL ) { printf ( "unable to allocate memory for descriptor for C\n" ); return -1; } //D with dimensions (Ddim,Ddim) is distributed over all processes in ICTXT2D, with the first element in process (0,0) //D is distributed into blocks of size (blocksize,blocksize), having a local leading dimension lld_D in this specific process descinit_ ( DESCD, &Ddim, &Ddim, &blocksize, &blocksize, &i_zero, &i_zero, &ICTXT2D, &lld_D, &info ); if ( info!=0 ) { printf ( "Descriptor of matrix C returns info: %d\n",info ); return info; } //Allocate the space necessary to store the part of D that is held into memory of this process. D = ( double* ) calloc ( Drows * blocksize * Dcols * blocksize,sizeof ( double ) ); if ( D==NULL ) { printf ( "unable to allocate memory for Matrix D (required: %ld bytes)\n", Drows * blocksize * Dcols * blocksize * sizeof ( double ) ); return EXIT_FAILURE; } blacs_barrier_ ( &ICTXT2D,"ALL" ); if (iam==0) printf ( "Start set up of B & D\n" ); blacs_barrier_ ( &ICTXT2D,"ALL" ); //set_up_BD is declared in readdist.cpp and constructs the parts of matrices B & D in each processor //which are necessary to create the distributed Schur complement of D info = set_up_BD ( DESCD, D, BT_i, B_j, Btsparse ); //printdense(Drows*blocksize, Dcols * blocksize,D,"matrix_D.txt"); blacs_barrier_ ( &ICTXT2D,"ALL" ); if (iam==0) printf ( "Matrices B & D set up\n" ); if(printD_bool) { int array_of_gsizes[2], array_of_distribs[2], array_of_dargs[2], array_of_psize[2] ; int buffersize; MPI_Datatype file_type; MPI_File fh; MPI_Status status; array_of_gsizes[0]=Dblocks * blocksize; array_of_gsizes[1]=Dblocks * blocksize; array_of_distribs[0]=MPI_DISTRIBUTE_CYCLIC; array_of_distribs[1]=MPI_DISTRIBUTE_CYCLIC; array_of_dargs[0]=blocksize; array_of_dargs[1]=blocksize; array_of_psize[0]=*dims; array_of_psize[1]=*(dims + 1); MPI_Type_create_darray(size,iam,2,array_of_gsizes, array_of_distribs, array_of_dargs, array_of_psize, MPI_ORDER_FORTRAN, MPI_DOUBLE, &file_type); MPI_Type_commit(&file_type); info = MPI_File_open(MPI_COMM_WORLD, filenameD, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); /*if ( ( Drows-1 ) % *(dims+1) == *position && ( Dcols-1 ) % *(dims) == pcol && Ddim%blocksize !=0 ) buffersize=((Drows-1) * blocksize + Ddim % blocksize) * ((Dcols-1) * blocksize + Ddim % blocksize); else if ( ( Drows-1 ) % *(dims+1) == *position && Ddim%blocksize !=0 ) buffersize=((Drows-1) * blocksize + Ddim % blocksize) * Dcols * blocksize; else if ( ( Dcols-1 ) % *(dims) == *position && Ddim%blocksize !=0 ) buffersize=((Dcols-1) * blocksize + Ddim % blocksize) * Drows * blocksize; else*/ buffersize= Dcols * Drows * blocksize * blocksize; MPI_File_set_view(fh, 0, MPI_DOUBLE, file_type, "native", MPI_INFO_NULL); info =MPI_File_write_all(fh, D,buffersize, MPI_DOUBLE, &status); MPI_File_close(&fh); if(iam==0) { printf("Matrix D (dimension %d) is printed in file %s\n", Dblocks*blocksize,filenameD); } if(filenameD != NULL) free(filenameD); filenameD=NULL; //delete[] array_of_gsizes, delete[] array_of_distribs, delete[] array_of_dargs, delete[] array_of_psize; } //Now every matrix has to set up the sparse matrix A, consisting of X'X, X'Z, Z'X and Z'Z + lambda*I Xsparse.loadFromFile ( filenameX ); Zsparse.loadFromFile ( filenameZ ); if(filenameX != NULL) free(filenameX); filenameX=NULL; if(filenameZ != NULL) free(filenameZ); filenameZ=NULL; smat_t *X_smat, *Z_smat; X_smat= (smat_t *) calloc(1,sizeof(smat_t)); Z_smat= (smat_t *) calloc(1,sizeof(smat_t)); X_smat = smat_new_from ( Xsparse.nrows,Xsparse.ncols,Xsparse.pRows,Xsparse.pCols,Xsparse.pData,0,0 ); Z_smat = smat_new_from ( Zsparse.nrows,Zsparse.ncols,Zsparse.pRows,Zsparse.pCols,Zsparse.pData,0,0 ); smat_t *Xt_smat, *Zt_smat; Xt_smat= (smat_t *) calloc(1,sizeof(smat_t)); Zt_smat= (smat_t *) calloc(1,sizeof(smat_t)); Xt_smat = smat_copy_trans ( X_smat ); Zt_smat = smat_copy_trans ( Z_smat ); CSRdouble Asparse; smat_t *XtX_smat, *XtZ_smat, *ZtZ_smat, *lambda_smat, *ZtZlambda_smat; XtX_smat= (smat_t *) calloc(1,sizeof(smat_t)); XtZ_smat= (smat_t *) calloc(1,sizeof(smat_t)); ZtZ_smat= (smat_t *) calloc(1,sizeof(smat_t)); XtX_smat = smat_matmul ( Xt_smat, X_smat ); XtZ_smat = smat_matmul ( Xt_smat, Z_smat ); ZtZ_smat = smat_matmul ( Zt_smat,Z_smat ); Xsparse.clear(); Zsparse.clear(); smat_free(Xt_smat); smat_free(Zt_smat); /*smat_free(X_smat); smat_free(Z_smat);*/ CSRdouble Imat; makeIdentity ( l, Imat ); lambda_smat= (smat_t *) calloc(1,sizeof(smat_t)); lambda_smat = smat_new_from ( Imat.nrows,Imat.ncols,Imat.pRows,Imat.pCols,Imat.pData,0,0 ); smat_scale_diag ( lambda_smat, -lambda ); ZtZlambda_smat= (smat_t *) calloc(1,sizeof(smat_t)); ZtZlambda_smat = smat_add ( lambda_smat, ZtZ_smat ); smat_free(ZtZ_smat); //smat_free(lambda_smat); smat_to_symmetric_structure ( XtX_smat ); smat_to_symmetric_structure ( ZtZlambda_smat ); CSRdouble XtX_sparse, XtZ_sparse, ZtZ_sparse; XtX_sparse.make2 ( XtX_smat->m,XtX_smat->n,XtX_smat->nnz,XtX_smat->ia,XtX_smat->ja,XtX_smat->a ); XtZ_sparse.make2 ( XtZ_smat->m,XtZ_smat->n,XtZ_smat->nnz,XtZ_smat->ia,XtZ_smat->ja,XtZ_smat->a ); ZtZ_sparse.make2 ( ZtZlambda_smat->m,ZtZlambda_smat->n,ZtZlambda_smat->nnz,ZtZlambda_smat->ia,ZtZlambda_smat->ja,ZtZlambda_smat->a ); /*smat_free(XtX_smat); smat_free(XtZ_smat); smat_free(ZtZlambda_smat);*/ Imat.clear(); if (iam==0) { cout << "*** [ t t ] *** " << endl; cout << "*** [ X X X Z ] *** " << endl; cout << "*** [ ] *** " << endl; cout << "*** G e n e r a t i n g m a t r i x A = [ ] *** " << endl; cout << "*** [ t t ] *** " << endl; cout << "*** [ Z X Z Z ] *** " << endl; } //Sparse matrix A only contains the upper triangular part of A create2x2SymBlockMatrix ( XtX_sparse, XtZ_sparse, ZtZ_sparse, Asparse ); //Asparse.writeToFile("A_sparse.csr"); smat_free(XtX_smat); smat_free(XtZ_smat); smat_free(ZtZlambda_smat); XtX_sparse.clear(); XtZ_sparse.clear(); ZtZ_sparse.clear(); blacs_barrier_ ( &ICTXT2D,"ALL" ); if(printsparseC_bool) { CSRdouble Dmat, Dblock, Csparse; Dblock.nrows=Dblocks * blocksize; Dblock.ncols=Dblocks * blocksize; Dblock.allocate(Dblocks * blocksize, Dblocks * blocksize, 0); Dmat.allocate(0,0,0); for (i=0; i<Drows; ++i) { for(j=0; j<Dcols; ++j) { dense2CSR_sub(D + i * blocksize + j * lld_D * blocksize,blocksize,blocksize,lld_D,Dblock,( * ( dims) * i + *position ) *blocksize, ( * ( dims+1 ) * j + pcol ) *blocksize); if ( Dblock.nonzeros>0 ) { if ( Dmat.nonzeros==0 ) { Dmat.make2 ( Dblock.nrows,Dblock.ncols,Dblock.nonzeros,Dblock.pRows,Dblock.pCols,Dblock.pData ); } else { Dmat.addBCSR ( Dblock ); } } Dblock.clear(); } } blacs_barrier_(&ICTXT2D,"A"); if ( iam!=0 ) { //Each process other than root sends its Dmat to the root process. MPI_Send ( & ( Dmat.nonzeros ),1, MPI_INT,0,iam,MPI_COMM_WORLD ); MPI_Send ( & ( Dmat.pRows[0] ),Dmat.nrows + 1, MPI_INT,0,iam+size,MPI_COMM_WORLD ); MPI_Send ( & ( Dmat.pCols[0] ),Dmat.nonzeros, MPI_INT,0,iam+2*size,MPI_COMM_WORLD ); MPI_Send ( & ( Dmat.pData[0] ),Dmat.nonzeros, MPI_DOUBLE,0,iam+3*size,MPI_COMM_WORLD ); Dmat.clear(); } else { for ( i=1; i<size; ++i ) { // The root process receives parts of Dmat sequentially from all processes and directly adds them together. int nonzeroes, count; MPI_Recv ( &nonzeroes,1,MPI_INT,i,i,MPI_COMM_WORLD,&status ); /*MPI_Get_count(&status, MPI_INT, &count); printf("Process 0 received %d elements of process %d\n",count,i);*/ if(nonzeroes>0) { printf("Nonzeroes : %d\n ",nonzeroes); Dblock.allocate ( Dblocks * blocksize,Dblocks * blocksize,nonzeroes ); MPI_Recv ( & ( Dblock.pRows[0] ), Dblocks * blocksize + 1, MPI_INT,i,i+size,MPI_COMM_WORLD,&status ); /*MPI_Get_count(&status, MPI_INT, &count); printf("Process 0 received %d elements of process %d\n",count,i);*/ MPI_Recv ( & ( Dblock.pCols[0] ),nonzeroes, MPI_INT,i,i+2*size,MPI_COMM_WORLD,&status ); /*MPI_Get_count(&status, MPI_INT, &count); printf("Process 0 received %d elements of process %d\n",count,i);*/ MPI_Recv ( & ( Dblock.pData[0] ),nonzeroes, MPI_DOUBLE,i,i+3*size,MPI_COMM_WORLD,&status ); /*MPI_Get_count(&status, MPI_DOUBLE, &count); printf("Process 0 received %d elements of process %d\n",count,i);*/ Dmat.addBCSR ( Dblock ); } } //Dmat.writeToFile("D_sparse.csr"); Dmat.reduceSymmetric(); Btsparse.transposeIt(1); create2x2SymBlockMatrix(Asparse,Btsparse, Dmat, Csparse); Btsparse.clear(); Dmat.clear(); Csparse.writeToFile(filenameC); Csparse.clear(); if(filenameC != NULL) free(filenameC); filenameC=NULL; } } Btsparse.clear(); blacs_barrier_(&ICTXT2D,"A"); //AB_sol will contain the solution of A*X=B, distributed across the process rows. Processes in the same process row possess the same part of AB_sol double * AB_sol; int * DESCAB_sol; DESCAB_sol= ( int* ) malloc ( DLEN_ * sizeof ( int ) ); if ( DESCAB_sol==NULL ) { printf ( "unable to allocate memory for descriptor for AB_sol\n" ); return -1; } //AB_sol (Adim, Ddim) is distributed across all processes in ICTXT2D starting from process (0,0) into blocks of size (Adim, blocksize) descinit_ ( DESCAB_sol, &Adim, &Ddim, &Adim, &blocksize, &i_zero, &i_zero, &ICTXT2D, &Adim, &info ); if ( info!=0 ) { printf ( "Descriptor of matrix C returns info: %d\n",info ); return info; } AB_sol=(double *) calloc(Adim * Dcols*blocksize,sizeof(double)); // Each process calculates the Schur complement of the part of D at its disposal. (see src/schur.cpp) // The solution of A * Y = B_j is stored in AB_sol (= A^-1 * B_j) blacs_barrier_(&ICTXT2D,"A"); make_Sij_parallel_denseB ( Asparse, BT_i, B_j, D, lld_D, AB_sol ); BT_i.clear(); B_j.clear(); //From here on the Schur complement S of D is stored in D blacs_barrier_ ( &ICTXT2D,"ALL" ); //The Schur complement is factorised (by ScaLAPACK) pdpotrf_ ( "U",&k,D,&i_one,&i_one,DESCD,&info ); if ( info != 0 ) { printf ( "Cholesky decomposition of D was unsuccessful, error returned: %d\n",info ); return -1; } //From here on the factorization of the Schur complement S is stored in D blacs_barrier_ ( &ICTXT2D,"ALL" ); //The Schur complement is inverted (by ScaLAPACK) pdpotri_ ( "U",&k,D,&i_one,&i_one,DESCD,&info ); if ( info != 0 ) { printf ( "Inverse of D was unsuccessful, error returned: %d\n",info ); return -1; } //From here on the inverse of the Schur complement S is stored in D blacs_barrier_(&ICTXT2D,"A"); double* InvD_T_Block = ( double* ) calloc ( Dblocks * blocksize + Adim ,sizeof ( double ) ); //Diagonal elements of the (1,1) block of C^-1 are still distributed and here they are gathered in InvD_T_Block in the root process. if(*position == pcol) { for (i=0; i<Ddim; ++i) { if (pcol == (i/blocksize) % *dims) { int Dpos = i%blocksize + ((i/blocksize) / *dims) * blocksize ; *(InvD_T_Block + Adim +i) = *( D + Dpos + lld_D * Dpos); } } for ( i=0,j=0; i<Dblocks; ++i,++j ) { if ( j==*dims ) j=0; if ( *position==j ) { dgesd2d_ ( &ICTXT2D,&blocksize,&i_one,InvD_T_Block + Adim + i * blocksize,&blocksize,&i_zero,&i_zero ); } if ( *position==0 ) { dgerv2d_ ( &ICTXT2D,&blocksize,&i_one,InvD_T_Block + Adim + blocksize*i,&blocksize,&j,&j ); } } } blacs_barrier_(&ICTXT2D,"A"); //Only the root process performs a selected inversion of A. if (iam==0) { int pardiso_message_level = 1; int pardiso_mtype=-2; ParDiSO pardiso ( pardiso_mtype, pardiso_message_level ); int number_of_processors = 1; char* var = getenv("OMP_NUM_THREADS"); if(var != NULL) { sscanf( var, "%d", &number_of_processors ); } else { printf("Set environment OMP_NUM_THREADS to 1"); exit(1); } pardiso.iparm[2] = 2; pardiso.iparm[3] = number_of_processors; pardiso.iparm[8] = 0; pardiso.iparm[11] = 1; pardiso.iparm[13] = 0; pardiso.iparm[28] = 0; //This function calculates the factorisation of A once again so this might be optimized. pardiso.findInverseOfA ( Asparse ); printf("Processor %d inverted matrix A\n",iam); } blacs_barrier_(&ICTXT2D,"A"); // To minimize memory usage, and because only the diagonal elements of the inverse are needed, Y' * S is calculated row by rowblocks // the diagonal element is calculates as the dot product of this row and the corresponding column of Y. (Y is solution of AY=B) double* YSrow= ( double* ) calloc ( Dcols * blocksize,sizeof ( double ) ); int * DESCYSROW; DESCYSROW= ( int* ) malloc ( DLEN_ * sizeof ( int ) ); if ( DESCYSROW==NULL ) { printf ( "unable to allocate memory for descriptor for AB_sol\n" ); return -1; } //YSrow (1,Ddim) is distributed across processes of ICTXT2D starting from process (0,0) into blocks of size (1,blocksize) descinit_ ( DESCYSROW, &i_one, &Ddim, &i_one,&blocksize, &i_zero, &i_zero, &ICTXT2D, &i_one, &info ); if ( info!=0 ) { printf ( "Descriptor of matrix C returns info: %d\n",info ); return info; } blacs_barrier_(&ICTXT2D,"A"); //Calculating diagonal elements 1 by 1 of the (0,0)-block of C^-1. for (i=1; i<=Adim; ++i) { pdsymm_ ("R","U",&i_one,&Ddim,&d_one,D,&i_one,&i_one,DESCD,AB_sol,&i,&i_one,DESCAB_sol,&d_zero,YSrow,&i_one,&i_one,DESCYSROW); pddot_(&Ddim,InvD_T_Block+i-1,AB_sol,&i,&i_one,DESCAB_sol,&Adim,YSrow,&i_one,&i_one,DESCYSROW,&i_one); /*if(*position==1 && pcol==1) printf("Dot product in process (1,1) is: %g\n", *(InvD_T_Block+i-1)); if(*position==0 && pcol==1) printf("Dot product in process (0,1) is: %g\n",*(InvD_T_Block+i-1));*/ } blacs_barrier_(&ICTXT2D,"A"); if(YSrow != NULL) free(YSrow); YSrow = NULL; if(DESCYSROW != NULL) free(DESCYSROW); DESCYSROW = NULL; if(AB_sol != NULL) free(AB_sol); AB_sol = NULL; if(DESCAB_sol != NULL) free(DESCAB_sol); DESCAB_sol = NULL; if(D != NULL) free(D); D = NULL; if(DESCD != NULL) free(DESCD); DESCD = NULL; //Only in the root process we add the diagonal elements of A^-1 if (iam ==0) { for(i=0; i<Adim; ++i) { j=Asparse.pRows[i]; *(InvD_T_Block+i) += Asparse.pData[j]; } Asparse.clear(); printdense ( Adim+k,1,InvD_T_Block,"diag_inverse_C_parallel.txt" ); } if(InvD_T_Block != NULL) free(InvD_T_Block); InvD_T_Block = NULL; blacs_gridexit_(&ICTXT2D); } //cout << iam << " reached end before MPI_Barrier" << endl; MPI_Barrier(MPI_COMM_WORLD); //MPI_Finalize(); return 0; }
static int test_indexed_with_zeros(char *filename, int testcase) { int i, rank, np, buflen, num, err, nr_errors=0; int nelms[MAXLEN], buf[MAXLEN], indices[MAXLEN], blocklen[MAXLEN]; MPI_File fh; MPI_Status status; MPI_Datatype filetype; MPI_Datatype types[MAXLEN]; MPI_Aint addrs[MAXLEN]; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &np); /* set up the number of integers to write in each iteration */ for (i=0; i<MAXLEN; i++) nelms[i] = 0; if (rank == 0) nelms[4]=nelms[5]=nelms[7]=1; if (rank == 1) nelms[0]=nelms[1]=nelms[2]=nelms[3]=nelms[6]=nelms[8]=1; /* pre-fill the file with integers -999 */ if (rank == 0) { for (i=0; i<MAXLEN; i++) buf[i] = -999; err =MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_open"); err = MPI_File_write(fh, buf, MAXLEN, MPI_INT, &status); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_write"); err = MPI_File_close(&fh); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_close"); } MPI_Barrier(MPI_COMM_WORLD); /* define a filetype with spurious leading zeros */ buflen = num = 0; for (i=0; i<MAXLEN; i++) { buflen += nelms[i]; indices[num] = i; addrs[num] = i*sizeof(int); blocklen[num] = nelms[i]; types[num] = MPI_INT; num++; } switch (testcase) { case INDEXED: MPI_Type_indexed(num, blocklen, indices, MPI_INT, &filetype); break; case HINDEXED: MPI_Type_hindexed(num, blocklen, addrs, MPI_INT, &filetype); break; case STRUCT: MPI_Type_create_struct(num, blocklen, addrs, types, &filetype); break; default: fprintf(stderr, "unknown testcase!\n"); return(-100); } MPI_Type_commit(&filetype); /* initialize write buffer and write to file*/ for (i=0; i<MAXLEN; i++) buf[i] = 1; err =MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_open"); err = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_set_view"); err = MPI_File_write_all(fh, buf, buflen, MPI_INT, &status); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_write_all"); MPI_Type_free(&filetype); err = MPI_File_close(&fh); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_close"); /* read back and check */ if (rank == 0) { err = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_open"); err = MPI_File_read(fh,buf, MAXLEN, MPI_INT, &status); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_read"); err = MPI_File_close(&fh); if (err != MPI_SUCCESS) handle_error(err, "MPI_File_close"); for (i=0; i<MAXLEN; i++) { if (buf[i] < 0) { nr_errors++; printf("Error: unexpected value for case %d at buf[%d] == %d\n", testcase,i,buf[i]); } } } return nr_errors; }
/* *************************** * * Main computational kernel * * *************************** */ int correlationKernel(int rank, int size, double* dataMatrixX, double* dataMatrixY, int columns, int rows, char *out_filename, int distance_flag) { int local_check = 0, global_check = 0; int i = 0, j, taskNo; int err, count = 0; unsigned long long fair_chunk = 0, coeff_count = 0; unsigned int init_and_cleanup_loop_iter=0; unsigned long long cor_cur_size = 0; double start_time, end_time; // Variables needed by the Indexed Datatype MPI_Datatype coeff_index_dt; MPI_File fh; int *blocklens, *indices; MPI_Status stat; MPI_Comm comm = MPI_COMM_WORLD; // Master processor keeps track of tasks if (rank == 0) { // Make sure everything will work fine even if there are // less genes than available workers (there are size-1 workers // master does not count) if ( (size-1) > rows ) init_and_cleanup_loop_iter = rows+1; else init_and_cleanup_loop_iter = size; // Start timer start_time = MPI_Wtime(); // Send out initial tasks (remember you have size-1 workers, master does not count) for (i=1; i<init_and_cleanup_loop_iter; i++) { taskNo = i-1; err = MPI_Send(&taskNo, 1, MPI_INT, i, 0, comm); } // Terminate any processes that were not working due to the fact // that the number of rows where less than the actual available workers for(i=init_and_cleanup_loop_iter; i < size; i++) { PROF(rank, "\nPROF_idle : Worker %d terminated due to insufficient work load", i); err = -1; err = MPI_Send(&err, 1, MPI_INT, i, 0, comm); } // Wait for workers to finish their work assignment and ask for more for (i=init_and_cleanup_loop_iter-1; i<rows; i++) { err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat); // Check taskNo to make sure everything is ok. Negative means there is problem // thus terminate gracefully all remaining working workers if ( taskNo < 0 ) { // Reduce by one because one worker is already terminated init_and_cleanup_loop_iter--; // Break and cleanup break; } // The sending processor is ready to work: // It's ID is in stat.MPI_SOURCE // Send it the current task (i) err = MPI_Send(&i, 1, MPI_INT, stat.MPI_SOURCE, 0, comm); } // Clean up processors for (i=1; i<init_and_cleanup_loop_iter; i++) { // All tasks complete - shutdown workers err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat); // If process failed then it will not be waiting to receive anything // We have to ignore the send because it will deadlock if ( taskNo < 0 ) continue; err = -1; err = MPI_Send(&err, 1, MPI_INT, stat.MPI_SOURCE, 0, comm); } // Master is *always* OK local_check = 0; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Check failed, abort if ( global_check != 0 ) { return -1; } // Stop timer end_time = MPI_Wtime(); PROF(rank, "\nPROF_comp (workers=%d) : Time taken by correlation coefficients computations : %g\n", size-1, end_time - start_time); // Start timer start_time = MPI_Wtime(); // Master process must call MPI_File_set_view as well, it's a collective call // Open the file handler MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); // Create the file view MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL); // Write data to disk MPI_File_write_all(fh, &cor[0], 0, MPI_DOUBLE, &stat); // Stop timer end_time = MPI_Wtime(); PROF(rank, "\nPROF_write (workers=%d) : Time taken for global write-file : %g\n", size-1, end_time - start_time); } else { // Compute how many workers will share the work load // Two scenarios exist: // (1) more OR equal number of workers and rows exist // (2) more rows than workers if ( (size-1) > rows ) { // For this scenario each worker will get exaclty one work asssignment. // There is not going to be any other work so it only compute "rows" number // of coefficients fair_chunk = rows; cor_cur_size = fair_chunk; } else { // For this scenario we are going to allocate space equal to a fair // distribution of work assignments *plus* an extra amount of space to // cover any load imbalancing. This amount is expressed as a percentage // of the fair work distribution (see on top, 20% for now) // Plus 1 to round it up or just add some extra space, both are fine fair_chunk = (rows / (size-1)) + 1; DEBUG("fair_chunk %d \n", fair_chunk); // We can use "j" as temporary variable. // Plus 1 to avoid getting 0 from the multiplication. j = (fair_chunk * MEM_PERC) + 1; cor_cur_size = (fair_chunk + j) * rows; DEBUG("cor_cur_size %lld \n", cor_cur_size); } // Allocate memory DEBUG("cor_cur_size %lld \n", cor_cur_size); long long double_size = sizeof(double); DEBUG("malloc size %lld \n", (double_size * cor_cur_size)); cor = (double *)malloc(double_size * cor_cur_size); blocklens = (int *)malloc(sizeof(int) * rows); indices = (int *)malloc(sizeof(int) * rows); mean_value_vectorX = (double *)malloc(sizeof(double) * rows); Sxx_vector = (double *)malloc(sizeof(double) * rows); mean_value_vectorY = (double *)malloc(sizeof(double) * rows); Syy_vector = (double *)malloc(sizeof(double) * rows); // Check that all memory is successfully allocated if ( ( cor == NULL ) || ( blocklens == NULL ) || ( indices == NULL ) || ( mean_value_vectorX == NULL ) || ( Sxx_vector == NULL ) || ( mean_value_vectorY == NULL ) || ( Syy_vector == NULL ) ) { ERR("**ERROR** : Memory allocation failed on worker process %d. Aborting.\n", rank); // Free allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); // Let the master process know its aborting in order to terminate // the rest of the working workers // We have to receive a work assignment first and then terminate // otherwise the master will deadlock trying to give work to this worker err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat); taskNo = -1; err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); // This worker failed local_check = 1; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); return -1; } // Compute necessary parameters for Pearson method // (this will transform the values of the input array to more meaningful data // and save us from a lot of redundant computations) compute_parameters(dataMatrixX, dataMatrixY, rows, columns); // Main loop for workers. They get work from master, compute coefficients, // save them to their *local* vector and ask for more work for(;;) { // Get work err = 0; err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat); // If received task is -1, function is terminated if ( taskNo == -1 ) break; // Check if there is enough memory to store the new coefficients, if not reallocate // the current memory and expand it by MEM_PERC of the approximated size if ( cor_cur_size < (coeff_count + rows) ) { PROF(0, "\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank); DEBUG("\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank); // Use j as temporary again. Add two (or any other value) to avoid 0. // (two is just a random value, you can put any value really...) j = (fair_chunk * MEM_PERC) + 2; cor_cur_size += (j * rows); // Reallocate and check cor = (double *)realloc(cor, sizeof(double) * cor_cur_size); if ( cor == NULL ) { ERR("**ERROR** : Memory re-allocation failed on worker process %d. Aborting.\n", rank); // Let the master process know its aborting in order to terminate // the rest of the working workers taskNo = -1; err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); // This worker failed local_check = 1; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Free all allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); return -1; } } // Compute the correlation coefficients if(dataMatrixY != NULL) { for (j=0; j < rows; j++) { cor[coeff_count] = pearson_XY(dataMatrixX, dataMatrixY, j, taskNo, columns); coeff_count++; } } else { for (j=0; j < rows; j++) { // Set main diagonal to 1 if ( j == taskNo ) { cor[coeff_count] = 1.0; coeff_count++; continue; } cor[coeff_count] = pearson(dataMatrixX, taskNo, j, columns); coeff_count++; } } // The value of blocklens[] represents the number of coefficients on each // row of the corellation array blocklens[count] = rows; // The value of indices[] represents the offset of each row in the data file indices[count] = (taskNo * rows); count++; // Give the master the taskID err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); } // There are two possibilities // (a) everything went well and all workers finished ok // (b) some processes finished ok but one or more of the remaining working workers failed // To make sure all is well an all-reduce will be performed to sync all workers and guarantee success // before moving on to write the output file // This worker is OK local_check = 0; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Check failed if ( global_check != 0 ) { // Free all allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); return -1; } PROF(0, "\nPROF_stats (thread %3d) : Fair chunk of work : %d \t\t Allocated : %d \t\t Computed : %d\n", rank, fair_chunk, cor_cur_size, coeff_count); // If the distance_flag is set, then transform all correlation coefficients to distances if ( distance_flag == 1 ) { for(j=0; j < coeff_count; j++) { cor[j] = 1 - cor[j]; } } // Create and commit the Indexed datatype *ONLY* if there are data available if ( coeff_count != 0 ) { MPI_Type_indexed(count, blocklens, indices, MPI_DOUBLE, &coeff_index_dt); MPI_Type_commit(&coeff_index_dt); } // Open the file handler MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); // Create the file view if ( coeff_count != 0 ) { MPI_File_set_view(fh, 0, MPI_DOUBLE, coeff_index_dt, "native", MPI_INFO_NULL); } else { MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL); } // Write data to disk // TODO coeff_count cannot be greater than max int (for use in the MPI_File_write_all call). // A better fix should be possible, for now throw error. DEBUG("\ncoeff_count is %lld\n", coeff_count); DEBUG("\INT_MAX is %d\n", INT_MAX); if(coeff_count>INT_MAX) { ERR("**ERROR** : Could not run as the chunks of data are too large. Try running again with more MPI processes.\n"); // Free allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); // Let the master process know its aborting in order to terminate // the rest of the working workers // We have to receive a work assignment first and then terminate // otherwise the master will deadlock trying to give work to this worker err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat); taskNo = -1; err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm); // This worker failed local_check = 1; MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); return -1; } DEBUG("\nWriting %d to disk\n", coeff_count); MPI_File_write_all(fh, &cor[0], coeff_count, MPI_DOUBLE, &stat); if (coeff_count != 0 ) MPI_Type_free(&coeff_index_dt); // Free all allocated memory free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector); } DEBUG("\nAbout to write to disk %d\n", rank); MPI_File_sync( fh ) ; // Causes all previous writes to be transferred to the storage device DEBUG("\nWritten to disk %d\n",rank); // MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine. DEBUG("\nAfter barrier \n", rank); // Close file handler MPI_File_close(&fh); DEBUG("\nAfter file closed /n"); // MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine. DEBUG("\nAbout to return from kernel /n"); return 0; }
int main(int argc, char **argv) { int *buf, i, mynod, nprocs, len, b[3]; int errs = 0, toterrs; MPI_Aint d[3]; MPI_File fh; MPI_Status status; char *filename; MPI_Datatype typevec, newtype, t[3]; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &mynod); if (nprocs != 2) { fprintf(stderr, "Run this program on two processes\n"); MPI_Abort(MPI_COMM_WORLD, 1); } /* process 0 takes the file name as a command-line argument and broadcasts it to other processes */ if (!mynod) { i = 1; while ((i < argc) && strcmp("-fname", *argv)) { i++; argv++; } if (i >= argc) { fprintf(stderr, "\n*# Usage: noncontig_coll -fname filename\n\n"); MPI_Abort(MPI_COMM_WORLD, 1); } argv++; len = strlen(*argv); filename = (char *) malloc(len + 1); strcpy(filename, *argv); MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD); } else { MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); filename = (char *) malloc(len + 1); MPI_Bcast(filename, len + 1, MPI_CHAR, 0, MPI_COMM_WORLD); } buf = (int *) malloc(SIZE * sizeof(int)); MPI_Type_vector(SIZE / 2, 1, 2, MPI_INT, &typevec); b[0] = b[1] = b[2] = 1; d[0] = 0; d[1] = mynod * sizeof(int); d[2] = SIZE * sizeof(int); t[0] = MPI_LB; t[1] = typevec; t[2] = MPI_UB; MPI_Type_struct(3, b, d, t, &newtype); MPI_Type_commit(&newtype); MPI_Type_free(&typevec); if (!mynod) { #if VERBOSE fprintf(stderr, "\ntesting noncontiguous in memory, noncontiguous in file using collective I/O\n"); #endif MPI_File_delete(filename, MPI_INFO_NULL); } MPI_Barrier(MPI_COMM_WORLD); MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL)); for (i = 0; i < SIZE; i++) buf[i] = i + mynod * SIZE; MPI_CHECK(MPI_File_write_all(fh, buf, 1, newtype, &status)); MPI_Barrier(MPI_COMM_WORLD); for (i = 0; i < SIZE; i++) buf[i] = -1; MPI_CHECK(MPI_File_read_at_all(fh, 0, buf, 1, newtype, &status)); for (i = 0; i < SIZE; i++) { if (!mynod) { if ((i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } if (!(i % 2) && (buf[i] != i)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i); } } else { if ((i % 2) && (buf[i] != i + mynod * SIZE)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i + mynod * SIZE); } if (!(i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } } } MPI_CHECK(MPI_File_close(&fh)); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { #if VERBOSE fprintf(stderr, "\ntesting noncontiguous in memory, contiguous in file using collective I/O\n"); #endif MPI_File_delete(filename, MPI_INFO_NULL); } MPI_Barrier(MPI_COMM_WORLD); MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); for (i = 0; i < SIZE; i++) buf[i] = i + mynod * SIZE; MPI_CHECK(MPI_File_write_at_all(fh, mynod * (SIZE / 2) * sizeof(int), buf, 1, newtype, &status)); MPI_Barrier(MPI_COMM_WORLD); for (i = 0; i < SIZE; i++) buf[i] = -1; MPI_CHECK(MPI_File_read_at_all(fh, mynod * (SIZE / 2) * sizeof(int), buf, 1, newtype, &status)); for (i = 0; i < SIZE; i++) { if (!mynod) { if ((i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } if (!(i % 2) && (buf[i] != i)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i); } } else { if ((i % 2) && (buf[i] != i + mynod * SIZE)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i + mynod * SIZE); } if (!(i % 2) && (buf[i] != -1)) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be -1\n", mynod, i, buf[i]); } } } MPI_CHECK(MPI_File_close(&fh)); MPI_Barrier(MPI_COMM_WORLD); if (!mynod) { #if VERBOSE fprintf(stderr, "\ntesting contiguous in memory, noncontiguous in file using collective I/O\n"); #endif MPI_File_delete(filename, MPI_INFO_NULL); } MPI_Barrier(MPI_COMM_WORLD); MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); MPI_CHECK(MPI_File_set_view(fh, 0, MPI_INT, newtype, "native", MPI_INFO_NULL)); for (i = 0; i < SIZE; i++) buf[i] = i + mynod * SIZE; MPI_CHECK(MPI_File_write_all(fh, buf, SIZE, MPI_INT, &status)); MPI_Barrier(MPI_COMM_WORLD); for (i = 0; i < SIZE; i++) buf[i] = -1; MPI_CHECK(MPI_File_read_at_all(fh, 0, buf, SIZE, MPI_INT, &status)); for (i = 0; i < SIZE; i++) { if (!mynod) { if (buf[i] != i) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i); } } else { if (buf[i] != i + mynod * SIZE) { errs++; fprintf(stderr, "Process %d: buf %d is %d, should be %d\n", mynod, i, buf[i], i + mynod * SIZE); } } } MPI_CHECK(MPI_File_close(&fh)); MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (mynod == 0) { if (toterrs > 0) { fprintf(stderr, "Found %d errors\n", toterrs); } else { fprintf(stdout, " No Errors\n"); } } MPI_Type_free(&newtype); free(buf); free(filename); MPI_Finalize(); return 0; }