Matrix createMatrixMPI(int n1, int n2, int N1, int N2, MPI_Comm* comm) { int i, n12; Matrix result = (Matrix)calloc(1, sizeof(matrix_t)); result->as_vec = createVectorMPI(N1*N2, comm, 0); n12 = n1; if (n1 == -1) n1 = result->as_vec->len/N2; else n2 = result->as_vec->len/N1; result->rows = n1; result->cols = n2; result->glob_rows = N1; result->glob_cols = N2; result->data = (double **)calloc(n2 ,sizeof(double *)); result->data[0] = (double *)calloc(n1*n2,sizeof(double)); result->as_vec->data = result->data[0]; for (i=1; i < n2; i++) result->data[i] = result->data[i-1] + n1; result->col = malloc(n2*sizeof(Vector)); for (i=0;i<n2;++i) { if (n12 == N1) result->col[i] = createVectorMPI(N1, &SelfComm, 0); else result->col[i] = createVectorMPI(N1, comm, 0); result->col[i]->data = result->data[i]; } result->row = malloc(n1*sizeof(Vector)); for (i=0;i<n1;++i) { if (n12 == N1) result->row[i] = createVectorMPI(N2, comm, 0); else result->row[i] = createVectorMPI(N2, &SelfComm, 0); result->row[i]->data = result->data[0]+i; result->row[i]->stride = n1; } return result; }
int main(int argc, char **argv) { int rank, size; init_app(argc, argv, &rank, &size); Vector vec = createVectorMPI(30, &WorldComm, 1); if (rank == 0) { for (size_t i = 0; i < size; i++) { printf(">> Proc %d owns [%d,%d]\n", i, vec->displ[i], vec->displ[i] + vec->sizes[i] - 1); } } close_app(); return 0; }
Matrix createMatrixMPI(int n1, int n2, int N1, int N2, MPI_Comm* comm) { int i; Matrix result = (Matrix)calloc(1, sizeof(matrix_t)); int topo_type; MPI_Topo_test(*comm, &topo_type); if (topo_type == MPI_CART) { result->rows = n1; result->cols = n2; result->as_vec = malloc(sizeof(vector_t)); result->as_vec->comm = comm; result->as_vec->len = n1*n2; result->as_vec->stride = 1; MPI_Comm_rank(*comm, &result->as_vec->comm_rank); MPI_Comm_size(*comm, &result->as_vec->comm_size); result->data = (double **)calloc(n2 ,sizeof(double *)); result->data[0] = (double *)calloc(n1*n2,sizeof(double)); result->as_vec->data = result->data[0]; for (i=1; i < n2; i++) result->data[i] = result->data[i-1] + n1; result->col = malloc(n2*sizeof(Vector)); for (int i=0;i<n2;++i) { result->col[i] = malloc(sizeof(vector_t)); result->col[i]->data = result->data[i]; result->col[i]->stride = 1; result->col[i]->len = n1; } result->row = malloc(n1*sizeof(Vector)); for (int i=0;i<n1;++i) { result->row[i] = malloc(sizeof(vector_t)); result->row[i]->data = result->data[0]+i; result->row[i]->stride = n1; result->row[i]->len = n2; } return result; } result->as_vec = createVectorMPI(N1*N2, 0, comm); int n12 = n1; if (n1 == -1) n1 = result->as_vec->len/N2; else n2 = result->as_vec->len/N1; result->rows = n1; result->cols = n2; result->glob_rows = N1; result->glob_cols = N2; result->data = (double **)calloc(n2 ,sizeof(double *)); result->data[0] = (double *)calloc(n1*n2,sizeof(double)); result->as_vec->data = result->data[0]; for (i=1; i < n2; i++) result->data[i] = result->data[i-1] + n1; result->col = malloc(n2*sizeof(Vector)); for (int i=0;i<n2;++i) { if (n12 == N1) { result->col[i] = createVectorMPI(N1, 0, &SelfComm); } else { result->col[i] = createVectorMPI(N1, 0, comm); } result->col[i]->data = result->data[i]; } result->row = malloc(n1*sizeof(Vector)); for (int i=0;i<n1;++i) { if (n12 == N1) result->row[i] = createVectorMPI(N2, 0, comm); else result->row[i] = createVectorMPI(N2, 0, &SelfComm); result->row[i]->data = result->data[0]+i; result->row[i]->stride = n1; } return result; }
int main(int argc, char** argv) { int i, j, N, flag; Matrix A=NULL, Q=NULL; Vector b, grid, e, lambda=NULL; double time, sum, h, tol=1e-6; int rank, size; int mpi_top_coords; int mpi_top_sizes; init_app(argc, argv, &rank, &size); if (argc < 3) { printf("need two parameters, N and flag [and tolerance]\n"); printf(" - N is the problem size (in each direction\n"); printf(" - flag = 1 -> Matrix-free Gauss-Jacobi iterations\n"); printf(" - flag = 2 -> Matrix-free red-black Gauss-Seidel iterations\n"); printf(" - flag = 3 -> Matrix-free CG iterations\n"); printf(" - flag = 4 -> Matrix-free additive schwarz preconditioned+Cholesky CG iterations\n"); printf(" - flag = 5 -> Matrix-free additive schwarz preconditioned+CG CG iterations\n"); return 1; } N=atoi(argv[1]); flag=atoi(argv[2]); if (argc > 3) tol=atof(argv[3]); if (N < 0) { if (rank == 0) printf("invalid problem size given\n"); close_app(); return 2; } if (flag < 0 || flag > 5) { if (rank == 0) printf("invalid flag given\n"); close_app(); return 3; } if (flag == 2 && (N-1)%2 != 0 && ((N-1)/size) % 2 != 0) { if (rank == 0) printf("need an even size (per process) for red-black iterations\n"); close_app(); return 4; } // setup topology mpi_top_coords = 0; mpi_top_sizes = 0; MPI_Dims_create(size, 1, &mpi_top_sizes); int periodic = 0; MPI_Comm comm; MPI_Cart_create(MPI_COMM_WORLD, 1, &mpi_top_sizes, &periodic, 0, &comm); MPI_Cart_coords(comm, rank, 1, &mpi_top_coords); b = createVectorMPI(N+1, &comm, 1, 1); e = createVectorMPI(N+1, &comm, 1, 1); grid = equidistantMesh(0.0, 1.0, N); h = 1.0/N; evalMeshDispl(b, grid, source); scaleVector(b, pow(h, 2)); evalMeshDispl(e, grid, exact); axpy(b, e, alpha); b->data[0] = b->data[b->len-1] = 0.0; if (flag == 4) { int size = b->len; if (b->comm_rank == 0) size--; if (b->comm_rank == b->comm_size-1) size--; A1D = createMatrix(size, size); A1Dfactored = 0; diag(A1D, -1, -1.0); diag(A1D, 0, 2.0+alpha); diag(A1D, 1, -1.0); } int its=-1; char method[128]; time = WallTime(); if (flag == 1) { its=GaussJacobiPoisson1D(b, tol, 1000000); sprintf(method,"Gauss-Jacobi"); } if (flag == 2) { its=GaussSeidelPoisson1Drb(b, tol, 1000000); sprintf(method,"Gauss-Seidel"); } if (flag == 3) { its=cgMatrixFree(Poisson1D, b, tol); sprintf(method,"CG"); } if (flag == 4 || flag == 5) { its=pcgMatrixFree(Poisson1D, Poisson1DPre, b, tol); sprintf(method,"PCG"); } if (rank == 0) { printf("%s used %i iterations\n", method, its); printf("elapsed: %f\n", WallTime()-time); } evalMeshDispl(e, grid, exact); axpy(b,e,-1.0); b->data[0] = b->data[b->len-1] = 0.0; h = maxNorm(b); if (rank == 0) printf("max error: %e\n", h); if (A) freeMatrix(A); if (Q) freeMatrix(Q); freeVector(grid); freeVector(b); freeVector(e); if (lambda) freeVector(lambda); if (A1D) freeMatrix(A1D); MPI_Comm_free(&comm); close_app(); return 0; }