Matrix createMatrixMPI(int n1, int n2, int N1, int N2, MPI_Comm* comm)
{
int i, n12;
Matrix result = (Matrix)calloc(1, sizeof(matrix_t));
result->as_vec = createVectorMPI(N1*N2, comm, 0);
n12 = n1;
if (n1 == -1)
n1 = result->as_vec->len/N2;
else
n2 = result->as_vec->len/N1;
result->rows = n1;
result->cols = n2;
result->glob_rows = N1;
result->glob_cols = N2;
result->data = (double **)calloc(n2 ,sizeof(double *));
result->data[0] = (double *)calloc(n1*n2,sizeof(double));
result->as_vec->data = result->data[0];
for (i=1; i < n2; i++)
result->data[i] = result->data[i-1] + n1;
result->col = malloc(n2*sizeof(Vector));
for (i=0;i<n2;++i) {
if (n12 == N1)
result->col[i] = createVectorMPI(N1, &SelfComm, 0);
else
result->col[i] = createVectorMPI(N1, comm, 0);
result->col[i]->data = result->data[i];
}
result->row = malloc(n1*sizeof(Vector));
for (i=0;i<n1;++i) {
if (n12 == N1)
result->row[i] = createVectorMPI(N2, comm, 0);
else
result->row[i] = createVectorMPI(N2, &SelfComm, 0);
result->row[i]->data = result->data[0]+i;
result->row[i]->stride = n1;
}
return result;
}
int main(int argc, char **argv)
{
int rank, size;
init_app(argc, argv, &rank, &size);
Vector vec = createVectorMPI(30, &WorldComm, 1);
if (rank == 0) {
for (size_t i = 0; i < size; i++) {
printf(">> Proc %d owns [%d,%d]\n", i, vec->displ[i],
vec->displ[i] + vec->sizes[i] - 1);
}
}
close_app();
return 0;
}
Matrix createMatrixMPI(int n1, int n2, int N1, int N2, MPI_Comm* comm)
{
int i;
Matrix result = (Matrix)calloc(1, sizeof(matrix_t));
int topo_type;
MPI_Topo_test(*comm, &topo_type);
if (topo_type == MPI_CART) {
result->rows = n1;
result->cols = n2;
result->as_vec = malloc(sizeof(vector_t));
result->as_vec->comm = comm;
result->as_vec->len = n1*n2;
result->as_vec->stride = 1;
MPI_Comm_rank(*comm, &result->as_vec->comm_rank);
MPI_Comm_size(*comm, &result->as_vec->comm_size);
result->data = (double **)calloc(n2 ,sizeof(double *));
result->data[0] = (double *)calloc(n1*n2,sizeof(double));
result->as_vec->data = result->data[0];
for (i=1; i < n2; i++)
result->data[i] = result->data[i-1] + n1;
result->col = malloc(n2*sizeof(Vector));
for (int i=0;i<n2;++i) {
result->col[i] = malloc(sizeof(vector_t));
result->col[i]->data = result->data[i];
result->col[i]->stride = 1;
result->col[i]->len = n1;
}
result->row = malloc(n1*sizeof(Vector));
for (int i=0;i<n1;++i) {
result->row[i] = malloc(sizeof(vector_t));
result->row[i]->data = result->data[0]+i;
result->row[i]->stride = n1;
result->row[i]->len = n2;
}
return result;
}
result->as_vec = createVectorMPI(N1*N2, 0, comm);
int n12 = n1;
if (n1 == -1)
n1 = result->as_vec->len/N2;
else
n2 = result->as_vec->len/N1;
result->rows = n1;
result->cols = n2;
result->glob_rows = N1;
result->glob_cols = N2;
result->data = (double **)calloc(n2 ,sizeof(double *));
result->data[0] = (double *)calloc(n1*n2,sizeof(double));
result->as_vec->data = result->data[0];
for (i=1; i < n2; i++)
result->data[i] = result->data[i-1] + n1;
result->col = malloc(n2*sizeof(Vector));
for (int i=0;i<n2;++i) {
if (n12 == N1) {
result->col[i] = createVectorMPI(N1, 0, &SelfComm);
} else {
result->col[i] = createVectorMPI(N1, 0, comm);
}
result->col[i]->data = result->data[i];
}
result->row = malloc(n1*sizeof(Vector));
for (int i=0;i<n1;++i) {
if (n12 == N1)
result->row[i] = createVectorMPI(N2, 0, comm);
else
result->row[i] = createVectorMPI(N2, 0, &SelfComm);
result->row[i]->data = result->data[0]+i;
result->row[i]->stride = n1;
}
return result;
}
int main(int argc, char** argv)
{
int i, j, N, flag;
Matrix A=NULL, Q=NULL;
Vector b, grid, e, lambda=NULL;
double time, sum, h, tol=1e-6;
int rank, size;
int mpi_top_coords;
int mpi_top_sizes;
init_app(argc, argv, &rank, &size);
if (argc < 3) {
printf("need two parameters, N and flag [and tolerance]\n");
printf(" - N is the problem size (in each direction\n");
printf(" - flag = 1 -> Matrix-free Gauss-Jacobi iterations\n");
printf(" - flag = 2 -> Matrix-free red-black Gauss-Seidel iterations\n");
printf(" - flag = 3 -> Matrix-free CG iterations\n");
printf(" - flag = 4 -> Matrix-free additive schwarz preconditioned+Cholesky CG iterations\n");
printf(" - flag = 5 -> Matrix-free additive schwarz preconditioned+CG CG iterations\n");
return 1;
}
N=atoi(argv[1]);
flag=atoi(argv[2]);
if (argc > 3)
tol=atof(argv[3]);
if (N < 0) {
if (rank == 0)
printf("invalid problem size given\n");
close_app();
return 2;
}
if (flag < 0 || flag > 5) {
if (rank == 0)
printf("invalid flag given\n");
close_app();
return 3;
}
if (flag == 2 && (N-1)%2 != 0 && ((N-1)/size) % 2 != 0) {
if (rank == 0)
printf("need an even size (per process) for red-black iterations\n");
close_app();
return 4;
}
// setup topology
mpi_top_coords = 0;
mpi_top_sizes = 0;
MPI_Dims_create(size, 1, &mpi_top_sizes);
int periodic = 0;
MPI_Comm comm;
MPI_Cart_create(MPI_COMM_WORLD, 1, &mpi_top_sizes, &periodic, 0, &comm);
MPI_Cart_coords(comm, rank, 1, &mpi_top_coords);
b = createVectorMPI(N+1, &comm, 1, 1);
e = createVectorMPI(N+1, &comm, 1, 1);
grid = equidistantMesh(0.0, 1.0, N);
h = 1.0/N;
evalMeshDispl(b, grid, source);
scaleVector(b, pow(h, 2));
evalMeshDispl(e, grid, exact);
axpy(b, e, alpha);
b->data[0] = b->data[b->len-1] = 0.0;
if (flag == 4) {
int size = b->len;
if (b->comm_rank == 0)
size--;
if (b->comm_rank == b->comm_size-1)
size--;
A1D = createMatrix(size, size);
A1Dfactored = 0;
diag(A1D, -1, -1.0);
diag(A1D, 0, 2.0+alpha);
diag(A1D, 1, -1.0);
}
int its=-1;
char method[128];
time = WallTime();
if (flag == 1) {
its=GaussJacobiPoisson1D(b, tol, 1000000);
sprintf(method,"Gauss-Jacobi");
}
if (flag == 2) {
its=GaussSeidelPoisson1Drb(b, tol, 1000000);
sprintf(method,"Gauss-Seidel");
}
if (flag == 3) {
its=cgMatrixFree(Poisson1D, b, tol);
sprintf(method,"CG");
}
if (flag == 4 || flag == 5) {
its=pcgMatrixFree(Poisson1D, Poisson1DPre, b, tol);
sprintf(method,"PCG");
}
if (rank == 0) {
printf("%s used %i iterations\n", method, its);
printf("elapsed: %f\n", WallTime()-time);
}
evalMeshDispl(e, grid, exact);
axpy(b,e,-1.0);
b->data[0] = b->data[b->len-1] = 0.0;
h = maxNorm(b);
if (rank == 0)
printf("max error: %e\n", h);
if (A)
freeMatrix(A);
if (Q)
freeMatrix(Q);
freeVector(grid);
freeVector(b);
freeVector(e);
if (lambda)
freeVector(lambda);
if (A1D)
freeMatrix(A1D);
MPI_Comm_free(&comm);
close_app();
return 0;
}
