void hypre_F90_IFACE(hypre_structvectorprint, HYPRE_STRUCTVECTORPRINT) ( hypre_F90_Obj *vector, hypre_F90_Int *all, hypre_F90_Int *ierr ) { *ierr = (hypre_F90_Int) ( HYPRE_StructVectorPrint( "HYPRE_StructVector.out", hypre_F90_PassObj (HYPRE_StructVector, vector), hypre_F90_PassInt (all)) ); }
double *solve(double *Ab, int solver_id, struct parms parms) { int i, j; double final_res_norm; int time_index, n_pre, n_post, num_iterations; n_pre = 1; n_post = 1; double *A_val, *b_val; A_val = (double *) calloc(parms.N*parms.nsten, sizeof(double)); b_val = (double *) calloc(parms.N, sizeof(double)); for (i = 0; i < (parms.N*parms.nsten); i++){ A_val[i] = Ab[i]; } for (i = 0; i < parms.N; i++){ b_val[i] = Ab[i+parms.N*parms.nsten]; } // HYPRE // HYPRE_StructGrid grid; HYPRE_StructStencil stencil; HYPRE_StructMatrix A; HYPRE_StructVector b; HYPRE_StructVector x; HYPRE_StructSolver solver; HYPRE_StructSolver precond; #if Dim == 2 HYPRE_Int ilower[2] = {parms.x0, parms.y0}; HYPRE_Int iupper[2] = {parms.x1, parms.y1}; #endif #if Dim == 3 HYPRE_Int ilower[3] = {parms.x0, parms.y0, 0}; HYPRE_Int iupper[3] = {parms.x1, parms.y1, parms.Nz-1}; #endif { // Create an empty 2D grid object HYPRE_StructGridCreate(MPI_COMM_WORLD, Dim, &grid); // Add a new box to the grid HYPRE_StructGridSetExtents(grid, ilower, iupper); // 1. Set up periodic boundary condition in y-direction and create the grid int pset[3]; pset[0] = 0; pset[1] = parms.Ny; pset[2] = 0; #if Dim == 3 pset[2] = parms.Nz; #endif //HYPRE_StructGridSetNumGhost(grid,pset) HYPRE_StructGridSetPeriodic(grid, pset); HYPRE_StructGridAssemble(grid); } // 2. Define the discretization stencil { if (Dim == 2){ // Create an empty 2D, 5-pt stencil object HYPRE_StructStencilCreate(2, parms.nsten, &stencil); // Define the geometry of the stencil { int offsets[5][2] = {{0,0}, {-1,0}, {0,-1}, {0,1}, {1,0}}; for (i = 0; i < parms.nsten; i++) HYPRE_StructStencilSetElement(stencil, i, offsets[i]); } } else { HYPRE_StructStencilCreate(3, parms.nsten, &stencil); // Define the geometry of the 3D stencil { int offsets[7][3] = {{0,0,0}, {-1,0,0}, {0,-1,0}, {0,1,0}, {1,0,0}, {0,0,-1}, {0,0,1}}; for (i = 0; i < parms.nsten; i++) HYPRE_StructStencilSetElement(stencil, i, offsets[i]); } } } // 3. Set up a Struct Matrix A from Aval { HYPRE_Int stencil_indices[parms.nsten]; // Create an empty matrix object HYPRE_StructMatrixCreate(MPI_COMM_WORLD, grid, stencil, &A); // Indicate that the matrix coefficients are ready to be set HYPRE_StructMatrixInitialize(A); for (j = 0; j < parms.nsten; j++) stencil_indices[j] = j; HYPRE_StructMatrixSetBoxValues(A, ilower, iupper, parms.nsten, stencil_indices, A_val); free(A_val); } // 4. Set up Struct Vectors for b from b_val and set x = 0 { double *values; HYPRE_StructVectorCreate(MPI_COMM_WORLD, grid, &b); HYPRE_StructVectorCreate(MPI_COMM_WORLD, grid, &x); HYPRE_StructVectorInitialize(b); HYPRE_StructVectorInitialize(x); values = calloc((parms.N), sizeof(double)); for (i = 0; i < (parms.N); i++) values[i] = 0.0; HYPRE_StructVectorSetBoxValues(x, ilower, iupper, values); HYPRE_StructVectorSetBoxValues(b, ilower, iupper, b_val); free(b_val); free(values); } //Finalize the vector and matrix assembly HYPRE_StructMatrixAssemble(A); HYPRE_StructVectorAssemble(b); HYPRE_StructVectorAssemble(x); #if DEBUG == 3 HYPRE_StructMatrixPrint("./poisson.matrix", A, 0); HYPRE_StructVectorPrint("./poisson.rhs", b, 0); /*char fname[64]; char Aname[64], bname[64]; sprintf(Aname,"data/A%d.",parms.cyc); sprintf(bname,"data/b%d.",parms.cyc); filename(fname, Aname, parms.wkdir, parms); HYPRE_StructMatrixPrint(fname, A, 0); filename(fname, bname, parms.wkdir, parms); HYPRE_StructVectorPrint(fname, b, 0);*/ #endif // 6. Set up and use a solver (SMG) if (solver_id == 0) { time_index = hypre_InitializeTiming("SMG Setup"); hypre_BeginTiming(time_index); HYPRE_StructSMGCreate(MPI_COMM_WORLD, &solver); HYPRE_StructSMGSetMemoryUse(solver, 0); HYPRE_StructSMGSetMaxIter(solver, 100); HYPRE_StructSMGSetTol(solver, 1.0e-12); HYPRE_StructSMGSetRelChange(solver, 0); HYPRE_StructSMGSetNumPreRelax(solver, n_pre); HYPRE_StructSMGSetNumPostRelax(solver, n_post); // Logging must be on to get iterations and residual norm info below HYPRE_StructSMGSetLogging(solver, 1); // Setup and print setup timings HYPRE_StructSMGSetup(solver, A, b, x); hypre_EndTiming(time_index); #if DEBUG == 3 hypre_PrintTiming("Setup phase times", MPI_COMM_WORLD); #endif hypre_FinalizeTiming(time_index); hypre_ClearTiming(); // Solve and print solve timings time_index = hypre_InitializeTiming("SMG Solve"); hypre_BeginTiming(time_index); HYPRE_StructSMGSolve(solver, A, b, x); hypre_EndTiming(time_index); #if DEBUG == 3 hypre_PrintTiming("Solve phase times", MPI_COMM_WORLD); #endif hypre_FinalizeTiming(time_index); hypre_ClearTiming(); // Get some info on the run HYPRE_StructSMGGetNumIterations(solver, &num_iterations); HYPRE_StructSMGGetFinalRelativeResidualNorm(solver, &final_res_norm); #if DEBUG == 2 if (parms.rank == 0){ fprintf(stdout, "Number of Iterations = %4d ; Final Relative Residual Norm = %e\n\n", num_iterations, final_res_norm); } #endif // Clean up HYPRE_StructSMGDestroy(solver); } // 6. Set up and use a solver (PCG) with SMG Preconditioner if (solver_id == 1) { HYPRE_StructPCGCreate(MPI_COMM_WORLD, &solver); //HYPRE_StructPCGSetMemoryUse(solver, 0); HYPRE_StructPCGSetMaxIter(solver, 100); HYPRE_StructPCGSetTol(solver, 1.0e-12); HYPRE_StructPCGSetTwoNorm(solver, 1); HYPRE_StructPCGSetRelChange(solver, 0); //HYPRE_StructPCGSetPrintLevel(solver, 2 ); /* print each CG iteration */ HYPRE_StructPCGSetLogging(solver, 1); /* Use symmetric SMG as preconditioner */ HYPRE_StructSMGCreate(MPI_COMM_WORLD, &precond); HYPRE_StructSMGSetMemoryUse(precond, 0); HYPRE_StructSMGSetMaxIter(precond, 32); HYPRE_StructSMGSetTol(precond, 0.0); HYPRE_StructSMGSetZeroGuess(precond); HYPRE_StructSMGSetNumPreRelax(precond, 1); HYPRE_StructSMGSetNumPostRelax(precond, 1); /* Set the preconditioner and solve */ HYPRE_StructPCGSetPrecond(solver, HYPRE_StructSMGSolve, HYPRE_StructSMGSetup, precond); HYPRE_StructPCGSetup(solver, A, b, x); HYPRE_StructPCGSolve(solver, A, b, x); /* Get some info on the run */ HYPRE_StructPCGGetNumIterations(solver, &num_iterations); HYPRE_StructPCGGetFinalRelativeResidualNorm(solver, &final_res_norm); #if DEBUG == 2 if (parms.rank == 0){ fprintf(stdout, "Number of Iterations = %4d ; Final Relative Residual Norm = %e\n\n", num_iterations, final_res_norm); } #endif /* Clean up */ HYPRE_StructSMGDestroy(precond); HYPRE_StructPCGDestroy(solver); } // get the local solution double *values = calloc(parms.N, sizeof(double)); HYPRE_StructVectorGetBoxValues(x, ilower, iupper, values); // Free memory HYPRE_StructGridDestroy(grid); HYPRE_StructStencilDestroy(stencil); HYPRE_StructMatrixDestroy(A); HYPRE_StructVectorDestroy(b); HYPRE_StructVectorDestroy(x); free(Ab); return(values); }
hypre_int main( hypre_int argc, char *argv[] ) { HYPRE_Int arg_index; HYPRE_Int print_usage; HYPRE_Int nx, ny, nz; HYPRE_Int P, Q, R; HYPRE_Int bx, by, bz; HYPRE_StructGrid from_grid, to_grid; HYPRE_StructVector from_vector, to_vector, check_vector; HYPRE_CommPkg comm_pkg; HYPRE_Int time_index; HYPRE_Int num_procs, myid; HYPRE_Int p, q, r; HYPRE_Int dim; HYPRE_Int nblocks ; HYPRE_Int **ilower, **iupper, **iupper2; HYPRE_Int istart[3]; HYPRE_Int i, ix, iy, iz, ib; HYPRE_Int print_system = 0; HYPRE_Real check; /*----------------------------------------------------------- * Initialize some stuff *-----------------------------------------------------------*/ /* Initialize MPI */ hypre_MPI_Init(&argc, &argv); hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs ); hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid ); /*----------------------------------------------------------- * Set defaults *-----------------------------------------------------------*/ dim = 3; nx = 2; ny = 2; nz = 2; P = num_procs; Q = 1; R = 1; bx = 1; by = 1; bz = 1; istart[0] = 1; istart[1] = 1; istart[2] = 1; /*----------------------------------------------------------- * Parse command line *-----------------------------------------------------------*/ print_usage = 0; arg_index = 1; while (arg_index < argc) { if ( strcmp(argv[arg_index], "-n") == 0 ) { arg_index++; nx = atoi(argv[arg_index++]); ny = atoi(argv[arg_index++]); nz = atoi(argv[arg_index++]); } else if ( strcmp(argv[arg_index], "-istart") == 0 ) { arg_index++; istart[0] = atoi(argv[arg_index++]); istart[1] = atoi(argv[arg_index++]); istart[2] = atoi(argv[arg_index++]); } else if ( strcmp(argv[arg_index], "-P") == 0 ) { arg_index++; P = atoi(argv[arg_index++]); Q = atoi(argv[arg_index++]); R = atoi(argv[arg_index++]); } else if ( strcmp(argv[arg_index], "-b") == 0 ) { arg_index++; bx = atoi(argv[arg_index++]); by = atoi(argv[arg_index++]); bz = atoi(argv[arg_index++]); } else if ( strcmp(argv[arg_index], "-d") == 0 ) { arg_index++; dim = atoi(argv[arg_index++]); } else if ( strcmp(argv[arg_index], "-print") == 0 ) { arg_index++; print_system = 1; } else if ( strcmp(argv[arg_index], "-help") == 0 ) { print_usage = 1; break; } else { arg_index++; } } /*----------------------------------------------------------- * Print usage info *-----------------------------------------------------------*/ if ( (print_usage) && (myid == 0) ) { hypre_printf("\n"); hypre_printf("Usage: %s [<options>]\n", argv[0]); hypre_printf("\n"); hypre_printf(" -n <nx> <ny> <nz> : problem size per block\n"); hypre_printf(" -istart <ix> <iy> <iz> : start of box\n"); hypre_printf(" -P <Px> <Py> <Pz> : processor topology\n"); hypre_printf(" -b <bx> <by> <bz> : blocking per processor\n"); hypre_printf(" -d <dim> : problem dimension (2 or 3)\n"); hypre_printf(" -print : print vectors\n"); hypre_printf("\n"); } if ( print_usage ) { exit(1); } /*----------------------------------------------------------- * Check a few things *-----------------------------------------------------------*/ if ((P*Q*R) > num_procs) { if (myid == 0) { hypre_printf("Error: PxQxR is more than the number of processors\n"); } exit(1); } else if ((P*Q*R) < num_procs) { if (myid == 0) { hypre_printf("Warning: PxQxR is less than the number of processors\n"); } } /*----------------------------------------------------------- * Print driver parameters *-----------------------------------------------------------*/ if (myid == 0) { hypre_printf("Running with these driver parameters:\n"); hypre_printf(" (nx, ny, nz) = (%d, %d, %d)\n", nx, ny, nz); hypre_printf(" (ix, iy, iz) = (%d, %d, %d)\n", istart[0],istart[1],istart[2]); hypre_printf(" (Px, Py, Pz) = (%d, %d, %d)\n", P, Q, R); hypre_printf(" (bx, by, bz) = (%d, %d, %d)\n", bx, by, bz); hypre_printf(" dim = %d\n", dim); } /*----------------------------------------------------------- * Set up the stencil structure (7 points) when matrix is NOT read from file * Set up the grid structure used when NO files are read *-----------------------------------------------------------*/ switch (dim) { case 1: nblocks = bx; p = myid % P; break; case 2: nblocks = bx*by; p = myid % P; q = (( myid - p)/P) % Q; break; case 3: nblocks = bx*by*bz; p = myid % P; q = (( myid - p)/P) % Q; r = ( myid - p - P*q)/( P*Q ); break; } if (myid >= (P*Q*R)) { /* My processor has no data on it */ nblocks = bx = by = bz = 0; } /*----------------------------------------------------------- * prepare space for the extents *-----------------------------------------------------------*/ ilower = hypre_CTAlloc(HYPRE_Int*, nblocks); iupper = hypre_CTAlloc(HYPRE_Int*, nblocks); iupper2 = hypre_CTAlloc(HYPRE_Int*, nblocks); for (i = 0; i < nblocks; i++) { ilower[i] = hypre_CTAlloc(HYPRE_Int, dim); iupper[i] = hypre_CTAlloc(HYPRE_Int, dim); iupper2[i] = hypre_CTAlloc(HYPRE_Int, dim); } ib = 0; switch (dim) { case 1: for (ix = 0; ix < bx; ix++) { ilower[ib][0] = istart[0]+ nx*(bx*p+ix); iupper[ib][0] = istart[0]+ nx*(bx*p+ix+1) - 1; iupper2[ib][0] = iupper[ib][0]; if ( (ix == (bx-1)) && (p < (P-1)) ) iupper2[ib][0] = iupper[ib][0] + 1; ib++; } break; case 2: for (iy = 0; iy < by; iy++) for (ix = 0; ix < bx; ix++) { ilower[ib][0] = istart[0]+ nx*(bx*p+ix); iupper[ib][0] = istart[0]+ nx*(bx*p+ix+1) - 1; ilower[ib][1] = istart[1]+ ny*(by*q+iy); iupper[ib][1] = istart[1]+ ny*(by*q+iy+1) - 1; iupper2[ib][0] = iupper[ib][0]; iupper2[ib][1] = iupper[ib][1]; if ( (ix == (bx-1)) && (p < (P-1)) ) iupper2[ib][0] = iupper[ib][0] + 1; if ( (iy == (by-1)) && (q < (Q-1)) ) iupper2[ib][1] = iupper[ib][1] + 1; ib++; } break; case 3: for (iz = 0; iz < bz; iz++) for (iy = 0; iy < by; iy++) for (ix = 0; ix < bx; ix++) { ilower[ib][0] = istart[0]+ nx*(bx*p+ix); iupper[ib][0] = istart[0]+ nx*(bx*p+ix+1) - 1; ilower[ib][1] = istart[1]+ ny*(by*q+iy); iupper[ib][1] = istart[1]+ ny*(by*q+iy+1) - 1; ilower[ib][2] = istart[2]+ nz*(bz*r+iz); iupper[ib][2] = istart[2]+ nz*(bz*r+iz+1) - 1; iupper2[ib][0] = iupper[ib][0]; iupper2[ib][1] = iupper[ib][1]; iupper2[ib][2] = iupper[ib][2]; if ( (ix == (bx-1)) && (p < (P-1)) ) iupper2[ib][0] = iupper[ib][0] + 1; if ( (iy == (by-1)) && (q < (Q-1)) ) iupper2[ib][1] = iupper[ib][1] + 1; if ( (iz == (bz-1)) && (r < (R-1)) ) iupper2[ib][2] = iupper[ib][2] + 1; ib++; } break; } HYPRE_StructGridCreate(hypre_MPI_COMM_WORLD, dim, &from_grid); HYPRE_StructGridCreate(hypre_MPI_COMM_WORLD, dim, &to_grid); for (ib = 0; ib < nblocks; ib++) { HYPRE_StructGridSetExtents(from_grid, ilower[ib], iupper[ib]); HYPRE_StructGridSetExtents(to_grid, ilower[ib], iupper2[ib]); } HYPRE_StructGridAssemble(from_grid); HYPRE_StructGridAssemble(to_grid); /*----------------------------------------------------------- * Set up the vectors *-----------------------------------------------------------*/ HYPRE_StructVectorCreate(hypre_MPI_COMM_WORLD, from_grid, &from_vector); HYPRE_StructVectorInitialize(from_vector); AddValuesVector(from_grid, from_vector, 1.0); HYPRE_StructVectorAssemble(from_vector); HYPRE_StructVectorCreate(hypre_MPI_COMM_WORLD, to_grid, &to_vector); HYPRE_StructVectorInitialize(to_vector); AddValuesVector(to_grid, to_vector, 0.0); HYPRE_StructVectorAssemble(to_vector); /* Vector used to check the migration */ HYPRE_StructVectorCreate(hypre_MPI_COMM_WORLD, to_grid, &check_vector); HYPRE_StructVectorInitialize(check_vector); AddValuesVector(to_grid, check_vector, 1.0); HYPRE_StructVectorAssemble(check_vector); /*----------------------------------------------------------- * Migrate *-----------------------------------------------------------*/ time_index = hypre_InitializeTiming("Struct Migrate"); hypre_BeginTiming(time_index); HYPRE_StructVectorGetMigrateCommPkg(from_vector, to_vector, &comm_pkg); HYPRE_StructVectorMigrate(comm_pkg, from_vector, to_vector); HYPRE_CommPkgDestroy(comm_pkg); hypre_EndTiming(time_index); hypre_PrintTiming("Struct Migrate", hypre_MPI_COMM_WORLD); hypre_FinalizeTiming(time_index); /*----------------------------------------------------------- * Check the migration and print the result *-----------------------------------------------------------*/ hypre_StructAxpy(-1.0, to_vector, check_vector); check = hypre_StructInnerProd (check_vector, check_vector); if (myid == 0) { printf("\nCheck = %1.0f (success = 0)\n\n", check); } /*----------------------------------------------------------- * Print out the vectors *-----------------------------------------------------------*/ if (print_system) { HYPRE_StructVectorPrint("struct_migrate.out.xfr", from_vector, 0); HYPRE_StructVectorPrint("struct_migrate.out.xto", to_vector, 0); } /*----------------------------------------------------------- * Finalize things *-----------------------------------------------------------*/ HYPRE_StructGridDestroy(from_grid); HYPRE_StructGridDestroy(to_grid); for (i = 0; i < nblocks; i++) { hypre_TFree(ilower[i]); hypre_TFree(iupper[i]); hypre_TFree(iupper2[i]); } hypre_TFree(ilower); hypre_TFree(iupper); hypre_TFree(iupper2); HYPRE_StructVectorDestroy(from_vector); HYPRE_StructVectorDestroy(to_vector); HYPRE_StructVectorDestroy(check_vector); /* Finalize MPI */ hypre_MPI_Finalize(); return (0); }