int
main (int argc, char **argv)
{
int mpiret;
int mpirank, mpisize;
int i, j;
char cvalue, cresult;
int ivalue, iresult;
unsigned short usvalue, usresult;
long lvalue, lresult;
float fvalue[3], fresult[3], fexpect[3];
double dvalue, dresult;
MPI_Comm mpicomm;
mpiret = MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
mpicomm = MPI_COMM_WORLD;
mpiret = MPI_Comm_size (mpicomm, &mpisize);
SC_CHECK_MPI (mpiret);
mpiret = MPI_Comm_rank (mpicomm, &mpirank);
SC_CHECK_MPI (mpiret);
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
/* test allreduce int max */
ivalue = mpirank;
sc_allreduce (&ivalue, &iresult, 1, MPI_INT, MPI_MAX, mpicomm);
SC_CHECK_ABORT (iresult == mpisize - 1, "Allreduce mismatch");
/* test reduce float max */
fvalue[0] = (float) mpirank;
fexpect[0] = (float) (mpisize - 1);
fvalue[1] = (float) (mpirank % 9 - 4);
fexpect[1] = (float) (mpisize >= 9 ? 4 : (mpisize - 1) % 9 - 4);
fvalue[2] = (float) (mpirank % 6);
fexpect[2] = (float) (mpisize >= 6 ? 5 : (mpisize - 1) % 6);
for (i = 0; i < mpisize; ++i) {
sc_reduce (fvalue, fresult, 3, MPI_FLOAT, MPI_MAX, i, mpicomm);
if (i == mpirank) {
for (j = 0; j < 3; ++j) {
SC_CHECK_ABORTF (fresult[j] == fexpect[j], /* ok */
"Reduce mismatch in %d", j);
}
}
}
/* test allreduce char min */
cvalue = (char) (mpirank % 127);
sc_allreduce (&cvalue, &cresult, 1, MPI_CHAR, MPI_MIN, mpicomm);
SC_CHECK_ABORT (cresult == 0, "Allreduce mismatch");
/* test reduce unsigned short min */
usvalue = (unsigned short) (mpirank % 32767);
for (i = 0; i < mpisize; ++i) {
sc_reduce (&usvalue, &usresult, 1, MPI_UNSIGNED_SHORT, MPI_MIN, i,
mpicomm);
if (i == mpirank) {
SC_CHECK_ABORT (usresult == 0, "Reduce mismatch");
}
}
/* test allreduce long sum */
lvalue = (long) mpirank;
sc_allreduce (&lvalue, &lresult, 1, MPI_LONG, MPI_SUM, mpicomm);
SC_CHECK_ABORT (lresult == ((long) (mpisize - 1)) * mpisize / 2,
"Allreduce mismatch");
/* test reduce double sum */
dvalue = (double) mpirank;
for (i = 0; i < mpisize; ++i) {
sc_reduce (&dvalue, &dresult, 1, MPI_DOUBLE, MPI_SUM, i, mpicomm);
if (i == mpirank) {
SC_CHECK_ABORT (dresult == ((double) (mpisize - 1)) * mpisize / 2., /* ok */
"Reduce mismatch");
}
}
sc_finalize ();
mpiret = MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
void
curved_cg_nr_solve
(
p4est_t* p4est,
problem_data_t* vecs,
curved_weakeqn_ptrs_t* fcns,
void* params,
p4est_ghost_t* ghost,
curved_element_data_t* ghost_data,
dgmath_jit_dbase_t* dgmath_jit_dbase
)
{
int local_nodes;
double delta_new, delta_old, beta, alpha;
double* Au;
double* u;
double* rhs;
double* d;
double* r;
local_nodes = vecs->local_nodes;
Au = vecs->Au;
u = vecs->u;
rhs = vecs->rhs;
cg_nr_solver_params_t* cg_params = (cg_nr_solver_params_t*)params;
const int imax = cg_params->max_iter;
const double eta_max = fabs(cg_params->eta_max);
const double fnrm = cg_params->fnrm;
double d_dot_Au;
d = P4EST_ALLOC(double, local_nodes);
r = P4EST_ALLOC(double, local_nodes);
/* first iteration data, store Au in r */
debug("First Au calculation in CG solve starts");
fcns->apply_lhs(p4est, ghost, ghost_data, vecs, dgmath_jit_dbase);
debug("First Au calculation in CG solve ends");
linalg_copy_1st_to_2nd(Au, r, local_nodes);
/* r = f - Au ; Au is stored in r so r = rhs - r */
linalg_vec_xpby(rhs, -1., r, local_nodes);
linalg_copy_1st_to_2nd(r, d, local_nodes);
delta_new = linalg_vec_dot(r,r,local_nodes);
/* delta_new = (element_data_compute_l2_norm(p4est, r)); */
double delta_new_global;
double d_dot_Au_global;
sc_allreduce
(
&delta_new,
&delta_new_global,
1,
sc_MPI_DOUBLE,
sc_MPI_SUM,
sc_MPI_COMM_WORLD
);
delta_new = delta_new_global;
/* delta_0 = delta_new; */
/* start working on d */
vecs->u = d;
int i;
for (i = 0; i < imax && (delta_new > eta_max*fnrm*eta_max*fnrm); i++){
/* Au = A*d; */
fcns->apply_lhs
(
p4est,
ghost,
ghost_data,
vecs,
dgmath_jit_dbase
);
/* sc_MPI_Barrier(sc_MPI_COMM_WORLD); */
d_dot_Au = linalg_vec_dot(d,Au,local_nodes);
sc_allreduce
(
&d_dot_Au,
&d_dot_Au_global,
1,
sc_MPI_DOUBLE,
sc_MPI_SUM,
sc_MPI_COMM_WORLD
);
d_dot_Au = d_dot_Au_global;
alpha = delta_new/d_dot_Au;
linalg_vec_axpy(alpha, d, u, local_nodes);
/* r = r - Au*alpha */
linalg_vec_axpy(-alpha, Au, r, local_nodes);
delta_old = delta_new;
delta_new = linalg_vec_dot(r, r, local_nodes);
sc_allreduce
(
&delta_new,
&delta_new_global,
1,
sc_MPI_DOUBLE,
sc_MPI_SUM,
sc_MPI_COMM_WORLD
);
delta_new = delta_new_global;
beta = delta_new/delta_old;
linalg_vec_xpby(r, beta, d, local_nodes);
#ifdef VERBOSE
/* log_info_quiet("%d CG ITER %03d RNRMSQR %.10f", */
/* mpi_rank, i, delta_new); */
if (cg_params->mpi_rank == 0)
printf("[CG_NR_SOLVER]: ITER %03d FNRMSQR %.20f\n", i, delta_new_global);
#endif
}
cg_params->final_iter = i;
cg_params->final_fnrm = sqrt(delta_new_global);
vecs->u = u;
P4EST_FREE(d);
P4EST_FREE(r);
}