/*!
Use p == 1 to get maximum norm.
*/
template<class Grid_T> double get_diff_lp_norm(
const std::vector<uint64_t>& cells,
const Grid_T& grid,
const double p,
const double cell_volume,
const size_t dimension
) {
double local_norm = 0, global_norm = 0;
for (const auto& cell: cells) {
const auto* const cell_data = grid[cell];
if (cell_data == NULL) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": No data for cell " << cell
<< std::endl;
abort();
}
local_norm += std::pow(
std::fabs((*cell_data)[Vector_Field()][dimension] - div_removed_function()),
p
);
}
local_norm *= cell_volume;
if (p == 1) {
MPI_Comm comm = grid.get_communicator();
MPI_Allreduce(&local_norm, &global_norm, 1, MPI_DOUBLE, MPI_MAX, comm);
MPI_Comm_free(&comm);
return global_norm;
} else {
MPI_Comm comm = grid.get_communicator();
MPI_Allreduce(&local_norm, &global_norm, 1, MPI_DOUBLE, MPI_SUM, comm);
MPI_Comm_free(&comm);
return std::pow(global_norm, 1.0 / p);
}
}
int main(int argc, char* argv[])
{
if (MPI_Init(&argc, &argv) != MPI_SUCCESS) {
std::cerr << "Couldn't initialize MPI." << std::endl;
abort();
}
MPI_Comm comm = MPI_COMM_WORLD;
int rank = 0, comm_size = 0;
if (MPI_Comm_rank(comm, &rank) != MPI_SUCCESS) {
std::cerr << "Couldn't obtain MPI rank." << std::endl;
abort();
}
if (MPI_Comm_size(comm, &comm_size) != MPI_SUCCESS) {
std::cerr << "Couldn't obtain size of MPI communicator." << std::endl;
abort();
}
// intialize Zoltan
float zoltan_version;
if (Zoltan_Initialize(argc, argv, &zoltan_version) != ZOLTAN_OK) {
std::cerr << "Zoltan_Initialize failed." << std::endl;
abort();
}
const unsigned int neighborhood_size = 0;
const int max_refinement_level = 0;
double old_div = std::numeric_limits<double>::max();
size_t old_nr_of_cells = 0;
for (size_t nr_of_cells = 8; nr_of_cells <= 128; nr_of_cells *= 2) {
dccrg::Dccrg<Cell, dccrg::Cartesian_Geometry> grid;
const std::array<uint64_t, 3> grid_size{{1, nr_of_cells + 2, nr_of_cells + 2}};
if (
not grid.initialize(
grid_size,
comm,
"RANDOM",
neighborhood_size,
max_refinement_level,
false,
false,
false
)
) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Couldn't initialize grid."
<< std::endl;
abort();
}
const std::array<double, 3>
cell_length{{
1,
2 * M_PI / (grid_size[1] - 2),
2 * M_PI / (grid_size[2] - 2)
}},
grid_start{{
0, -cell_length[1], -cell_length[2]
}};
dccrg::Cartesian_Geometry::Parameters geom_params;
geom_params.start = grid_start;
geom_params.level_0_cell_length = cell_length;
if (not grid.set_geometry(geom_params)) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Couldn't set grid geometry."
<< std::endl;
abort();
}
grid.balance_load();
const auto all_cells = grid.get_cells();
for (const auto& cell: all_cells) {
auto* const cell_data = grid[cell];
if (cell_data == NULL) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": No data for cell " << cell
<< std::endl;
abort();
}
const auto center = grid.geometry.get_center(cell);
(*cell_data)[Vector_Field()] = function(center);
}
grid.update_copies_of_remote_neighbors();
// classify cells
std::vector<uint64_t>
solve_cells,
boundary_cells;
for (const auto& cell: all_cells) {
const auto index = grid.mapping.get_indices(cell);
if (
index[1] > 0
and index[1] < grid_size[1] - 1
and index[2] > 0
and index[2] < grid_size[2] - 1
) {
solve_cells.push_back(cell);
} else {
boundary_cells.push_back(cell);
}
}
// apply copy boundaries
for (const auto& cell: boundary_cells) {
const auto index = grid.mapping.get_indices(cell);
auto neighbor_index = index;
if (index[1] == 0) {
neighbor_index[1] = index[1] + 1;
} else if (index[1] == grid_size[1] - 1) {
neighbor_index[1] = index[1] - 1;
} else if (index[2] == 0) {
neighbor_index[2] = index[2] + 1;
} else if (index[2] == grid_size[2] - 1) {
neighbor_index[2] = index[2] - 1;
}
const auto neighbor = grid.mapping.get_cell_from_indices(neighbor_index, 0);
const auto* const neighbor_data = grid[neighbor];
auto* const cell_data = grid[cell];
if (cell_data == NULL or neighbor_data == NULL) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
(*cell_data)[Vector_Field()] = (*neighbor_data)[Vector_Field()];
}
grid.update_copies_of_remote_neighbors();
auto Vector_Getter = [](Cell& cell_data) -> Vector_Field::data_type& {
return cell_data[Vector_Field()];
};
auto Div_After_Getter = [](Cell& cell_data) -> Divergence_After::data_type& {
return cell_data[Divergence_After()];
};
const double div_before = pamhd::divergence::get_divergence(
solve_cells,
grid,
Vector_Getter,
[](Cell& cell_data) -> Divergence_Before::data_type& {
return cell_data[Divergence_Before()];
}
);
pamhd::divergence::remove(
solve_cells,
boundary_cells,
{},
grid,
Vector_Getter,
Div_After_Getter,
[](Cell& cell_data) -> Gradient::data_type& {
return cell_data[Gradient()];
},
2000, 0, 1e-15, 2, 100, 0, false, false
);
grid.update_copies_of_remote_neighbors();
// update copy boundaries to correspond to removed divergence
for (const auto& cell: boundary_cells) {
const auto index = grid.mapping.get_indices(cell);
auto neighbor_index = index;
if (index[1] == 0) {
neighbor_index[1] = index[1] + 1;
} else if (index[1] == grid_size[1] - 1) {
neighbor_index[1] = index[1] - 1;
} else if (index[2] == 0) {
neighbor_index[2] = index[2] + 1;
} else if (index[2] == grid_size[2] - 1) {
neighbor_index[2] = index[2] - 1;
}
const auto neighbor = grid.mapping.get_cell_from_indices(neighbor_index, 0);
const auto* const neighbor_data = grid[neighbor];
auto* const cell_data = grid[cell];
if (cell_data == NULL or neighbor_data == NULL) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
(*cell_data)[Vector_Field()] = (*neighbor_data)[Vector_Field()];
}
grid.update_copies_of_remote_neighbors();
const double div_after = pamhd::divergence::get_divergence(
solve_cells,
grid,
Vector_Getter,
Div_After_Getter
);
if (div_after > div_before) {
if (grid.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Divergence after removal " << div_after
<< " is larger than before " << div_before
<< " with " << nr_of_cells << " cells."
<< std::endl;
}
abort();
}
if (old_nr_of_cells > 0) {
const double
order_of_accuracy
= -log(div_after / old_div)
/ log(double(nr_of_cells) / old_nr_of_cells);
if (order_of_accuracy < 0.9) {
if (grid.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Order of accuracy from "
<< old_nr_of_cells << " to " << nr_of_cells
<< " is too low: " << order_of_accuracy
<< std::endl;
}
abort();
}
}
old_nr_of_cells = nr_of_cells;
old_div = div_after;
}
MPI_Finalize();
return EXIT_SUCCESS;
}
int main(int argc, char* argv[])
{
if (MPI_Init(&argc, &argv) != MPI_SUCCESS) {
std::cerr << "Couldn't initialize MPI." << std::endl;
abort();
}
MPI_Comm comm = MPI_COMM_WORLD;
int rank = 0, comm_size = 0;
if (MPI_Comm_rank(comm, &rank) != MPI_SUCCESS) {
std::cerr << "Couldn't obtain MPI rank." << std::endl;
abort();
}
if (MPI_Comm_size(comm, &comm_size) != MPI_SUCCESS) {
std::cerr << "Couldn't obtain size of MPI communicator." << std::endl;
abort();
}
// intialize Zoltan
float zoltan_version;
if (Zoltan_Initialize(argc, argv, &zoltan_version) != ZOLTAN_OK) {
std::cerr << "Zoltan_Initialize failed." << std::endl;
abort();
}
double
old_norm_x = std::numeric_limits<double>::max(),
old_norm_y = std::numeric_limits<double>::max(),
old_norm_z = std::numeric_limits<double>::max();
size_t old_nr_of_cells = 0;
for (size_t nr_of_cells = 8; nr_of_cells <= 2048; nr_of_cells *= 2) {
dccrg::Dccrg<Cell, dccrg::Cartesian_Geometry> grid_x, grid_y, grid_z;
const std::array<uint64_t, 3>
grid_size_x{{nr_of_cells + 2, 1, 1}},
grid_size_y{{1, nr_of_cells + 2, 1}},
grid_size_z{{1, 1, nr_of_cells + 2}};
if (not grid_x.initialize(grid_size_x,comm,"RANDOM",0,0,false,false,false)) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
if (not grid_y.initialize(grid_size_y,comm,"RANDOM",0,0,false,false,false)) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
if (not grid_z.initialize(grid_size_z,comm,"RANDOM",0,0,false,false,false)) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
const std::array<double, 3>
cell_length_x{{2 * M_PI / (grid_size_x[0] - 2), 1, 1}},
cell_length_y{{1, 2 * M_PI / (grid_size_y[0] - 2), 1}},
cell_length_z{{1, 1, 2 * M_PI / (grid_size_z[0] - 2)}},
grid_start_x{{-cell_length_x[0], 0, 0}},
grid_start_y{{0, -cell_length_y[1], 0}},
grid_start_z{{0, 0, -cell_length_z[2]}};
const double cell_volume
= cell_length_x[0] * cell_length_x[1] * cell_length_x[2];
dccrg::Cartesian_Geometry::Parameters geom_params_x,geom_params_y,geom_params_z;
geom_params_x.start = grid_start_x;
geom_params_x.level_0_cell_length = cell_length_x;
geom_params_y.start = grid_start_y;
geom_params_y.level_0_cell_length = cell_length_y;
geom_params_z.start = grid_start_z;
geom_params_z.level_0_cell_length = cell_length_z;
if (not grid_x.set_geometry(geom_params_x)) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
if (not grid_y.set_geometry(geom_params_y)) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
if (not grid_z.set_geometry(geom_params_z)) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
const auto all_cells = grid_x.get_cells();
std::vector<uint64_t>
solve_cells,
boundary_cells;
for (const auto& cell: all_cells) {
auto
*const cell_data_x = grid_x[cell],
*const cell_data_y = grid_y[cell],
*const cell_data_z = grid_z[cell];
if (cell_data_x == NULL or cell_data_y == NULL or cell_data_z == NULL) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
const auto index = grid_x.mapping.get_indices(cell);
if (index[0] > 0 and index[0] < grid_size_x[0] - 1) {
solve_cells.push_back(cell);
} else {
boundary_cells.push_back(cell);
}
const auto x = grid_x.geometry.get_center(cell)[0];
auto
&vec_x = (*cell_data_x)[Vector_Field()],
&vec_y = (*cell_data_y)[Vector_Field()],
&vec_z = (*cell_data_z)[Vector_Field()];
vec_x[0] =
vec_y[1] =
vec_z[2] = function(x);
vec_x[1] =
vec_x[2] =
vec_y[0] =
vec_y[2] =
vec_z[0] =
vec_z[1] = 0;
}
grid_x.update_copies_of_remote_neighbors();
grid_y.update_copies_of_remote_neighbors();
grid_z.update_copies_of_remote_neighbors();
// use copy boundaries
const uint64_t
neg_bdy_cell = 1,
pos_bdy_cell = nr_of_cells + 2;
if (grid_x.is_local(neg_bdy_cell)) {
const auto
*const neighbor_data_x = grid_x[neg_bdy_cell + 1],
*const neighbor_data_y = grid_y[neg_bdy_cell + 1],
*const neighbor_data_z = grid_z[neg_bdy_cell + 1];
auto
*const cell_data_x = grid_x[neg_bdy_cell],
*const cell_data_y = grid_y[neg_bdy_cell],
*const cell_data_z = grid_z[neg_bdy_cell];
if (
cell_data_x == NULL or neighbor_data_x == NULL
or cell_data_y == NULL or neighbor_data_y == NULL
or cell_data_z == NULL or neighbor_data_z == NULL
) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
(*cell_data_x)[Vector_Field()] = (*neighbor_data_x)[Vector_Field()];
(*cell_data_y)[Vector_Field()] = (*neighbor_data_y)[Vector_Field()];
(*cell_data_z)[Vector_Field()] = (*neighbor_data_z)[Vector_Field()];
}
if (grid_x.is_local(pos_bdy_cell)) {
const auto
*const neighbor_data_x = grid_x[pos_bdy_cell - 1],
*const neighbor_data_y = grid_y[pos_bdy_cell - 1],
*const neighbor_data_z = grid_z[pos_bdy_cell - 1];
auto
*const cell_data_x = grid_x[pos_bdy_cell],
*const cell_data_y = grid_y[pos_bdy_cell],
*const cell_data_z = grid_z[pos_bdy_cell];
if (
cell_data_x == NULL or neighbor_data_x == NULL
or cell_data_y == NULL or neighbor_data_y == NULL
or cell_data_z == NULL or neighbor_data_z == NULL
) {
std::cerr << __FILE__ << ":" << __LINE__ << std::endl;
abort();
}
(*cell_data_x)[Vector_Field()] = (*neighbor_data_x)[Vector_Field()];
(*cell_data_y)[Vector_Field()] = (*neighbor_data_y)[Vector_Field()];
(*cell_data_z)[Vector_Field()] = (*neighbor_data_z)[Vector_Field()];
}
auto Vector_Getter = [](Cell& cell_data) -> Vector_Field::data_type& {
return cell_data[Vector_Field()];
};
auto Divergence_Getter = [](Cell& cell_data) -> Divergence::data_type& {
return cell_data[Divergence()];
};
auto Gradient_Getter = [](Cell& cell_data) -> Gradient::data_type& {
return cell_data[Gradient()];
};
pamhd::divergence::remove(
solve_cells,
boundary_cells,
{},
grid_x,
Vector_Getter,
Divergence_Getter,
Gradient_Getter,
2000, 0, 1e-15, 2, 100, false
);
pamhd::divergence::remove(
solve_cells,
boundary_cells,
{},
grid_y,
Vector_Getter,
Divergence_Getter,
Gradient_Getter,
2000, 0, 1e-15, 2, 100, false
);
pamhd::divergence::remove(
solve_cells,
boundary_cells,
{},
grid_z,
Vector_Getter,
Divergence_Getter,
Gradient_Getter,
2000, 0, 1e-15, 2, 100, false
);
const double
p_of_norm = 2,
norm_x = get_diff_lp_norm(solve_cells, grid_x, p_of_norm, cell_volume, 0),
norm_y = get_diff_lp_norm(solve_cells, grid_y, p_of_norm, cell_volume, 1),
norm_z = get_diff_lp_norm(solve_cells, grid_z, p_of_norm, cell_volume, 2);
if (norm_x > old_norm_x) {
if (grid_x.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": X norm with " << nr_of_cells
<< " cells " << norm_x
<< " is larger than with " << nr_of_cells / 2
<< " cells " << old_norm_x
<< std::endl;
}
abort();
}
if (norm_y > old_norm_y) {
if (grid_y.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Y norm with " << nr_of_cells
<< " cells " << norm_y
<< " is larger than with " << nr_of_cells / 2
<< " cells " << old_norm_y
<< std::endl;
}
abort();
}
if (norm_y > old_norm_z) {
if (grid_z.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Z norm with " << nr_of_cells
<< " cells " << norm_z
<< " is larger than with " << nr_of_cells / 2
<< " cells " << old_norm_z
<< std::endl;
}
abort();
}
if (old_nr_of_cells > 0) {
const double
order_of_accuracy_x
= -log(norm_x / old_norm_x)
/ log(double(nr_of_cells) / old_nr_of_cells),
order_of_accuracy_y
= -log(norm_y / old_norm_y)
/ log(double(nr_of_cells) / old_nr_of_cells),
order_of_accuracy_z
= -log(norm_z / old_norm_z)
/ log(double(nr_of_cells) / old_nr_of_cells);
if (order_of_accuracy_x < 1.5) {
if (grid_x.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": X order of accuracy from "
<< old_nr_of_cells << " to " << nr_of_cells
<< " is too low: " << order_of_accuracy_x
<< std::endl;
}
abort();
}
if (order_of_accuracy_y < 1.5) {
if (grid_y.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Y order of accuracy from "
<< old_nr_of_cells << " to " << nr_of_cells
<< " is too low: " << order_of_accuracy_y
<< std::endl;
}
abort();
}
if (order_of_accuracy_z < 1.5) {
if (grid_z.get_rank() == 0) {
std::cerr << __FILE__ << ":" << __LINE__
<< ": Z order of accuracy from "
<< old_nr_of_cells << " to " << nr_of_cells
<< " is too low: " << order_of_accuracy_z
<< std::endl;
}
abort();
}
}
old_nr_of_cells = nr_of_cells;
old_norm_x = norm_x;
old_norm_y = norm_y;
old_norm_z = norm_z;
}
MPI_Finalize();
return EXIT_SUCCESS;
}
