void Opm::ReorderSolverInterface::reorderAndTransport(const UnstructuredGrid& grid, const double* darcyflux)
{
// Compute reordered sequence of single-cell problems
sequence_.resize(grid.number_of_cells);
components_.resize(grid.number_of_cells + 1);
int ncomponents;
time::StopWatch clock;
clock.start();
compute_sequence(&grid, darcyflux, &sequence_[0], &components_[0], &ncomponents);
clock.stop();
std::cout << "Topological sort took: " << clock.secsSinceStart() << " seconds." << std::endl;
// Make vector's size match actual used data.
components_.resize(ncomponents + 1);
// Invoke appropriate solve method for each interdependent component.
for (int comp = 0; comp < ncomponents; ++comp) {
#if 0
#ifdef MATLAB_MEX_FILE
// \TODO replace this with general signal handling code, check if it costs performance.
if (interrupt_signal) {
mexPrintf("Reorder loop interrupted by user: %d of %d "
"cells finished.\n", i, grid.number_of_cells);
break;
}
#endif
#endif
const int comp_size = components_[comp + 1] - components_[comp];
if (comp_size == 1) {
solveSingleCell(sequence_[components_[comp]]);
} else {
solveMultiCell(comp_size, &sequence_[components_[comp]]);
}
}
}
void TofReorder::solveMultiCell(const int num_cells, const int* cells)
{
++num_multicell_;
max_size_multicell_ = std::max(max_size_multicell_, num_cells);
// std::cout << "Multiblock solve with " << num_cells << " cells." << std::endl;
// Using a Gauss-Seidel approach.
double max_delta = 1e100;
int num_iter = 0;
while (max_delta > gauss_seidel_tol_) {
max_delta = 0.0;
++num_iter;
for (int ci = 0; ci < num_cells; ++ci) {
const int cell = cells[ci];
const double tof_before = tof_[cell];
solveSingleCell(cell);
max_delta = std::max(max_delta, std::fabs(tof_[cell] - tof_before));
}
// std::cout << "Max delta = " << max_delta << std::endl;
}
max_iter_multicell_ = std::max(max_iter_multicell_, num_iter);
}
void TransportSolverCompressibleTwophaseReorder::solveMultiCell(const int num_cells, const int* cells)
{
// Experiment: when a cell changes more than the tolerance,
// mark all downwind cells as needing updates. After
// computing a single update in each cell, use marks
// to guide further updating. Clear mark in cell when
// its solution gets updated.
// Verdict: this is a good one! Approx. halved total time.
std::vector<int> needs_update(num_cells, 1);
// This one also needs the mapping from all cells to
// the strongly connected subset to filter out connections
std::vector<int> pos(grid_.number_of_cells, -1);
for (int i = 0; i < num_cells; ++i) {
const int cell = cells[i];
pos[cell] = i;
}
// Note: partially copied from below.
const double tol = 1e-9;
const int max_iters = 300;
// Must store s0 before we start.
std::vector<double> s0(num_cells);
// Must set initial fractional flows before we start.
// Also, we compute the # of upstream neighbours.
// std::vector<int> num_upstream(num_cells);
for (int i = 0; i < num_cells; ++i) {
const int cell = cells[i];
fractionalflow_[cell] = fracFlow(saturation_[cell], cell);
s0[i] = saturation_[cell];
// num_upstream[i] = ia_upw_[cell + 1] - ia_upw_[cell];
}
// Solve once in each cell.
// std::vector<int> fully_marked_stack;
// fully_marked_stack.reserve(num_cells);
int num_iters = 0;
int update_count = 0; // Change name/meaning to cells_updated?
do {
update_count = 0; // Must reset count for every iteration.
for (int i = 0; i < num_cells; ++i) {
// while (!fully_marked_stack.empty()) {
// // std::cout << "# fully marked cells = " << fully_marked_stack.size() << std::endl;
// const int fully_marked_ci = fully_marked_stack.back();
// fully_marked_stack.pop_back();
// ++update_count;
// const int cell = cells[fully_marked_ci];
// const double old_s = saturation_[cell];
// saturation_[cell] = s0[fully_marked_ci];
// solveSingleCell(cell);
// const double s_change = std::fabs(saturation_[cell] - old_s);
// if (s_change > tol) {
// // Mark downwind cells.
// for (int j = ia_downw_[cell]; j < ia_downw_[cell+1]; ++j) {
// const int downwind_cell = ja_downw_[j];
// int ci = pos[downwind_cell];
// ++needs_update[ci];
// if (needs_update[ci] == num_upstream[ci]) {
// fully_marked_stack.push_back(ci);
// }
// }
// }
// // Unmark this cell.
// needs_update[fully_marked_ci] = 0;
// }
if (!needs_update[i]) {
continue;
}
++update_count;
const int cell = cells[i];
const double old_s = saturation_[cell];
// solveSingleCell() requires saturation_[cell]
// to be s0.
saturation_[cell] = s0[i];
solveSingleCell(cell);
const double s_change = std::fabs(saturation_[cell] - old_s);
if (s_change > tol) {
// Mark downwind cells.
for (int j = ia_downw_[cell]; j < ia_downw_[cell+1]; ++j) {
const int downwind_cell = ja_downw_[j];
int ci = pos[downwind_cell];
if (ci != -1) {
needs_update[ci] = 1;
}
// ++needs_update[ci];
// if (needs_update[ci] == num_upstream[ci]) {
// fully_marked_stack.push_back(ci);
// }
}
}
// Unmark this cell.
needs_update[i] = 0;
}
// std::cout << "Iter = " << num_iters << " update_count = " << update_count
// << " # marked cells = "
// << std::accumulate(needs_update.begin(), needs_update.end(), 0) << std::endl;
} while (update_count > 0 && ++num_iters < max_iters);
// Done with iterations, check if we succeeded.
if (update_count > 0) {
OPM_THROW(std::runtime_error, "In solveMultiCell(), we did not converge after "
<< num_iters << " iterations. Remaining update count = " << update_count);
}
std::cout << "Solved " << num_cells << " cell multicell problem in "
<< num_iters << " iterations." << std::endl;
}
void TransportSolverTwophaseReorder::solveMultiCell(const int num_cells, const int* cells)
{
// std::ofstream os("dump");
// std::copy(cells, cells + num_cells, std::ostream_iterator<double>(os, "\n"));
// Experiment: try a breath-first search to build a more suitable ordering.
// Verdict: failed to improve #iterations.
// {
// std::vector<int> pos(grid_.number_of_cells, -1);
// for (int i = 0; i < num_cells; ++i) {
// const int cell = cells[i];
// pos[cell] = i;
// }
// std::vector<int> done_pos(num_cells, 0);
// std::vector<int> upstream_pos;
// std::vector<int> new_pos;
// upstream_pos.push_back(0);
// done_pos[0] = 1;
// int current = 0;
// while (int(new_pos.size()) < num_cells) {
// const int i = upstream_pos[current++];
// new_pos.push_back(i);
// const int cell = cells[i];
// for (int j = ia_[cell]; j < ia_[cell+1]; ++j) {
// const int opos = pos[ja_[j]];
// if (!done_pos[opos]) {
// upstream_pos.push_back(opos);
// done_pos[opos] = 1;
// }
// }
// }
// std::reverse(new_pos.begin(), new_pos.end());
// std::copy(new_pos.begin(), new_pos.end(), const_cast<int*>(cells));
// }
// Experiment: try a random ordering.
// Verdict: amazingly, reduced #iterations by approx. 25%!
// int* c = const_cast<int*>(cells);
// std::random_shuffle(c, c + num_cells);
// Experiment: compute topological tof from first cell.
// Verdict: maybe useful, not tried to exploit it yet.
// std::vector<int> tof;
// TofComputer comp(grid_.number_of_cells, &ia_[0], &ja_[0], cells[0], tof);
// std::ofstream tofdump("tofdump");
// std::copy(tof.begin(), tof.end(), std::ostream_iterator<double>(tofdump, "\n"));
// Experiment: implement a metric measuring badness of ordering
// as average distance in (cyclic) ordering from
// upstream neighbours.
// Verdict: does not seem to predict #iterations very well, if at all.
// std::vector<int> pos(grid_.number_of_cells, -1);
// for (int i = 0; i < num_cells; ++i) {
// const int cell = cells[i];
// pos[cell] = i;
// }
// double diffsum = 0;
// for (int i = 0; i < num_cells; ++i) {
// const int cell = cells[i];
// int num_upstream = 0;
// int loc_diffsum = 0;
// for (int j = ia_[cell]; j < ia_[cell+1]; ++j) {
// const int opos = pos[ja_[j]];
// if (opos == -1) {
// std::cout << "Hmmm" << std::endl;
// continue;
// }
// ++num_upstream;
// const int diff = (i - opos + num_cells) % num_cells;
// loc_diffsum += diff;
// }
// diffsum += double(loc_diffsum)/double(num_upstream);
// }
// std::cout << "Average distance from upstream neighbours: " << diffsum/double(num_cells)
// << std::endl;
#ifdef EXPERIMENT_GAUSS_SEIDEL
// Experiment: when a cell changes more than the tolerance,
// mark all downwind cells as needing updates. After
// computing a single update in each cell, use marks
// to guide further updating. Clear mark in cell when
// its solution gets updated.
// Verdict: this is a good one! Approx. halved total time.
std::vector<int> needs_update(num_cells, 1);
// This one also needs the mapping from all cells to
// the strongly connected subset to filter out connections
std::vector<int> pos(grid_.number_of_cells, -1);
for (int i = 0; i < num_cells; ++i) {
const int cell = cells[i];
pos[cell] = i;
}
// Note: partially copied from below.
const double tol = 1e-9;
const int max_iters = 300;
// Must store s0 before we start.
std::vector<double> s0(num_cells);
// Must set initial fractional flows before we start.
// Also, we compute the # of upstream neighbours.
// std::vector<int> num_upstream(num_cells);
for (int i = 0; i < num_cells; ++i) {
const int cell = cells[i];
fractionalflow_[cell] = fracFlow(saturation_[cell], cell);
s0[i] = saturation_[cell];
// num_upstream[i] = ia_upw_[cell + 1] - ia_upw_[cell];
}
// Solve once in each cell.
// std::vector<int> fully_marked_stack;
// fully_marked_stack.reserve(num_cells);
int num_iters = 0;
int update_count = 0; // Change name/meaning to cells_updated?
do {
update_count = 0; // Must reset count for every iteration.
for (int i = 0; i < num_cells; ++i) {
// while (!fully_marked_stack.empty()) {
// // std::cout << "# fully marked cells = " << fully_marked_stack.size() << std::endl;
// const int fully_marked_ci = fully_marked_stack.back();
// fully_marked_stack.pop_back();
// ++update_count;
// const int cell = cells[fully_marked_ci];
// const double old_s = saturation_[cell];
// saturation_[cell] = s0[fully_marked_ci];
// solveSingleCell(cell);
// const double s_change = std::fabs(saturation_[cell] - old_s);
// if (s_change > tol) {
// // Mark downwind cells.
// for (int j = ia_downw_[cell]; j < ia_downw_[cell+1]; ++j) {
// const int downwind_cell = ja_downw_[j];
// int ci = pos[downwind_cell];
// ++needs_update[ci];
// if (needs_update[ci] == num_upstream[ci]) {
// fully_marked_stack.push_back(ci);
// }
// }
// }
// // Unmark this cell.
// needs_update[fully_marked_ci] = 0;
// }
if (!needs_update[i]) {
continue;
}
++update_count;
const int cell = cells[i];
const double old_s = saturation_[cell];
saturation_[cell] = s0[i];
solveSingleCell(cell);
const double s_change = std::fabs(saturation_[cell] - old_s);
if (s_change > tol) {
// Mark downwind cells.
for (int j = ia_downw_[cell]; j < ia_downw_[cell+1]; ++j) {
const int downwind_cell = ja_downw_[j];
int ci = pos[downwind_cell];
if (ci != -1) {
needs_update[ci] = 1;
}
// ++needs_update[ci];
// if (needs_update[ci] == num_upstream[ci]) {
// fully_marked_stack.push_back(ci);
// }
}
}
// Unmark this cell.
needs_update[i] = 0;
}
// std::cout << "Iter = " << num_iters << " update_count = " << update_count
// << " # marked cells = "
// << std::accumulate(needs_update.begin(), needs_update.end(), 0) << std::endl;
} while (update_count > 0 && ++num_iters < max_iters);
// Done with iterations, check if we succeeded.
if (update_count > 0) {
OPM_THROW(std::runtime_error, "In solveMultiCell(), we did not converge after "
<< num_iters << " iterations. Remaining update count = " << update_count);
}
std::cout << "Solved " << num_cells << " cell multicell problem in "
<< num_iters << " iterations." << std::endl;
#else
double max_s_change = 0.0;
const double tol = 1e-9;
const int max_iters = 300;
int num_iters = 0;
// Must store s0 before we start.
std::vector<double> s0(num_cells);
// Must set initial fractional flows before we start.
for (int i = 0; i < num_cells; ++i) {
const int cell = cells[i];
fractionalflow_[cell] = fracFlow(saturation_[cell], cell);
s0[i] = saturation_[cell];
}
do {
max_s_change = 0.0;
for (int i = 0; i < num_cells; ++i) {
const int cell = cells[i];
const double old_s = saturation_[cell];
saturation_[cell] = s0[i];
solveSingleCell(cell);
double s_change = std::fabs(saturation_[cell] - old_s);
// std::cout << "cell = " << cell << " delta s = " << s_change << std::endl;
if (max_s_change < s_change) {
max_s_change = s_change;
}
}
// std::cout << "Iter = " << num_iters << " max_s_change = " << max_s_change
// << " in cell " << max_change_cell << std::endl;
} while (max_s_change > tol && ++num_iters < max_iters);
if (max_s_change > tol) {
OPM_THROW(std::runtime_error, "In solveMultiCell(), we did not converge after "
<< num_iters << " iterations. Delta s = " << max_s_change);
}
std::cout << "Solved " << num_cells << " cell multicell problem in "
<< num_iters << " iterations." << std::endl;
#endif // EXPERIMENT_GAUSS_SEIDEL
}
