int numFaces(const Dune::CpGrid& grid)
{
return grid.numFaces();
}
inline
void distributeGridAndData( Dune::CpGrid& grid,
Opm::DeckConstPtr deck,
EclipseStateConstPtr eclipseState,
BlackoilState& state,
BlackoilPropsAdFromDeck& properties,
DerivedGeology& geology,
std::shared_ptr<BlackoilPropsAdFromDeck::MaterialLawManager>& material_law_manager,
std::vector<double>& threshold_pressures,
boost::any& parallelInformation,
const bool useLocalPerm)
{
Dune::CpGrid global_grid ( grid );
global_grid.switchToGlobalView();
// distribute the grid and switch to the distributed view
grid.loadBalance(eclipseState, geology.transmissibility().data());
grid.switchToDistributedView();
std::vector<int> compressedToCartesianIdx;
Opm::createGlobalCellArray(grid, compressedToCartesianIdx);
typedef BlackoilPropsAdFromDeck::MaterialLawManager MaterialLawManager;
auto distributed_material_law_manager = std::make_shared<MaterialLawManager>();
distributed_material_law_manager->initFromDeck(deck, eclipseState, compressedToCartesianIdx);
// copy the values from the global to the local MaterialLawManager
// We should actually communicate these to be future proof. But that is
// really, really cumbersome for the underlying vector<shared_ptr>
// where the classes pointed to even have more shared_ptr stored in them.
typedef Dune::CpGrid::ParallelIndexSet IndexSet;
const IndexSet& local_indices = grid.getCellIndexSet();
for ( auto index : local_indices )
{
distributed_material_law_manager->materialLawParamsPointerReferenceHack(index.local()) =
material_law_manager->materialLawParamsPointerReferenceHack(index.global());
distributed_material_law_manager->oilWaterScaledEpsInfoDrainagePointerReferenceHack(index.local()) =
material_law_manager->oilWaterScaledEpsInfoDrainagePointerReferenceHack(index.global());
}
BlackoilPropsAdFromDeck distributed_props(properties,
distributed_material_law_manager,
grid.numCells());
BlackoilState distributed_state(grid.numCells(), grid.numFaces(), state.numPhases());
BlackoilStateDataHandle state_handle(global_grid, grid,
state, distributed_state);
BlackoilPropsDataHandle props_handle(properties,
distributed_props);
grid.scatterData(state_handle);
grid.scatterData(props_handle);
// Create a distributed Geology. Some values will be updated using communication
// below
DerivedGeology distributed_geology(grid,
distributed_props, eclipseState,
useLocalPerm, geology.gravity());
GeologyDataHandle geo_handle(global_grid, grid,
geology, distributed_geology);
grid.scatterData(geo_handle);
std::vector<double> distributed_pressures;
if( !threshold_pressures.empty() ) // Might be empty if not specified
{
if( threshold_pressures.size() !=
static_cast<std::size_t>(UgGridHelpers::numFaces(global_grid)) )
{
OPM_THROW(std::runtime_error, "NNCs not yet supported for parallel runs. "
<< UgGridHelpers::numFaces(grid) << " faces but " <<
threshold_pressures.size()<<" threshold pressure values");
}
distributed_pressures.resize(UgGridHelpers::numFaces(grid));
ThresholdPressureDataHandle press_handle(global_grid, grid,
threshold_pressures,
distributed_pressures);
grid.scatterData(press_handle);
}
// copy states
properties = distributed_props;
geology = distributed_geology;
state = distributed_state;
material_law_manager = distributed_material_law_manager;
threshold_pressures = distributed_pressures;
extractParallelGridInformationToISTL(grid, parallelInformation);
}