/** Read velocity block data of all existing particle species.
* @param file VLSV reader.
* @param meshName Name of the spatial mesh.
* @param fileCells Vector containing spatial cell IDs.
* @param localCellStartOffset Offset into fileCells, determines where the cells belonging
* to this process start.
* @param localCells Number of spatial cells assigned to this process.
* @param mpiGrid Parallel grid library.
* @return If true, velocity block data was read successfully.*/
bool readBlockData(
vlsv::ParallelReader& file,
const string& meshName,
const vector<CellID>& fileCells,
const uint64_t localCellStartOffset,
const uint64_t localCells,
dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid
) {
bool success = true;
const uint64_t bytesReadStart = file.getBytesRead();
int N_processes;
MPI_Comm_size(MPI_COMM_WORLD,&N_processes);
uint64_t arraySize;
uint64_t vectorSize;
vlsv::datatype::type dataType;
uint64_t byteSize;
uint64_t* offsetArray = new uint64_t[N_processes];
for (uint popID=0; popID<getObjectWrapper().particleSpecies.size(); ++popID) {
const string& popName = getObjectWrapper().particleSpecies[popID].name;
// Create a cellID remapping lambda that can renumber our velocity space, should it's size have changed.
// By default, this is a no-op that keeps the blockIDs untouched.
std::function<vmesh::GlobalID(vmesh::GlobalID)> blockIDremapper = [](vmesh::GlobalID oldID) -> vmesh::GlobalID {return oldID;};
// Check that velocity space extents and DV matches the grids we have created
list<pair<string,string> > attribs;
attribs.push_back(make_pair("mesh",popName));
std::array<unsigned int, 6> fileMeshBBox;
unsigned int* bufferpointer = &fileMeshBBox[0];
if (file.read("MESH_BBOX",attribs,0,6,bufferpointer,false) == false) {
logFile << "(RESTART) ERROR: Failed to read MESH_BBOX at " << __FILE__ << ":" << __LINE__ << endl << write;
success = false;
}
const size_t meshID = getObjectWrapper().particleSpecies[popID].velocityMesh;
const vmesh::MeshParameters& ourMeshParams = getObjectWrapper().velocityMeshes[meshID];
if(fileMeshBBox[0] != ourMeshParams.gridLength[0] ||
fileMeshBBox[1] != ourMeshParams.gridLength[1] ||
fileMeshBBox[2] != ourMeshParams.gridLength[2]) {
logFile << "(RESTART) INFO: velocity mesh sizes don't match:" << endl
<< " restart file has " << fileMeshBBox[0] << " x " << fileMeshBBox[1] << " x " << fileMeshBBox[2] << "," << endl
<< " config specifies " << ourMeshParams.gridLength[0] << " x " << ourMeshParams.gridLength[1] << " x " << ourMeshParams.gridLength[2] << endl << write;
if(ourMeshParams.gridLength[0] < fileMeshBBox[0] ||
ourMeshParams.gridLength[1] < fileMeshBBox[1] ||
ourMeshParams.gridLength[2] < fileMeshBBox[2]) {
logFile << "(RESTART) ERROR: trying to shrink velocity space." << endl << write;
abort();
}
// If we are mismatched, we have to iterate through the velocity coords to see if we have a
// chance at renumbering.
std::vector<Real> fileVelCoordsX(fileMeshBBox[0]*fileMeshBBox[3]+1);
std::vector<Real> fileVelCoordsY(fileMeshBBox[1]*fileMeshBBox[4]+1);
std::vector<Real> fileVelCoordsZ(fileMeshBBox[2]*fileMeshBBox[5]+1);
Real* tempPointer = fileVelCoordsX.data();
if (file.read("MESH_NODE_CRDS_X",attribs,0,fileMeshBBox[0]*fileMeshBBox[3]+1,tempPointer,false) == false) {
logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write;
success = false;
}
tempPointer = fileVelCoordsY.data();
if (file.read("MESH_NODE_CRDS_Y",attribs,0,fileMeshBBox[1]*fileMeshBBox[4]+1,tempPointer,false) == false) {
logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write;
success = false;
}
tempPointer = fileVelCoordsZ.data();
if (file.read("MESH_NODE_CRDS_Z",attribs,0,fileMeshBBox[2]*fileMeshBBox[5]+1,tempPointer,false) == false) {
logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write;
success = false;
}
const Real dVx = getObjectWrapper().velocityMeshes[meshID].cellSize[0];
for(const auto& c : fileVelCoordsX) {
Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0]) / dVx;
if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) {
logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl
<< " (X coordinate " << c << " = " << cellindex <<" * " << dVx << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0] << endl
<< " coordinate = cellindex * dV + meshMinLimits)" << endl << write;
abort();
}
}
const Real dVy = getObjectWrapper().velocityMeshes[meshID].cellSize[1];
for(const auto& c : fileVelCoordsY) {
Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1]) / dVy;
if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) {
logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl
<< " (Y coordinate " << c << " = " << cellindex <<" * " << dVy << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1] << endl
<< " coordinate = cellindex * dV + meshMinLimits)" << endl << write;
abort();
}
}
const Real dVz = getObjectWrapper().velocityMeshes[meshID].cellSize[2];
for(const auto& c : fileVelCoordsY) {
Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2]) / dVz;
if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) {
logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl
<< " (Z coordinate " << c << " = " << cellindex <<" * " << dVz << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2] << endl
<< " coordinate = cellindex * dV + meshMinLimits)" << endl << write;
abort();
}
}
// If we haven't aborted above, we can apparently renumber our
// cellIDs. Build an approprita blockIDremapper lambda for this purpose.
std::array<int, 3> velGridOffset;
velGridOffset[0] = (fileVelCoordsX[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0]) / dVx;
velGridOffset[1] = (fileVelCoordsY[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1]) / dVy;
velGridOffset[2] = (fileVelCoordsZ[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2]) / dVz;
if((velGridOffset[0] % ourMeshParams.blockLength[0] != 0) ||
(velGridOffset[1] % ourMeshParams.blockLength[1] != 0) ||
(velGridOffset[2] % ourMeshParams.blockLength[2] != 0)) {
logFile << "(RESTART) ERROR: resizing velocity space on restart must end up with the old velocity space" << endl
<< " at a block boundary of the new space!" << endl
<< " (It now starts at cell [" << velGridOffset[0] << ", " << velGridOffset[1] << "," << velGridOffset[2] << "])" << endl << write;
abort();
}
velGridOffset[0] /= ourMeshParams.blockLength[0];
velGridOffset[1] /= ourMeshParams.blockLength[1];
velGridOffset[2] /= ourMeshParams.blockLength[2];
blockIDremapper = [fileMeshBBox,velGridOffset,ourMeshParams](vmesh::GlobalID oldID) -> vmesh::GlobalID {
unsigned int x,y,z;
x = oldID % fileMeshBBox[0];
y = (oldID / fileMeshBBox[0]) % fileMeshBBox[1];
z = oldID / (fileMeshBBox[0] * fileMeshBBox[1]);
x += velGridOffset[0];
y += velGridOffset[1];
z += velGridOffset[2];
//logFile << " Remapping " << oldID << "(" << x << "," << y << "," << z << ") to " << x + y * ourMeshParams.gridLength[0] + z* ourMeshParams.gridLength[0] * ourMeshParams.gridLength[1] << endl << write;
return x + y * ourMeshParams.gridLength[0] + z* ourMeshParams.gridLength[0] * ourMeshParams.gridLength[1];
};
logFile << " => Resizing velocity space by renumbering GlobalIDs." << endl << endl << write;
}
// In restart files each spatial cell has an entry in CELLSWITHBLOCKS.
// Each process calculates how many velocity blocks it has for this species.
attribs.clear();
attribs.push_back(make_pair("mesh",meshName));
attribs.push_back(make_pair("name",popName));
vmesh::LocalID* blocksPerCell = NULL;
if (file.read("BLOCKSPERCELL",attribs,localCellStartOffset,localCells,blocksPerCell,true) == false) {
logFile << "(RESTART) ERROR: Failed to read BLOCKSPERCELL at " << __FILE__ << ":" << __LINE__ << endl << write;
success = false;
}
// Count how many velocity blocks this process gets
uint64_t blockSum = 0;
for (uint64_t i=0; i<localCells; ++i){
blockSum += blocksPerCell[i];
}
// Gather all block sums to master process who will them broadcast
// the values to everyone
MPI_Allgather(&blockSum,1,MPI_Type<uint64_t>(),offsetArray,1,MPI_Type<uint64_t>(),MPI_COMM_WORLD);
// Calculate the offset from which this process starts reading block data
uint64_t myOffset = 0;
for (int64_t i=0; i<mpiGrid.get_rank(); ++i) myOffset += offsetArray[i];
if (file.getArrayInfo("BLOCKVARIABLE",attribs,arraySize,vectorSize,dataType,byteSize) == false) {
logFile << "(RESTART) ERROR: Failed to read BLOCKVARIABLE INFO" << endl << write;
return false;
}
// Call _readBlockData
if (dataType == vlsv::datatype::type::FLOAT) {
switch (byteSize) {
case sizeof(double):
if (_readBlockData<double>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
break;
case sizeof(float):
if (_readBlockData<float>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
break;
}
} else if (dataType == vlsv::datatype::type::UINT) {
switch (byteSize) {
case sizeof(uint32_t):
if (_readBlockData<uint32_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
break;
case sizeof(uint64_t):
if (_readBlockData<uint64_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
break;
}
} else if (dataType == vlsv::datatype::type::INT) {
switch (byteSize) {
case sizeof(int32_t):
if (_readBlockData<int32_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
break;
case sizeof(int64_t):
if (_readBlockData<int64_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
break;
}
} else {
logFile << "(RESTART) ERROR: Failed to read data type at readCellParamsVariable" << endl << write;
success = false;
}
delete [] blocksPerCell; blocksPerCell = NULL;
} // for-loop over particle species
delete [] offsetArray; offsetArray = NULL;
const uint64_t bytesReadEnd = file.getBytesRead() - bytesReadStart;
logFile << "Velocity meshes and data read, approximate data rate is ";
logFile << vlsv::printDataRate(bytesReadEnd,file.getReadTime()) << endl << write;
return success;
}
> void initialize(
const Geometries& geometries,
Init_Cond& initial_conditions,
const Background_Magnetic_Field& bg_B,
dccrg::Dccrg<Cell, Geometry>& grid,
const std::vector<uint64_t>& cells,
const double time,
const double adiabatic_index,
const double vacuum_permeability,
const double proton_mass,
const bool verbose,
const Mass_Density_Getter Mas,
const Momentum_Density_Getter Mom,
const Total_Energy_Density_Getter Nrj,
const Magnetic_Field_Getter Mag,
const Background_Magnetic_Field_Pos_X_Getter Bg_B_Pos_X,
const Background_Magnetic_Field_Pos_Y_Getter Bg_B_Pos_Y,
const Background_Magnetic_Field_Pos_Z_Getter Bg_B_Pos_Z,
const Mass_Density_Flux_Getter Mas_f,
const Momentum_Density_Flux_Getter Mom_f,
const Total_Energy_Density_Flux_Getter Nrj_f,
const Magnetic_Field_Flux_Getter Mag_f
) {
if (verbose and grid.get_rank() == 0) {
std::cout << "Setting default MHD state... ";
std::cout.flush();
}
// set default state
for (const auto cell_id: cells) {
auto* const cell_data = grid[cell_id];
if (cell_data == nullptr) {
std::cerr << __FILE__ << "(" << __LINE__ << ") No data for cell: "
<< cell_id
<< std::endl;
abort();
}
// zero fluxes and background fields
Mas_f(*cell_data) =
Nrj_f(*cell_data) =
Mom_f(*cell_data)[0] =
Mom_f(*cell_data)[1] =
Mom_f(*cell_data)[2] =
Mag_f(*cell_data)[0] =
Mag_f(*cell_data)[1] =
Mag_f(*cell_data)[2] = 0;
const auto c = grid.geometry.get_center(cell_id);
const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
const auto
lat = asin(c[2] / r),
lon = atan2(c[1], c[0]);
const auto mass_density
= proton_mass
* initial_conditions.get_default_data(
Number_Density(),
time,
c[0], c[1], c[2],
r, lat, lon
);
const auto velocity
= initial_conditions.get_default_data(
Velocity(),
time,
c[0], c[1], c[2],
r, lat, lon
);
const auto pressure
= initial_conditions.get_default_data(
Pressure(),
time,
c[0], c[1], c[2],
r, lat, lon
);
const auto magnetic_field
= initial_conditions.get_default_data(
Magnetic_Field(),
time,
c[0], c[1], c[2],
r, lat, lon
);
Mas(*cell_data) = mass_density;
Mom(*cell_data) = mass_density * velocity;
Mag(*cell_data) = magnetic_field;
Nrj(*cell_data) = get_total_energy_density(
mass_density,
velocity,
pressure,
magnetic_field,
adiabatic_index,
vacuum_permeability
);
const auto cell_end = grid.geometry.get_max(cell_id);
Bg_B_Pos_X(*cell_data) = bg_B.get_background_field(
{cell_end[0], c[1], c[2]},
vacuum_permeability
);
Bg_B_Pos_Y(*cell_data) = bg_B.get_background_field(
{c[0], cell_end[1], c[2]},
vacuum_permeability
);
Bg_B_Pos_Z(*cell_data) = bg_B.get_background_field(
{c[0], c[1], cell_end[2]},
vacuum_permeability
);
}
// set non-default initial conditions
if (verbose and grid.get_rank() == 0) {
std::cout << "done\nSetting non-default initial MHD state... ";
std::cout.flush();
}
/*
Set non-default initial conditions
*/
// mass density
for (
size_t i = 0;
i < initial_conditions.get_number_of_regions(Number_Density());
i++
) {
const auto& init_cond = initial_conditions.get_initial_condition(Number_Density(), i);
const auto& geometry_id = init_cond.get_geometry_id();
const auto& cells = geometries.get_cells(geometry_id);
for (const auto& cell: cells) {
const auto c = grid.geometry.get_center(cell);
const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
const auto
lat = asin(c[2] / r),
lon = atan2(c[1], c[0]);
const auto mass_density
= proton_mass
* initial_conditions.get_data(
Number_Density(),
geometry_id,
time,
c[0], c[1], c[2],
r, lat, lon
);
auto* const cell_data = grid[cell];
if (cell_data == NULL) {
std::cerr << __FILE__ << "(" << __LINE__ << std::endl;
abort();
}
Mas(*cell_data) = mass_density;
}
}
// velocity
for (
size_t i = 0;
i < initial_conditions.get_number_of_regions(Velocity());
i++
) {
const auto& init_cond = initial_conditions.get_initial_condition(Velocity(), i);
const auto& geometry_id = init_cond.get_geometry_id();
const auto& cells = geometries.get_cells(geometry_id);
for (const auto& cell: cells) {
const auto c = grid.geometry.get_center(cell);
const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
const auto
lat = asin(c[2] / r),
lon = atan2(c[1], c[0]);
const auto velocity = initial_conditions.get_data(
Velocity(),
geometry_id,
time,
c[0], c[1], c[2],
r, lat, lon
);
auto* const cell_data = grid[cell];
if (cell_data == NULL) {
std::cerr << __FILE__ << "(" << __LINE__
<< ") No data for cell: " << cell
<< std::endl;
abort();
}
Mom(*cell_data) = Mas(*cell_data) * velocity;
}
}
// magnetic field
for (
size_t i = 0;
i < initial_conditions.get_number_of_regions(Magnetic_Field());
i++
) {
const auto& init_cond = initial_conditions.get_initial_condition(Magnetic_Field(), i);
const auto& geometry_id = init_cond.get_geometry_id();
const auto& cells = geometries.get_cells(geometry_id);
for (const auto& cell: cells) {
const auto c = grid.geometry.get_center(cell);
const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
const auto
lat = asin(c[2] / r),
lon = atan2(c[1], c[0]);
const auto magnetic_field = initial_conditions.get_data(
Magnetic_Field(),
geometry_id,
time,
c[0], c[1], c[2],
r, lat, lon
);
auto* const cell_data = grid[cell];
if (cell_data == NULL) {
std::cerr << __FILE__ << "(" << __LINE__
<< ") No data for cell: " << cell
<< std::endl;
abort();
}
Mag(*cell_data) = magnetic_field;
}
}
// pressure
for (
size_t i = 0;
i < initial_conditions.get_number_of_regions(Pressure());
i++
) {
std::cout << std::endl;
const auto& init_cond = initial_conditions.get_initial_condition(Pressure(), i);
const auto& geometry_id = init_cond.get_geometry_id();
std::cout << geometry_id << std::endl;
const auto& cells = geometries.get_cells(geometry_id);
std::cout << cells.size() << std::endl;
for (const auto& cell: cells) {
const auto c = grid.geometry.get_center(cell);
const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
const auto
lat = asin(c[2] / r),
lon = atan2(c[1], c[0]);
const auto pressure = initial_conditions.get_data(
Pressure(),
geometry_id,
time,
c[0], c[1], c[2],
r, lat, lon
);
auto* const cell_data = grid[cell];
if (cell_data == NULL) {
std::cerr << __FILE__ << "(" << __LINE__
<< ") No data for cell: " << cell
<< std::endl;
abort();
}
Nrj(*cell_data) = get_total_energy_density(
Mas(*cell_data),
Mom(*cell_data) / Mas(*cell_data),
pressure,
Mag(*cell_data),
adiabatic_index,
vacuum_permeability
);
}
}
if (verbose and grid.get_rank() == 0) {
std::cout << "done" << std::endl;
}
}