/** Read velocity block data of all existing particle species. * @param file VLSV reader. * @param meshName Name of the spatial mesh. * @param fileCells Vector containing spatial cell IDs. * @param localCellStartOffset Offset into fileCells, determines where the cells belonging * to this process start. * @param localCells Number of spatial cells assigned to this process. * @param mpiGrid Parallel grid library. * @return If true, velocity block data was read successfully.*/ bool readBlockData( vlsv::ParallelReader& file, const string& meshName, const vector<CellID>& fileCells, const uint64_t localCellStartOffset, const uint64_t localCells, dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid ) { bool success = true; const uint64_t bytesReadStart = file.getBytesRead(); int N_processes; MPI_Comm_size(MPI_COMM_WORLD,&N_processes); uint64_t arraySize; uint64_t vectorSize; vlsv::datatype::type dataType; uint64_t byteSize; uint64_t* offsetArray = new uint64_t[N_processes]; for (uint popID=0; popID<getObjectWrapper().particleSpecies.size(); ++popID) { const string& popName = getObjectWrapper().particleSpecies[popID].name; // Create a cellID remapping lambda that can renumber our velocity space, should it's size have changed. // By default, this is a no-op that keeps the blockIDs untouched. std::function<vmesh::GlobalID(vmesh::GlobalID)> blockIDremapper = [](vmesh::GlobalID oldID) -> vmesh::GlobalID {return oldID;}; // Check that velocity space extents and DV matches the grids we have created list<pair<string,string> > attribs; attribs.push_back(make_pair("mesh",popName)); std::array<unsigned int, 6> fileMeshBBox; unsigned int* bufferpointer = &fileMeshBBox[0]; if (file.read("MESH_BBOX",attribs,0,6,bufferpointer,false) == false) { logFile << "(RESTART) ERROR: Failed to read MESH_BBOX at " << __FILE__ << ":" << __LINE__ << endl << write; success = false; } const size_t meshID = getObjectWrapper().particleSpecies[popID].velocityMesh; const vmesh::MeshParameters& ourMeshParams = getObjectWrapper().velocityMeshes[meshID]; if(fileMeshBBox[0] != ourMeshParams.gridLength[0] || fileMeshBBox[1] != ourMeshParams.gridLength[1] || fileMeshBBox[2] != ourMeshParams.gridLength[2]) { logFile << "(RESTART) INFO: velocity mesh sizes don't match:" << endl << " restart file has " << fileMeshBBox[0] << " x " << fileMeshBBox[1] << " x " << fileMeshBBox[2] << "," << endl << " config specifies " << ourMeshParams.gridLength[0] << " x " << ourMeshParams.gridLength[1] << " x " << ourMeshParams.gridLength[2] << endl << write; if(ourMeshParams.gridLength[0] < fileMeshBBox[0] || ourMeshParams.gridLength[1] < fileMeshBBox[1] || ourMeshParams.gridLength[2] < fileMeshBBox[2]) { logFile << "(RESTART) ERROR: trying to shrink velocity space." << endl << write; abort(); } // If we are mismatched, we have to iterate through the velocity coords to see if we have a // chance at renumbering. std::vector<Real> fileVelCoordsX(fileMeshBBox[0]*fileMeshBBox[3]+1); std::vector<Real> fileVelCoordsY(fileMeshBBox[1]*fileMeshBBox[4]+1); std::vector<Real> fileVelCoordsZ(fileMeshBBox[2]*fileMeshBBox[5]+1); Real* tempPointer = fileVelCoordsX.data(); if (file.read("MESH_NODE_CRDS_X",attribs,0,fileMeshBBox[0]*fileMeshBBox[3]+1,tempPointer,false) == false) { logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write; success = false; } tempPointer = fileVelCoordsY.data(); if (file.read("MESH_NODE_CRDS_Y",attribs,0,fileMeshBBox[1]*fileMeshBBox[4]+1,tempPointer,false) == false) { logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write; success = false; } tempPointer = fileVelCoordsZ.data(); if (file.read("MESH_NODE_CRDS_Z",attribs,0,fileMeshBBox[2]*fileMeshBBox[5]+1,tempPointer,false) == false) { logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write; success = false; } const Real dVx = getObjectWrapper().velocityMeshes[meshID].cellSize[0]; for(const auto& c : fileVelCoordsX) { Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0]) / dVx; if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) { logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl << " (X coordinate " << c << " = " << cellindex <<" * " << dVx << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0] << endl << " coordinate = cellindex * dV + meshMinLimits)" << endl << write; abort(); } } const Real dVy = getObjectWrapper().velocityMeshes[meshID].cellSize[1]; for(const auto& c : fileVelCoordsY) { Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1]) / dVy; if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) { logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl << " (Y coordinate " << c << " = " << cellindex <<" * " << dVy << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1] << endl << " coordinate = cellindex * dV + meshMinLimits)" << endl << write; abort(); } } const Real dVz = getObjectWrapper().velocityMeshes[meshID].cellSize[2]; for(const auto& c : fileVelCoordsY) { Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2]) / dVz; if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) { logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl << " (Z coordinate " << c << " = " << cellindex <<" * " << dVz << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2] << endl << " coordinate = cellindex * dV + meshMinLimits)" << endl << write; abort(); } } // If we haven't aborted above, we can apparently renumber our // cellIDs. Build an approprita blockIDremapper lambda for this purpose. std::array<int, 3> velGridOffset; velGridOffset[0] = (fileVelCoordsX[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0]) / dVx; velGridOffset[1] = (fileVelCoordsY[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1]) / dVy; velGridOffset[2] = (fileVelCoordsZ[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2]) / dVz; if((velGridOffset[0] % ourMeshParams.blockLength[0] != 0) || (velGridOffset[1] % ourMeshParams.blockLength[1] != 0) || (velGridOffset[2] % ourMeshParams.blockLength[2] != 0)) { logFile << "(RESTART) ERROR: resizing velocity space on restart must end up with the old velocity space" << endl << " at a block boundary of the new space!" << endl << " (It now starts at cell [" << velGridOffset[0] << ", " << velGridOffset[1] << "," << velGridOffset[2] << "])" << endl << write; abort(); } velGridOffset[0] /= ourMeshParams.blockLength[0]; velGridOffset[1] /= ourMeshParams.blockLength[1]; velGridOffset[2] /= ourMeshParams.blockLength[2]; blockIDremapper = [fileMeshBBox,velGridOffset,ourMeshParams](vmesh::GlobalID oldID) -> vmesh::GlobalID { unsigned int x,y,z; x = oldID % fileMeshBBox[0]; y = (oldID / fileMeshBBox[0]) % fileMeshBBox[1]; z = oldID / (fileMeshBBox[0] * fileMeshBBox[1]); x += velGridOffset[0]; y += velGridOffset[1]; z += velGridOffset[2]; //logFile << " Remapping " << oldID << "(" << x << "," << y << "," << z << ") to " << x + y * ourMeshParams.gridLength[0] + z* ourMeshParams.gridLength[0] * ourMeshParams.gridLength[1] << endl << write; return x + y * ourMeshParams.gridLength[0] + z* ourMeshParams.gridLength[0] * ourMeshParams.gridLength[1]; }; logFile << " => Resizing velocity space by renumbering GlobalIDs." << endl << endl << write; } // In restart files each spatial cell has an entry in CELLSWITHBLOCKS. // Each process calculates how many velocity blocks it has for this species. attribs.clear(); attribs.push_back(make_pair("mesh",meshName)); attribs.push_back(make_pair("name",popName)); vmesh::LocalID* blocksPerCell = NULL; if (file.read("BLOCKSPERCELL",attribs,localCellStartOffset,localCells,blocksPerCell,true) == false) { logFile << "(RESTART) ERROR: Failed to read BLOCKSPERCELL at " << __FILE__ << ":" << __LINE__ << endl << write; success = false; } // Count how many velocity blocks this process gets uint64_t blockSum = 0; for (uint64_t i=0; i<localCells; ++i){ blockSum += blocksPerCell[i]; } // Gather all block sums to master process who will them broadcast // the values to everyone MPI_Allgather(&blockSum,1,MPI_Type<uint64_t>(),offsetArray,1,MPI_Type<uint64_t>(),MPI_COMM_WORLD); // Calculate the offset from which this process starts reading block data uint64_t myOffset = 0; for (int64_t i=0; i<mpiGrid.get_rank(); ++i) myOffset += offsetArray[i]; if (file.getArrayInfo("BLOCKVARIABLE",attribs,arraySize,vectorSize,dataType,byteSize) == false) { logFile << "(RESTART) ERROR: Failed to read BLOCKVARIABLE INFO" << endl << write; return false; } // Call _readBlockData if (dataType == vlsv::datatype::type::FLOAT) { switch (byteSize) { case sizeof(double): if (_readBlockData<double>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell, myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false; break; case sizeof(float): if (_readBlockData<float>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell, myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false; break; } } else if (dataType == vlsv::datatype::type::UINT) { switch (byteSize) { case sizeof(uint32_t): if (_readBlockData<uint32_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell, myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false; break; case sizeof(uint64_t): if (_readBlockData<uint64_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell, myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false; break; } } else if (dataType == vlsv::datatype::type::INT) { switch (byteSize) { case sizeof(int32_t): if (_readBlockData<int32_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell, myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false; break; case sizeof(int64_t): if (_readBlockData<int64_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell, myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false; break; } } else { logFile << "(RESTART) ERROR: Failed to read data type at readCellParamsVariable" << endl << write; success = false; } delete [] blocksPerCell; blocksPerCell = NULL; } // for-loop over particle species delete [] offsetArray; offsetArray = NULL; const uint64_t bytesReadEnd = file.getBytesRead() - bytesReadStart; logFile << "Velocity meshes and data read, approximate data rate is "; logFile << vlsv::printDataRate(bytesReadEnd,file.getReadTime()) << endl << write; return success; }
> void initialize( const Geometries& geometries, Init_Cond& initial_conditions, const Background_Magnetic_Field& bg_B, dccrg::Dccrg<Cell, Geometry>& grid, const std::vector<uint64_t>& cells, const double time, const double adiabatic_index, const double vacuum_permeability, const double proton_mass, const bool verbose, const Mass_Density_Getter Mas, const Momentum_Density_Getter Mom, const Total_Energy_Density_Getter Nrj, const Magnetic_Field_Getter Mag, const Background_Magnetic_Field_Pos_X_Getter Bg_B_Pos_X, const Background_Magnetic_Field_Pos_Y_Getter Bg_B_Pos_Y, const Background_Magnetic_Field_Pos_Z_Getter Bg_B_Pos_Z, const Mass_Density_Flux_Getter Mas_f, const Momentum_Density_Flux_Getter Mom_f, const Total_Energy_Density_Flux_Getter Nrj_f, const Magnetic_Field_Flux_Getter Mag_f ) { if (verbose and grid.get_rank() == 0) { std::cout << "Setting default MHD state... "; std::cout.flush(); } // set default state for (const auto cell_id: cells) { auto* const cell_data = grid[cell_id]; if (cell_data == nullptr) { std::cerr << __FILE__ << "(" << __LINE__ << ") No data for cell: " << cell_id << std::endl; abort(); } // zero fluxes and background fields Mas_f(*cell_data) = Nrj_f(*cell_data) = Mom_f(*cell_data)[0] = Mom_f(*cell_data)[1] = Mom_f(*cell_data)[2] = Mag_f(*cell_data)[0] = Mag_f(*cell_data)[1] = Mag_f(*cell_data)[2] = 0; const auto c = grid.geometry.get_center(cell_id); const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]); const auto lat = asin(c[2] / r), lon = atan2(c[1], c[0]); const auto mass_density = proton_mass * initial_conditions.get_default_data( Number_Density(), time, c[0], c[1], c[2], r, lat, lon ); const auto velocity = initial_conditions.get_default_data( Velocity(), time, c[0], c[1], c[2], r, lat, lon ); const auto pressure = initial_conditions.get_default_data( Pressure(), time, c[0], c[1], c[2], r, lat, lon ); const auto magnetic_field = initial_conditions.get_default_data( Magnetic_Field(), time, c[0], c[1], c[2], r, lat, lon ); Mas(*cell_data) = mass_density; Mom(*cell_data) = mass_density * velocity; Mag(*cell_data) = magnetic_field; Nrj(*cell_data) = get_total_energy_density( mass_density, velocity, pressure, magnetic_field, adiabatic_index, vacuum_permeability ); const auto cell_end = grid.geometry.get_max(cell_id); Bg_B_Pos_X(*cell_data) = bg_B.get_background_field( {cell_end[0], c[1], c[2]}, vacuum_permeability ); Bg_B_Pos_Y(*cell_data) = bg_B.get_background_field( {c[0], cell_end[1], c[2]}, vacuum_permeability ); Bg_B_Pos_Z(*cell_data) = bg_B.get_background_field( {c[0], c[1], cell_end[2]}, vacuum_permeability ); } // set non-default initial conditions if (verbose and grid.get_rank() == 0) { std::cout << "done\nSetting non-default initial MHD state... "; std::cout.flush(); } /* Set non-default initial conditions */ // mass density for ( size_t i = 0; i < initial_conditions.get_number_of_regions(Number_Density()); i++ ) { const auto& init_cond = initial_conditions.get_initial_condition(Number_Density(), i); const auto& geometry_id = init_cond.get_geometry_id(); const auto& cells = geometries.get_cells(geometry_id); for (const auto& cell: cells) { const auto c = grid.geometry.get_center(cell); const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]); const auto lat = asin(c[2] / r), lon = atan2(c[1], c[0]); const auto mass_density = proton_mass * initial_conditions.get_data( Number_Density(), geometry_id, time, c[0], c[1], c[2], r, lat, lon ); auto* const cell_data = grid[cell]; if (cell_data == NULL) { std::cerr << __FILE__ << "(" << __LINE__ << std::endl; abort(); } Mas(*cell_data) = mass_density; } } // velocity for ( size_t i = 0; i < initial_conditions.get_number_of_regions(Velocity()); i++ ) { const auto& init_cond = initial_conditions.get_initial_condition(Velocity(), i); const auto& geometry_id = init_cond.get_geometry_id(); const auto& cells = geometries.get_cells(geometry_id); for (const auto& cell: cells) { const auto c = grid.geometry.get_center(cell); const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]); const auto lat = asin(c[2] / r), lon = atan2(c[1], c[0]); const auto velocity = initial_conditions.get_data( Velocity(), geometry_id, time, c[0], c[1], c[2], r, lat, lon ); auto* const cell_data = grid[cell]; if (cell_data == NULL) { std::cerr << __FILE__ << "(" << __LINE__ << ") No data for cell: " << cell << std::endl; abort(); } Mom(*cell_data) = Mas(*cell_data) * velocity; } } // magnetic field for ( size_t i = 0; i < initial_conditions.get_number_of_regions(Magnetic_Field()); i++ ) { const auto& init_cond = initial_conditions.get_initial_condition(Magnetic_Field(), i); const auto& geometry_id = init_cond.get_geometry_id(); const auto& cells = geometries.get_cells(geometry_id); for (const auto& cell: cells) { const auto c = grid.geometry.get_center(cell); const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]); const auto lat = asin(c[2] / r), lon = atan2(c[1], c[0]); const auto magnetic_field = initial_conditions.get_data( Magnetic_Field(), geometry_id, time, c[0], c[1], c[2], r, lat, lon ); auto* const cell_data = grid[cell]; if (cell_data == NULL) { std::cerr << __FILE__ << "(" << __LINE__ << ") No data for cell: " << cell << std::endl; abort(); } Mag(*cell_data) = magnetic_field; } } // pressure for ( size_t i = 0; i < initial_conditions.get_number_of_regions(Pressure()); i++ ) { std::cout << std::endl; const auto& init_cond = initial_conditions.get_initial_condition(Pressure(), i); const auto& geometry_id = init_cond.get_geometry_id(); std::cout << geometry_id << std::endl; const auto& cells = geometries.get_cells(geometry_id); std::cout << cells.size() << std::endl; for (const auto& cell: cells) { const auto c = grid.geometry.get_center(cell); const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]); const auto lat = asin(c[2] / r), lon = atan2(c[1], c[0]); const auto pressure = initial_conditions.get_data( Pressure(), geometry_id, time, c[0], c[1], c[2], r, lat, lon ); auto* const cell_data = grid[cell]; if (cell_data == NULL) { std::cerr << __FILE__ << "(" << __LINE__ << ") No data for cell: " << cell << std::endl; abort(); } Nrj(*cell_data) = get_total_energy_density( Mas(*cell_data), Mom(*cell_data) / Mas(*cell_data), pressure, Mag(*cell_data), adiabatic_index, vacuum_permeability ); } } if (verbose and grid.get_rank() == 0) { std::cout << "done" << std::endl; } }