Beispiel #1
0
/** Read velocity block data of all existing particle species.
 * @param file VLSV reader.
 * @param meshName Name of the spatial mesh.
 * @param fileCells Vector containing spatial cell IDs.
 * @param localCellStartOffset Offset into fileCells, determines where the cells belonging 
 * to this process start.
 * @param localCells Number of spatial cells assigned to this process.
 * @param mpiGrid Parallel grid library.
 * @return If true, velocity block data was read successfully.*/
bool readBlockData(
        vlsv::ParallelReader& file,
        const string& meshName,
        const vector<CellID>& fileCells,
        const uint64_t localCellStartOffset,
        const uint64_t localCells,
        dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid
   ) {
   bool success = true;

   const uint64_t bytesReadStart = file.getBytesRead();
   int N_processes;
   MPI_Comm_size(MPI_COMM_WORLD,&N_processes);

   uint64_t arraySize;
   uint64_t vectorSize;
   vlsv::datatype::type dataType;
   uint64_t byteSize;
   uint64_t* offsetArray = new uint64_t[N_processes];

   for (uint popID=0; popID<getObjectWrapper().particleSpecies.size(); ++popID) {
      const string& popName = getObjectWrapper().particleSpecies[popID].name;

      // Create a cellID remapping lambda that can renumber our velocity space, should it's size have changed.
      // By default, this is a no-op that keeps the blockIDs untouched.
      std::function<vmesh::GlobalID(vmesh::GlobalID)> blockIDremapper = [](vmesh::GlobalID oldID) -> vmesh::GlobalID {return oldID;};

      // Check that velocity space extents and DV matches the grids we have created
      list<pair<string,string> > attribs;
      attribs.push_back(make_pair("mesh",popName));
      std::array<unsigned int, 6> fileMeshBBox;
      unsigned int* bufferpointer = &fileMeshBBox[0];
      if (file.read("MESH_BBOX",attribs,0,6,bufferpointer,false) == false) {
         logFile << "(RESTART) ERROR: Failed to read MESH_BBOX at " << __FILE__ << ":" << __LINE__ << endl << write;
         success = false;
      }

      const size_t meshID = getObjectWrapper().particleSpecies[popID].velocityMesh;
      const vmesh::MeshParameters& ourMeshParams = getObjectWrapper().velocityMeshes[meshID];
      if(fileMeshBBox[0] != ourMeshParams.gridLength[0] ||
            fileMeshBBox[1] != ourMeshParams.gridLength[1] ||
            fileMeshBBox[2] != ourMeshParams.gridLength[2]) {

         logFile << "(RESTART) INFO: velocity mesh sizes don't match:" << endl
                 << "    restart file has " << fileMeshBBox[0] << " x " << fileMeshBBox[1] << " x " << fileMeshBBox[2] << "," << endl
                 << "    config specifies " << ourMeshParams.gridLength[0] << " x " <<  ourMeshParams.gridLength[1] << " x " <<  ourMeshParams.gridLength[2] << endl << write;

         if(ourMeshParams.gridLength[0] < fileMeshBBox[0] ||
               ourMeshParams.gridLength[1] < fileMeshBBox[1] ||
               ourMeshParams.gridLength[2] < fileMeshBBox[2]) {
            logFile << "(RESTART) ERROR: trying to shrink velocity space." << endl << write;
            abort();
         }

         // If we are mismatched, we have to iterate through the velocity coords to see if we have a
         // chance at renumbering.
         std::vector<Real> fileVelCoordsX(fileMeshBBox[0]*fileMeshBBox[3]+1);
         std::vector<Real> fileVelCoordsY(fileMeshBBox[1]*fileMeshBBox[4]+1);
         std::vector<Real> fileVelCoordsZ(fileMeshBBox[2]*fileMeshBBox[5]+1);

         Real* tempPointer = fileVelCoordsX.data();
         if (file.read("MESH_NODE_CRDS_X",attribs,0,fileMeshBBox[0]*fileMeshBBox[3]+1,tempPointer,false) == false) {
            logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write;
            success = false;
         }
         tempPointer = fileVelCoordsY.data();
         if (file.read("MESH_NODE_CRDS_Y",attribs,0,fileMeshBBox[1]*fileMeshBBox[4]+1,tempPointer,false) == false) {
            logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write;
            success = false;
         }
         tempPointer = fileVelCoordsZ.data();
         if (file.read("MESH_NODE_CRDS_Z",attribs,0,fileMeshBBox[2]*fileMeshBBox[5]+1,tempPointer,false) == false) {
            logFile << "(RESTART) ERROR: Failed to read MESH_NODE_CRDS_X at " << __FILE__ << ":" << __LINE__ << endl << write;
            success = false;
         }

         const Real dVx = getObjectWrapper().velocityMeshes[meshID].cellSize[0];
         for(const auto& c : fileVelCoordsX) {
            Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0]) / dVx;
            if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) {
               logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl
                  << "          (X coordinate " << c << " = " << cellindex <<" * " << dVx << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0] << endl
                  << "           coordinate  = cellindex *   dV  +  meshMinLimits)" << endl << write;
               abort();
            }
         }

         const Real dVy = getObjectWrapper().velocityMeshes[meshID].cellSize[1];
         for(const auto& c : fileVelCoordsY) {
            Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1]) / dVy;
            if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) {
               logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl
                  << "           (Y coordinate " << c << " = " << cellindex <<" * " << dVy << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1] << endl
                  << "           coordinate  = cellindex *   dV  +  meshMinLimits)" << endl << write;
               abort();
            }
         }

         const Real dVz = getObjectWrapper().velocityMeshes[meshID].cellSize[2];
         for(const auto& c : fileVelCoordsY) {
            Real cellindex = (c - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2]) / dVz;
            if(fabs(nearbyint(cellindex) - cellindex) > 1./10000.) {
               logFile << "(RESTART) ERROR: Can't resize velocity space as cell coordinates don't match." << endl
                  << "           (Z coordinate " << c << " = " << cellindex <<" * " << dVz << " + " << getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2] << endl
                  << "           coordinate  = cellindex *   dV  +  meshMinLimits)" << endl << write;
               abort();
            }
         }

         // If we haven't aborted above, we can apparently renumber our
         // cellIDs. Build an approprita blockIDremapper lambda for this purpose.
         std::array<int, 3> velGridOffset;
         velGridOffset[0] = (fileVelCoordsX[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[0]) / dVx;
         velGridOffset[1] = (fileVelCoordsY[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[1]) / dVy;
         velGridOffset[2] = (fileVelCoordsZ[0] - getObjectWrapper().velocityMeshes[meshID].meshMinLimits[2]) / dVz;

         if((velGridOffset[0] % ourMeshParams.blockLength[0] != 0) ||
               (velGridOffset[1] % ourMeshParams.blockLength[1] != 0) ||
               (velGridOffset[2] % ourMeshParams.blockLength[2] != 0)) {
            logFile << "(RESTART) ERROR: resizing velocity space on restart must end up with the old velocity space" << endl
                    << "                 at a block boundary of the new space!" << endl
                    << "                 (It now starts at cell [" << velGridOffset[0] << ", " << velGridOffset[1] << "," << velGridOffset[2] << "])" << endl << write;
            abort();
         }

         velGridOffset[0] /= ourMeshParams.blockLength[0];
         velGridOffset[1] /= ourMeshParams.blockLength[1];
         velGridOffset[2] /= ourMeshParams.blockLength[2];

         blockIDremapper = [fileMeshBBox,velGridOffset,ourMeshParams](vmesh::GlobalID oldID) -> vmesh::GlobalID {
            unsigned int x,y,z;
            x = oldID % fileMeshBBox[0];
            y = (oldID / fileMeshBBox[0]) % fileMeshBBox[1];
            z = oldID / (fileMeshBBox[0] * fileMeshBBox[1]);

            x += velGridOffset[0];
            y += velGridOffset[1];
            z += velGridOffset[2];

            //logFile << " Remapping " << oldID << "(" << x << "," << y << "," << z << ") to " << x + y * ourMeshParams.gridLength[0] + z* ourMeshParams.gridLength[0] * ourMeshParams.gridLength[1] << endl << write;
            return x + y * ourMeshParams.gridLength[0] + z* ourMeshParams.gridLength[0] * ourMeshParams.gridLength[1];
         };

         logFile << "    => Resizing velocity space by renumbering GlobalIDs." << endl << endl << write;
      }

      // In restart files each spatial cell has an entry in CELLSWITHBLOCKS. 
      // Each process calculates how many velocity blocks it has for this species.
      attribs.clear();
      attribs.push_back(make_pair("mesh",meshName));
      attribs.push_back(make_pair("name",popName));
      vmesh::LocalID* blocksPerCell = NULL;
      
      if (file.read("BLOCKSPERCELL",attribs,localCellStartOffset,localCells,blocksPerCell,true) == false) {
         logFile << "(RESTART) ERROR: Failed to read BLOCKSPERCELL at " << __FILE__ << ":" << __LINE__ << endl << write;
         success = false;
      }

      // Count how many velocity blocks this process gets
      uint64_t blockSum = 0;
      for (uint64_t i=0; i<localCells; ++i){
         blockSum += blocksPerCell[i];
      }
      
      // Gather all block sums to master process who will them broadcast 
      // the values to everyone
      MPI_Allgather(&blockSum,1,MPI_Type<uint64_t>(),offsetArray,1,MPI_Type<uint64_t>(),MPI_COMM_WORLD);      
      
      // Calculate the offset from which this process starts reading block data
      uint64_t myOffset = 0;
      for (int64_t i=0; i<mpiGrid.get_rank(); ++i) myOffset += offsetArray[i];
      
      if (file.getArrayInfo("BLOCKVARIABLE",attribs,arraySize,vectorSize,dataType,byteSize) == false) {
         logFile << "(RESTART)  ERROR: Failed to read BLOCKVARIABLE INFO" << endl << write;
         return false;
      }

      // Call _readBlockData
      if (dataType == vlsv::datatype::type::FLOAT) {
         switch (byteSize) {
            case sizeof(double):
               if (_readBlockData<double>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
                                          myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
               break;
            case sizeof(float):
               if (_readBlockData<float>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
                                         myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
               break;
         }
      } else if (dataType == vlsv::datatype::type::UINT) {
         switch (byteSize) {
            case sizeof(uint32_t):
               if (_readBlockData<uint32_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
                                            myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
               break;
            case sizeof(uint64_t):
               if (_readBlockData<uint64_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
                                            myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
               break;
         }
      } else if (dataType == vlsv::datatype::type::INT) {
         switch (byteSize) {
            case sizeof(int32_t):
               if (_readBlockData<int32_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
                                           myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
               break;
            case sizeof(int64_t):
               if (_readBlockData<int64_t>(file,meshName,fileCells,localCellStartOffset,localCells,blocksPerCell,
                                           myOffset,blockSum,mpiGrid,blockIDremapper,popID) == false) success = false;
               break;
         }
      } else {
         logFile << "(RESTART) ERROR: Failed to read data type at readCellParamsVariable" << endl << write;
         success = false;
      }
      delete [] blocksPerCell; blocksPerCell = NULL;
   } // for-loop over particle species

   delete [] offsetArray; offsetArray = NULL;
   
   const uint64_t bytesReadEnd = file.getBytesRead() - bytesReadStart;
   logFile << "Velocity meshes and data read, approximate data rate is ";
   logFile << vlsv::printDataRate(bytesReadEnd,file.getReadTime()) << endl << write;

   return success;
}
Beispiel #2
0
    > void initialize(
        const Geometries& geometries,
        Init_Cond& initial_conditions,
        const Background_Magnetic_Field& bg_B,
        dccrg::Dccrg<Cell, Geometry>& grid,
        const std::vector<uint64_t>& cells,
        const double time,
        const double adiabatic_index,
        const double vacuum_permeability,
        const double proton_mass,
        const bool verbose,
        const Mass_Density_Getter Mas,
        const Momentum_Density_Getter Mom,
        const Total_Energy_Density_Getter Nrj,
        const Magnetic_Field_Getter Mag,
        const Background_Magnetic_Field_Pos_X_Getter Bg_B_Pos_X,
        const Background_Magnetic_Field_Pos_Y_Getter Bg_B_Pos_Y,
        const Background_Magnetic_Field_Pos_Z_Getter Bg_B_Pos_Z,
        const Mass_Density_Flux_Getter Mas_f,
        const Momentum_Density_Flux_Getter Mom_f,
        const Total_Energy_Density_Flux_Getter Nrj_f,
        const Magnetic_Field_Flux_Getter Mag_f
    ) {
    if (verbose and grid.get_rank() == 0) {
        std::cout << "Setting default MHD state... ";
        std::cout.flush();
    }
    // set default state
    for (const auto cell_id: cells) {
        auto* const cell_data = grid[cell_id];
        if (cell_data == nullptr) {
            std::cerr <<  __FILE__ << "(" << __LINE__ << ") No data for cell: "
                      << cell_id
                      << std::endl;
            abort();
        }

        // zero fluxes and background fields
        Mas_f(*cell_data)         =
            Nrj_f(*cell_data)         =
                Mom_f(*cell_data)[0]      =
                    Mom_f(*cell_data)[1]      =
                        Mom_f(*cell_data)[2]      =
                            Mag_f(*cell_data)[0]      =
                                Mag_f(*cell_data)[1]      =
                                    Mag_f(*cell_data)[2]      = 0;

        const auto c = grid.geometry.get_center(cell_id);
        const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
        const auto
        lat = asin(c[2] / r),
        lon = atan2(c[1], c[0]);

        const auto mass_density
            = proton_mass
              * initial_conditions.get_default_data(
                  Number_Density(),
                  time,
                  c[0], c[1], c[2],
                  r, lat, lon
              );
        const auto velocity
            = initial_conditions.get_default_data(
                  Velocity(),
                  time,
                  c[0], c[1], c[2],
                  r, lat, lon
              );
        const auto pressure
            = initial_conditions.get_default_data(
                  Pressure(),
                  time,
                  c[0], c[1], c[2],
                  r, lat, lon
              );
        const auto magnetic_field
            = initial_conditions.get_default_data(
                  Magnetic_Field(),
                  time,
                  c[0], c[1], c[2],
                  r, lat, lon
              );

        Mas(*cell_data) = mass_density;
        Mom(*cell_data) = mass_density * velocity;
        Mag(*cell_data) = magnetic_field;
        Nrj(*cell_data) = get_total_energy_density(
                              mass_density,
                              velocity,
                              pressure,
                              magnetic_field,
                              adiabatic_index,
                              vacuum_permeability
                          );

        const auto cell_end = grid.geometry.get_max(cell_id);
        Bg_B_Pos_X(*cell_data) = bg_B.get_background_field(
        {cell_end[0], c[1], c[2]},
        vacuum_permeability
        );
        Bg_B_Pos_Y(*cell_data) = bg_B.get_background_field(
        {c[0], cell_end[1], c[2]},
        vacuum_permeability
        );
        Bg_B_Pos_Z(*cell_data) = bg_B.get_background_field(
        {c[0], c[1], cell_end[2]},
        vacuum_permeability
        );
    }

    // set non-default initial conditions
    if (verbose and grid.get_rank() == 0) {
        std::cout << "done\nSetting non-default initial MHD state... ";
        std::cout.flush();
    }

    /*
    Set non-default initial conditions
    */

    // mass density
    for (
        size_t i = 0;
        i < initial_conditions.get_number_of_regions(Number_Density());
        i++
    ) {
        const auto& init_cond = initial_conditions.get_initial_condition(Number_Density(), i);
        const auto& geometry_id = init_cond.get_geometry_id();
        const auto& cells = geometries.get_cells(geometry_id);
        for (const auto& cell: cells) {
            const auto c = grid.geometry.get_center(cell);
            const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
            const auto
            lat = asin(c[2] / r),
            lon = atan2(c[1], c[0]);

            const auto mass_density
                = proton_mass
                  * initial_conditions.get_data(
                      Number_Density(),
                      geometry_id,
                      time,
                      c[0], c[1], c[2],
                      r, lat, lon
                  );

            auto* const cell_data = grid[cell];
            if (cell_data == NULL) {
                std::cerr <<  __FILE__ << "(" << __LINE__ << std::endl;
                abort();
            }

            Mas(*cell_data) = mass_density;
        }
    }

    // velocity
    for (
        size_t i = 0;
        i < initial_conditions.get_number_of_regions(Velocity());
        i++
    ) {
        const auto& init_cond = initial_conditions.get_initial_condition(Velocity(), i);
        const auto& geometry_id = init_cond.get_geometry_id();
        const auto& cells = geometries.get_cells(geometry_id);
        for (const auto& cell: cells) {
            const auto c = grid.geometry.get_center(cell);
            const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
            const auto
            lat = asin(c[2] / r),
            lon = atan2(c[1], c[0]);

            const auto velocity = initial_conditions.get_data(
                                      Velocity(),
                                      geometry_id,
                                      time,
                                      c[0], c[1], c[2],
                                      r, lat, lon
                                  );

            auto* const cell_data = grid[cell];
            if (cell_data == NULL) {
                std::cerr <<  __FILE__ << "(" << __LINE__
                          << ") No data for cell: " << cell
                          << std::endl;
                abort();
            }

            Mom(*cell_data) = Mas(*cell_data) * velocity;
        }
    }

    // magnetic field
    for (
        size_t i = 0;
        i < initial_conditions.get_number_of_regions(Magnetic_Field());
        i++
    ) {
        const auto& init_cond = initial_conditions.get_initial_condition(Magnetic_Field(), i);
        const auto& geometry_id = init_cond.get_geometry_id();
        const auto& cells = geometries.get_cells(geometry_id);
        for (const auto& cell: cells) {
            const auto c = grid.geometry.get_center(cell);
            const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
            const auto
            lat = asin(c[2] / r),
            lon = atan2(c[1], c[0]);

            const auto magnetic_field = initial_conditions.get_data(
                                            Magnetic_Field(),
                                            geometry_id,
                                            time,
                                            c[0], c[1], c[2],
                                            r, lat, lon
                                        );

            auto* const cell_data = grid[cell];
            if (cell_data == NULL) {
                std::cerr <<  __FILE__ << "(" << __LINE__
                          << ") No data for cell: " << cell
                          << std::endl;
                abort();
            }

            Mag(*cell_data) = magnetic_field;
        }
    }

    // pressure
    for (
        size_t i = 0;
        i < initial_conditions.get_number_of_regions(Pressure());
        i++
    ) {
        std::cout << std::endl;
        const auto& init_cond = initial_conditions.get_initial_condition(Pressure(), i);
        const auto& geometry_id = init_cond.get_geometry_id();
        std::cout << geometry_id << std::endl;
        const auto& cells = geometries.get_cells(geometry_id);
        std::cout << cells.size() << std::endl;
        for (const auto& cell: cells) {
            const auto c = grid.geometry.get_center(cell);
            const auto r = sqrt(c[0]*c[0] + c[1]*c[1] + c[2]*c[2]);
            const auto
            lat = asin(c[2] / r),
            lon = atan2(c[1], c[0]);

            const auto pressure = initial_conditions.get_data(
                                      Pressure(),
                                      geometry_id,
                                      time,
                                      c[0], c[1], c[2],
                                      r, lat, lon
                                  );

            auto* const cell_data = grid[cell];
            if (cell_data == NULL) {
                std::cerr <<  __FILE__ << "(" << __LINE__
                          << ") No data for cell: " << cell
                          << std::endl;
                abort();
            }

            Nrj(*cell_data) = get_total_energy_density(
                                  Mas(*cell_data),
                                  Mom(*cell_data) / Mas(*cell_data),
                                  pressure,
                                  Mag(*cell_data),
                                  adiabatic_index,
                                  vacuum_permeability
                              );
        }
    }

    if (verbose and grid.get_rank() == 0) {
        std::cout << "done" << std::endl;
    }
}