示例#1
0
void NetcdfFile::Sync(Parallel::Comm const& commIn) {
  int nc_vars[23];
  if (commIn.Master()) {
    nc_vars[0] = ncframe_;
    nc_vars[1] = TempVID_;
    nc_vars[2] = coordVID_;
    nc_vars[3] = velocityVID_;
    nc_vars[4] = frcVID_;
    nc_vars[5] = cellAngleVID_;
    nc_vars[6] = cellLengthVID_;
    nc_vars[7] = timeVID_;
    nc_vars[8] = remd_dimension_;
    nc_vars[9] = indicesVID_;
    nc_vars[10] = ncdebug_;
    nc_vars[11] = ensembleDID_;
    nc_vars[12] = frameDID_;
    nc_vars[13] = atomDID_;
    nc_vars[14] = ncatom_;
    nc_vars[15] = ncatom3_;
    nc_vars[16] = spatialDID_;
    nc_vars[17] = labelDID_;
    nc_vars[18] = cell_spatialDID_;
    nc_vars[19] = cell_angularDID_;
    nc_vars[20] = spatialVID_;
    nc_vars[21] = cell_spatialVID_;
    nc_vars[22] = cell_angularVID_;
  }
  commIn.MasterBcast( nc_vars, 23, MPI_INT );
  if (!commIn.Master()) {
    ncframe_ = nc_vars[0];
    TempVID_ = nc_vars[1];
    coordVID_ = nc_vars[2];
    velocityVID_ = nc_vars[3];
    frcVID_ = nc_vars[4];
    cellAngleVID_ = nc_vars[5];
    cellLengthVID_ = nc_vars[6];
    timeVID_ = nc_vars[7];
    remd_dimension_ = nc_vars[8];
    indicesVID_ = nc_vars[9];
    ncdebug_ = nc_vars[10];
    ensembleDID_ = nc_vars[11];
    frameDID_ = nc_vars[12];
    atomDID_ = nc_vars[13];
    ncatom_ = nc_vars[14];
    ncatom3_ = nc_vars[15];
    spatialDID_ = nc_vars[16];
    labelDID_ = nc_vars[17];
    cell_spatialDID_ = nc_vars[18];
    cell_angularDID_ = nc_vars[19];
    spatialVID_ = nc_vars[20];
    cell_spatialVID_ = nc_vars[21];
    cell_angularVID_ = nc_vars[22];
  }
}
示例#2
0
int DataSet_GridDbl::Sync(size_t total, std::vector<int> const& rank_frames,
                          Parallel::Comm const& commIn)
{
  if (commIn.Master()) {
    std::vector<double> buf( grid_.size() );
    commIn.Reduce( &(buf[0]), &(grid_[0]), grid_.size(), MPI_FLOAT, MPI_SUM );
    std::copy( buf.begin(), buf.end(), grid_.begin() );
  } else
    commIn.Reduce( 0,         &(grid_[0]), grid_.size(), MPI_FLOAT, MPI_SUM );
  return 0;
}
示例#3
0
/** First master performs all necessary setup, then sends info to all children.
  */
int Traj_AmberNetcdf::parallelSetupTrajout(FileName const& fname, Topology* trajParm,
                                           CoordinateInfo const& cInfoIn,
                                           int NframesToWrite, bool append,
                                           Parallel::Comm const& commIn)
{
  int err = 0;
  if (commIn.Master()) {
    err = setupTrajout(fname, trajParm, cInfoIn, NframesToWrite, append);
    // NOTE: setupTrajout leaves file open. Should this change?
    NC_close();
  }
  commIn.MasterBcast(&err, 1, MPI_INT);
  if (err != 0) return 1;
  // Synchronize netcdf info on non-master threads.
  Sync(commIn);
  if (!commIn.Master()) {
    // Non masters need filename and allocate Coord
    filename_ = fname;
    if (Coord_ != 0) delete[] Coord_;
    Coord_ = new float[ Ncatom3() ];
  }
  return 0;
}
示例#4
0
// DataSet_double::Sync()
int DataSet_double::Sync(size_t total, std::vector<int> const& rank_frames,
                         Parallel::Comm const& commIn)
{
  if (commIn.Size()==1) return 0;
  if (commIn.Master()) {
    // Resize for total number of frames.
    Data_.resize( total );
    double* endptr = &(Data_[0]) + rank_frames[0];
    // Receive data from each rank
    for (int rank = 1; rank < commIn.Size(); rank++) {
      commIn.SendMaster( endptr, rank_frames[rank], rank, MPI_DOUBLE );
      endptr += rank_frames[rank];
    }
  } else // Send data to master //TODO adjust for repeated additions?
    commIn.SendMaster( &(Data_[0]), Data_.size(), commIn.Rank(), MPI_DOUBLE );
  return 0;
}
示例#5
0
/** Each rank searches for replica based on lowest replica number. */
int TrajIOarray::SearchForReplicas(FileName const& fname, Parallel::Comm const& ensComm,
                                   Parallel::Comm const& trajComm)
{
  RepName repName(fname, debug_);
  if (repName.Error()) return 1;
  // TODO check for lower replica number?
  FileName replicaFilename = repName.RepFilename( ensComm.Rank() );
  // Only traj comm masters actually check for files.
  if (trajComm.Master()) {
    if (!File::Exists( replicaFilename )) {
      File::ErrorMsg( replicaFilename.full() );
      rprinterr("Error: File '%s' not accessible.\n", replicaFilename.full());
      return 1;
    }
  }
  // At this point each rank has found its replica. Populate filename array.
  for (int offset = 0; offset < ensComm.Size(); ++offset)
    replica_filenames_.push_back( repName.RepFilename( offset ) );
  return 0;
}
示例#6
0
/** Each rank checks that specified file is present. */
int TrajIOarray::AddReplicasFromArgs(FileName const& name0,
                                     std::string const& commaNames,
                                     Parallel::Comm const& ensComm,
                                     Parallel::Comm const& trajComm)
{
  // First set up filename array on all ranks.
  if (name0.empty()) return 1;
  replica_filenames_.push_back( name0 );
  ArgList remdtraj_list( commaNames, "," );
  for (ArgList::const_iterator fname = remdtraj_list.begin();
                               fname != remdtraj_list.end(); ++fname)
    replica_filenames_.push_back( FileName( *fname ) );
  if (ensComm.Size() != (int)replica_filenames_.size())
    return 1;
  else if (trajComm.Master()) { // Only traj comm master checks file
    if (!File::Exists( replica_filenames_[ ensComm.Rank() ])) {
      File::ErrorMsg( replica_filenames_[ensComm.Rank()].full() );
      rprinterr("Error: File '%s' not accessible.\n", replica_filenames_[ensComm.Rank()].full());
      return 1;
    }
  }
  return 0;
}
示例#7
0
/** Each rank only sets up file that it will process. */
int TrajIOarray::SetupIOarray(ArgList& argIn, TrajFrameCounter& counter,
                              CoordinateInfo& cInfo, Topology* trajParm,
                              Parallel::Comm const& ensComm, Parallel::Comm const& trajComm)
{
  // Sanity check
  if (!IOarray_.empty()) {
    mprinterr("Internal Error: SetupIOarray() has been called twice.\n");
    return 1;
  }
  // Detect format
  FileName const& repFname = replica_filenames_[ensComm.Rank()];
  TrajectoryFile::TrajFormatType repformat = TrajectoryFile::UNKNOWN_TRAJ;
  TrajectoryIO* replica0 = TrajectoryFile::DetectFormat( repFname, repformat );
  if ( replica0 == 0 ) {
    mprinterr("Error: Could not set up replica file %s\n", repFname.full());
    return 1;
  }
  mprintf("\tReading '%s' as %s\n", repFname.full(), TrajectoryFile::FormatString(repformat));
  replica0->SetDebug( debug_ );
  // Construct the IOarray_ with blanks for all except this rank.
  for (int member = 0; member != ensComm.Size(); member++)
    if (member == ensComm.Rank())
      IOarray_.push_back( replica0 );
    else
      IOarray_.push_back( 0 );
  // Process format-specific read args.
  replica0->processReadArgs( argIn );
  // Set up replica for reading and get # frames
  int nframes = replica0->setupTrajin( repFname, trajParm );
  if (nframes == TrajectoryIO::TRAJIN_ERR) {
    mprinterr("Error: Could not set up %s for reading.\n", repFname.full());
    return 1;
  }
  // Set coordinate info
  cInfo = replica0->CoordInfo();
  int totalFrames = nframes;
  if (cInfo.ReplicaDimensions().Ndims() > 0) { // TODO put in common routine
    mprintf("\tReplica dimensions:\n");
    for (int rd = 0; rd < cInfo.ReplicaDimensions().Ndims(); rd++)
      mprintf("\t\t%i: %s\n", rd+1, cInfo.ReplicaDimensions().Description(rd));
  }
  // Check # frames in all files, use lowest.
  Parallel::World().AllReduce( &totalFrames, &nframes, 1, MPI_INT, MPI_MIN );
  if (totalFrames != nframes) {
    rprintf("Warning: Replica '%s' frames (%i) is > # frames in shortest replica.\n",
            repFname.full(), nframes);
    mprintf("Warning: Setting total # of frames to read from replica ensemble to %i\n",
            totalFrames);
  }
  if (trajComm.Master()) {
    static const int iSize = 6;
    static const char* iTitle[iSize] = {"box", "velocity", "temperature", "time", "force",
                                        "replica dimensions"};
    // Check coordinate info of all files               0    1    2     3     4      5
    std::vector<int> Info( iSize * ensComm.Size() ); // box, vel, temp, time, force, nRepDims
    int rank_info[iSize];
    rank_info[0] = (int)cInfo.TrajBox().Type();
    rank_info[1] = (int)cInfo.HasVel();
    rank_info[2] = (int)cInfo.HasTemp();
    rank_info[3] = (int)cInfo.HasTime();
    rank_info[4] = (int)cInfo.HasForce();
    rank_info[5] = cInfo.ReplicaDimensions().Ndims();
    ensComm.AllGather( rank_info, iSize, MPI_INT, &Info[0] );
    // TODO Should mismatches be errors instead?
    for (int midx = 0; midx != iSize; midx++) {
      for (int ridx = midx + iSize; ridx < (int)Info.size(); ridx += iSize) {
        if (Info[midx] != Info[ridx]) {
          rprintf("Warning: Replica %i %s info does not match first replica.\n",
                  ridx/iSize, iTitle[midx]);
        }
      }
    }
  }
  // TODO: Put code below into a common routine with serial version
  // Check how many frames will actually be read
  if (counter.CheckFrameArgs( totalFrames, argIn )) return 1;
  // SANITY CHECK
  if (IOarray_.size() != replica_filenames_.size()) {
    mprinterr("Error: Not all replica files were set up.\n");
    return 1;
  }
  // Update ensemble size
  cInfo.SetEnsembleSize( (int)IOarray_.size() );
  if (debug_ > 0)
    cInfo.PrintCoordInfo( repFname.full(), trajParm->c_str() );

  return 0;
}
示例#8
0
/** First master performs all necessary setup, then sends info to all children.
  */
int Traj_GmxTrX::parallelSetupTrajout(FileName const& fname, Topology* trajParm,
                                      CoordinateInfo const& cInfoIn,
                                      int NframesToWrite, bool append,
                                      Parallel::Comm const& commIn)
{
    int err = 0;
    // In parallel MUST know # of frames to write in order to correctly set size
    if (NframesToWrite < 1) {
        mprinterr("Error: # frames to write must be known for TRR output in parallel.\n");
        err = 1;
    } else if (commIn.Master()) {
        err = setupTrajout(fname, trajParm, cInfoIn, NframesToWrite, append);
        // Determine header size, (18 * 4) + titleSize TODO put in setupTrajout?
        headerBytes_ = (18 * 4) + Title().size();
        // Determine frame size
        frameSize_ = headerBytes_ + box_size_ + x_size_ + v_size_;
        // NOTE: setupTrajout leaves file open. Should this change?
        file_.CloseFile();
    }
    commIn.MasterBcast(&err, 1, MPI_INT);
    if (err != 0) return 1;
    // Synchronize info on non-master threads.
    SyncTrajIO( commIn );
    commIn.MasterBcast( &ir_size_, 1, MPI_INT );
    commIn.MasterBcast( &e_size_,  1, MPI_INT );
    commIn.MasterBcast( &box_size_, 1, MPI_INT );
    commIn.MasterBcast( &vir_size_, 1, MPI_INT );
    commIn.MasterBcast( &pres_size_, 1, MPI_INT );
    commIn.MasterBcast( &top_size_, 1, MPI_INT );
    commIn.MasterBcast( &sym_size_, 1, MPI_INT );
    commIn.MasterBcast( &x_size_, 1, MPI_INT );
    commIn.MasterBcast( &v_size_, 1, MPI_INT );
    commIn.MasterBcast( &f_size_, 1, MPI_INT );
    commIn.MasterBcast( &natoms_, 1, MPI_INT );
    commIn.MasterBcast( &natom3_, 1, MPI_INT );
    commIn.MasterBcast( &step_, 1, MPI_INT );
    commIn.MasterBcast( &nre_, 1, MPI_INT );
    commIn.MasterBcast( &precision_, 1, MPI_INT );
    commIn.MasterBcast( &dt_, 1, MPI_FLOAT );
    commIn.MasterBcast( &lambda_, 1, MPI_FLOAT );
    // NOTE: cast these to unsigned long long to avoid ambiguity since MPI doesnt have size_t
    unsigned long long buf[2];
    if (commIn.Master()) {
        buf[0] = (unsigned long long)frameSize_;
        buf[1] = (unsigned long long)headerBytes_;
        commIn.MasterBcast( buf, 2, MPI_UNSIGNED_LONG_LONG );
    } else {
        commIn.MasterBcast( buf, 2, MPI_UNSIGNED_LONG_LONG );
        frameSize_ = (size_t)buf[0];
        headerBytes_ = (size_t)buf[1];
        AllocateCoords(); // Should already be done on master
    }
    if (append)
        file_.SetupWrite( fname, debug_ );
    else
        file_.SetupAppend( fname, debug_ );
    if (debug_ > 0)
        rprintf("Gromacs TRR: parallel headerSize= %zu  frameSize= %zu\n", headerBytes_, frameSize_);

    return 0;
}
示例#9
0
/** Synchronize timeseries data from child ranks to master. */
int DataSetList::SynchronizeData(Parallel::Comm const& commIn) {
  if (commIn.Size() < 2) return 0;
  // Ensure that the number of sets that require sync is same on each rank.
  // FIXME: Make sure this allgather does not end up taking too much time.
  //        Should it be debug only?
  std::vector<int> size_on_rank;
  size_on_rank.reserve( DataList_.size() );
  DataListType SetsToSync;
  for (DataListType::iterator ds = DataList_.begin(); ds != DataList_.end(); ++ds)
    if ( (*ds)->NeedsSync() ) {
      SetsToSync.push_back( *ds );
      size_on_rank.push_back( (*ds)->Size() );
    }
// DEBUG
  //for (int rank = 0; rank != commIn.Size(); rank++) {
  //  if (rank == commIn.Rank())
  //    for (DataListType::const_iterator ds = SetsToSync.begin(); ds != SetsToSync.end(); ++ds)
  //      rprintf("SET '%s'\n", (*ds)->legend());
  //  commIn.Barrier();
  //}
// DEBUG END
  std::vector<int> n_on_rank( commIn.Size(), 0 );
  int nSets = (int)SetsToSync.size();
  commIn.AllGather( &nSets, 1, MPI_INT, &n_on_rank[0] );
  for (int rank = 1; rank < commIn.Size(); rank++)
    if (n_on_rank[rank] != n_on_rank[0]) {
      mprinterr("Internal Error: Number of sets to sync on rank %i (%i) != number on master %i\n",
                rank, n_on_rank[rank], n_on_rank[0]);
      return 1;
    }
  // Send all data set sizes to master.
  std::vector<int> all_rank_sizes;
  if (commIn.Master()) {
    all_rank_sizes.resize( nSets * commIn.Size() );
    commIn.GatherMaster( &size_on_rank[0], nSets, MPI_INT, &all_rank_sizes[0] );
  } else {
    commIn.GatherMaster( &size_on_rank[0], nSets, MPI_INT, 0 );
  }
  size_on_rank.clear();
  // Call Sync only for sets that need it.
  std::vector<int> rank_frames( commIn.Size() );
  int total = 0; //TODO size_t?
  int idx0 = 0;
  for (DataListType::iterator ds = SetsToSync.begin(); ds != SetsToSync.end(); ++ds, ++idx0) {
    if (commIn.Master()) {
      total = all_rank_sizes[idx0];
      rank_frames[0] = all_rank_sizes[idx0];
      int idx1 = idx0 + nSets;
      for (int rank = 1; rank < commIn.Size(); rank++, idx1 += nSets) {
        total += all_rank_sizes[idx1];
        rank_frames[rank] = all_rank_sizes[idx1];
      }
      //mprintf("DEBUG: Syncing '%s' (size=%zu, total=%i)\n", (*ds)->Meta().PrintName().c_str(),
      //        (*ds)->Size(), total);
    }
    if ( (*ds)->Sync(total, rank_frames, commIn) ) {
      rprintf( "Warning: Could not sync dataset '%s'\n",(*ds)->legend());
      //return;
    }
    (*ds)->SetNeedsSync( false );
  }
  return 0;
}
示例#10
0
/** First master performs all necessary setup, then sends info to all children.
  */
int Traj_AmberCoord::parallelSetupTrajout(FileName const& fname, Topology* trajParm,
                                           CoordinateInfo const& cInfoIn,
                                           int NframesToWrite, bool append,
                                           Parallel::Comm const& commIn)
{
  int err = 0;
  // In parallel MUST know # of frames to write in order to correctly set size
  if (NframesToWrite < 1) {
    mprinterr("Error: # frames to write must be known for Amber Coords output in parallel.\n");
    err = 1;
  } else if (commIn.Master()) {
    // NOTE: This writes the title.
    err = setupTrajout(fname, trajParm, cInfoIn, NframesToWrite, append);
    // NOTE: setupTrajout leaves file open. Should this change?
    file_.CloseFile();
  }
  commIn.MasterBcast(&err, 1, MPI_INT);
  if (err != 0) return 1;
  // Synchronize info on non-master threads.
  SyncTrajIO( commIn );
  // TODO For simplicity convert everything to double. Is this just lazy?
  double tmpArray[10];
  if (commIn.Master()) {
    tmpArray[0] = (double)natom3_;
    tmpArray[1] = (double)headerSize_;
    tmpArray[2] = (double)tStart_;
    tmpArray[3] = (double)tEnd_;
    tmpArray[4] = (double)numBoxCoords_;
    tmpArray[5] = boxAngle_[0];
    tmpArray[6] = boxAngle_[1];
    tmpArray[7] = boxAngle_[2];
    tmpArray[8] = (double)writeType_;
    tmpArray[9] = (double)highPrecision_;
    commIn.MasterBcast(tmpArray, 10, MPI_DOUBLE);
  } else {
    commIn.MasterBcast(tmpArray, 10, MPI_DOUBLE);
    natom3_        = (int)tmpArray[0];
    headerSize_    = (size_t)tmpArray[1];
    tStart_        = (size_t)tmpArray[2];
    tEnd_          = (size_t)tmpArray[3];
    numBoxCoords_  = (int)tmpArray[4];
    boxAngle_[0]   = tmpArray[5];
    boxAngle_[1]   = tmpArray[6];
    boxAngle_[2]   = tmpArray[7];
    writeType_     = (WriteType)tmpArray[8];
    highPrecision_ = (bool)tmpArray[9];
    if (append)
      file_.SetupAppend( fname, debug_ );
    else
      file_.SetupWrite( fname, debug_ );
    if (highPrecision_) outfmt_ = "%8.6f";
  }
  // For parallel output we will need to seek. Set up the buffer again with correct offsets.
  // Figure out the size of the written title.
  unsigned int titleSize = (unsigned int)Title().size() + 1; // +1 for newline
  titleSize = std::min(81U, titleSize);
  file_.SetupFrameBuffer( natom3_, 8, 10, headerSize_, titleSize );
  file_.ResizeBuffer( numBoxCoords_ );
  if (debug_>0)
    rprintf("'%s'(Parallel): Each frame has %lu bytes.\n", file_.Filename().base(),
            file_.FrameSize());
  // TODO set file size
  return 0;
}