void NetcdfFile::Sync(Parallel::Comm const& commIn) {
int nc_vars[23];
if (commIn.Master()) {
nc_vars[0] = ncframe_;
nc_vars[1] = TempVID_;
nc_vars[2] = coordVID_;
nc_vars[3] = velocityVID_;
nc_vars[4] = frcVID_;
nc_vars[5] = cellAngleVID_;
nc_vars[6] = cellLengthVID_;
nc_vars[7] = timeVID_;
nc_vars[8] = remd_dimension_;
nc_vars[9] = indicesVID_;
nc_vars[10] = ncdebug_;
nc_vars[11] = ensembleDID_;
nc_vars[12] = frameDID_;
nc_vars[13] = atomDID_;
nc_vars[14] = ncatom_;
nc_vars[15] = ncatom3_;
nc_vars[16] = spatialDID_;
nc_vars[17] = labelDID_;
nc_vars[18] = cell_spatialDID_;
nc_vars[19] = cell_angularDID_;
nc_vars[20] = spatialVID_;
nc_vars[21] = cell_spatialVID_;
nc_vars[22] = cell_angularVID_;
}
commIn.MasterBcast( nc_vars, 23, MPI_INT );
if (!commIn.Master()) {
ncframe_ = nc_vars[0];
TempVID_ = nc_vars[1];
coordVID_ = nc_vars[2];
velocityVID_ = nc_vars[3];
frcVID_ = nc_vars[4];
cellAngleVID_ = nc_vars[5];
cellLengthVID_ = nc_vars[6];
timeVID_ = nc_vars[7];
remd_dimension_ = nc_vars[8];
indicesVID_ = nc_vars[9];
ncdebug_ = nc_vars[10];
ensembleDID_ = nc_vars[11];
frameDID_ = nc_vars[12];
atomDID_ = nc_vars[13];
ncatom_ = nc_vars[14];
ncatom3_ = nc_vars[15];
spatialDID_ = nc_vars[16];
labelDID_ = nc_vars[17];
cell_spatialDID_ = nc_vars[18];
cell_angularDID_ = nc_vars[19];
spatialVID_ = nc_vars[20];
cell_spatialVID_ = nc_vars[21];
cell_angularVID_ = nc_vars[22];
}
}
/** First master performs all necessary setup, then sends info to all children.
*/
int Traj_AmberNetcdf::parallelSetupTrajout(FileName const& fname, Topology* trajParm,
CoordinateInfo const& cInfoIn,
int NframesToWrite, bool append,
Parallel::Comm const& commIn)
{
int err = 0;
if (commIn.Master()) {
err = setupTrajout(fname, trajParm, cInfoIn, NframesToWrite, append);
// NOTE: setupTrajout leaves file open. Should this change?
NC_close();
}
commIn.MasterBcast(&err, 1, MPI_INT);
if (err != 0) return 1;
// Synchronize netcdf info on non-master threads.
Sync(commIn);
if (!commIn.Master()) {
// Non masters need filename and allocate Coord
filename_ = fname;
if (Coord_ != 0) delete[] Coord_;
Coord_ = new float[ Ncatom3() ];
}
return 0;
}
/** First master performs all necessary setup, then sends info to all children.
*/
int Traj_GmxTrX::parallelSetupTrajout(FileName const& fname, Topology* trajParm,
CoordinateInfo const& cInfoIn,
int NframesToWrite, bool append,
Parallel::Comm const& commIn)
{
int err = 0;
// In parallel MUST know # of frames to write in order to correctly set size
if (NframesToWrite < 1) {
mprinterr("Error: # frames to write must be known for TRR output in parallel.\n");
err = 1;
} else if (commIn.Master()) {
err = setupTrajout(fname, trajParm, cInfoIn, NframesToWrite, append);
// Determine header size, (18 * 4) + titleSize TODO put in setupTrajout?
headerBytes_ = (18 * 4) + Title().size();
// Determine frame size
frameSize_ = headerBytes_ + box_size_ + x_size_ + v_size_;
// NOTE: setupTrajout leaves file open. Should this change?
file_.CloseFile();
}
commIn.MasterBcast(&err, 1, MPI_INT);
if (err != 0) return 1;
// Synchronize info on non-master threads.
SyncTrajIO( commIn );
commIn.MasterBcast( &ir_size_, 1, MPI_INT );
commIn.MasterBcast( &e_size_, 1, MPI_INT );
commIn.MasterBcast( &box_size_, 1, MPI_INT );
commIn.MasterBcast( &vir_size_, 1, MPI_INT );
commIn.MasterBcast( &pres_size_, 1, MPI_INT );
commIn.MasterBcast( &top_size_, 1, MPI_INT );
commIn.MasterBcast( &sym_size_, 1, MPI_INT );
commIn.MasterBcast( &x_size_, 1, MPI_INT );
commIn.MasterBcast( &v_size_, 1, MPI_INT );
commIn.MasterBcast( &f_size_, 1, MPI_INT );
commIn.MasterBcast( &natoms_, 1, MPI_INT );
commIn.MasterBcast( &natom3_, 1, MPI_INT );
commIn.MasterBcast( &step_, 1, MPI_INT );
commIn.MasterBcast( &nre_, 1, MPI_INT );
commIn.MasterBcast( &precision_, 1, MPI_INT );
commIn.MasterBcast( &dt_, 1, MPI_FLOAT );
commIn.MasterBcast( &lambda_, 1, MPI_FLOAT );
// NOTE: cast these to unsigned long long to avoid ambiguity since MPI doesnt have size_t
unsigned long long buf[2];
if (commIn.Master()) {
buf[0] = (unsigned long long)frameSize_;
buf[1] = (unsigned long long)headerBytes_;
commIn.MasterBcast( buf, 2, MPI_UNSIGNED_LONG_LONG );
} else {
commIn.MasterBcast( buf, 2, MPI_UNSIGNED_LONG_LONG );
frameSize_ = (size_t)buf[0];
headerBytes_ = (size_t)buf[1];
AllocateCoords(); // Should already be done on master
}
if (append)
file_.SetupWrite( fname, debug_ );
else
file_.SetupAppend( fname, debug_ );
if (debug_ > 0)
rprintf("Gromacs TRR: parallel headerSize= %zu frameSize= %zu\n", headerBytes_, frameSize_);
return 0;
}
/** First master performs all necessary setup, then sends info to all children.
*/
int Traj_AmberCoord::parallelSetupTrajout(FileName const& fname, Topology* trajParm,
CoordinateInfo const& cInfoIn,
int NframesToWrite, bool append,
Parallel::Comm const& commIn)
{
int err = 0;
// In parallel MUST know # of frames to write in order to correctly set size
if (NframesToWrite < 1) {
mprinterr("Error: # frames to write must be known for Amber Coords output in parallel.\n");
err = 1;
} else if (commIn.Master()) {
// NOTE: This writes the title.
err = setupTrajout(fname, trajParm, cInfoIn, NframesToWrite, append);
// NOTE: setupTrajout leaves file open. Should this change?
file_.CloseFile();
}
commIn.MasterBcast(&err, 1, MPI_INT);
if (err != 0) return 1;
// Synchronize info on non-master threads.
SyncTrajIO( commIn );
// TODO For simplicity convert everything to double. Is this just lazy?
double tmpArray[10];
if (commIn.Master()) {
tmpArray[0] = (double)natom3_;
tmpArray[1] = (double)headerSize_;
tmpArray[2] = (double)tStart_;
tmpArray[3] = (double)tEnd_;
tmpArray[4] = (double)numBoxCoords_;
tmpArray[5] = boxAngle_[0];
tmpArray[6] = boxAngle_[1];
tmpArray[7] = boxAngle_[2];
tmpArray[8] = (double)writeType_;
tmpArray[9] = (double)highPrecision_;
commIn.MasterBcast(tmpArray, 10, MPI_DOUBLE);
} else {
commIn.MasterBcast(tmpArray, 10, MPI_DOUBLE);
natom3_ = (int)tmpArray[0];
headerSize_ = (size_t)tmpArray[1];
tStart_ = (size_t)tmpArray[2];
tEnd_ = (size_t)tmpArray[3];
numBoxCoords_ = (int)tmpArray[4];
boxAngle_[0] = tmpArray[5];
boxAngle_[1] = tmpArray[6];
boxAngle_[2] = tmpArray[7];
writeType_ = (WriteType)tmpArray[8];
highPrecision_ = (bool)tmpArray[9];
if (append)
file_.SetupAppend( fname, debug_ );
else
file_.SetupWrite( fname, debug_ );
if (highPrecision_) outfmt_ = "%8.6f";
}
// For parallel output we will need to seek. Set up the buffer again with correct offsets.
// Figure out the size of the written title.
unsigned int titleSize = (unsigned int)Title().size() + 1; // +1 for newline
titleSize = std::min(81U, titleSize);
file_.SetupFrameBuffer( natom3_, 8, 10, headerSize_, titleSize );
file_.ResizeBuffer( numBoxCoords_ );
if (debug_>0)
rprintf("'%s'(Parallel): Each frame has %lu bytes.\n", file_.Filename().base(),
file_.FrameSize());
// TODO set file size
return 0;
}