void StatisticsSampler::saveToFile(System &system, ofstream &file)
{
// Convert all energy per atom, use eV
file << setw(15) << UnitConverter::timeToSI(system.steps()) << " "
<< setw(15) << UnitConverter::timeToSI(system.time()) << " "
<< setw(15) << UnitConverter::temperatureToSI(temperature()) << " "
<< setw(15) << UnitConverter::energyToEv(kineticEnergy()/system.atoms().size()) << " "
<< setw(15) << UnitConverter::energyToEv(potentialEnergy()/system.atoms().size()) << " "
<< setw(15) << UnitConverter::energyToEv(totalEnergy()/system.atoms().size()) << " "
<< setw(15) << diffusionConstant() << " "
<< setw(15) << m_rSquared << std::endl;
}
bool test_potential(const potential::BasePotential& potential,
const coord::PosVelT<coordSysT>& initial_conditions,
const double total_time, const double timestep)
{
std::cout << potential.name()<<" "<<coordSysT::name()<<" ("<<initial_conditions<<")\n";
double ic[6];
initial_conditions.unpack_to(ic);
std::vector<coord::PosVelT<coordSysT> > traj;
int numsteps = orbit::integrate(potential, initial_conditions, total_time, timestep, traj, integr_eps);
double avgH=0, avgLz=0, dispH=0, dispLz=0;
for(size_t i=0; i<traj.size(); i++) {
double H =totalEnergy(potential, traj[i]);
double Lz=coord::Lz(traj[i]);
avgH +=H; dispH +=pow_2(H);
avgLz+=Lz; dispLz+=pow_2(Lz);
if(output) {
double xv[6];
coord::toPosVelCar(traj[i]).unpack_to(xv);
std::cout << i*timestep<<" " <<xv[0]<<" "<<xv[1]<<" "<<xv[2]<<" "<<
xv[3]<<" "<<xv[4]<<" "<<xv[5]<<" "<<H<<" "<<Lz<<" "<<potential.density(traj[i])<<"\n";
}
}
avgH/=traj.size();
avgLz/=traj.size();
dispH/=traj.size();
dispLz/=traj.size();
dispH= sqrt(std::max<double>(0, dispH -pow_2(avgH)));
dispLz=sqrt(std::max<double>(0, dispLz-pow_2(avgLz)));
bool completed = traj.size()>0.99*total_time/timestep;
bool ok = dispH<eps && (dispLz<eps || !isAxisymmetric(potential));
if(completed)
std::cout <<numsteps<<" steps, ";
else if(numsteps>0) {
std::cout <<"\033[1;33mCRASHED\033[0m after "<<numsteps<<" steps, ";
// this may naturally happen in the degenerate case when an orbit with zero angular momentum
// approaches the origin -- not all combinations of potential and coordinate system
// can cope with this. If that happens for a non-degenerate case, this is an error.
if(avgLz!=0) ok=false;
}
else {
// this may happen for an orbit started at r==0 in some potentials where the force is singular,
// otherwise it's an error
if(toPosSph(initial_conditions).r != 0) {
std::cout <<"\033[1;31mFAILED\033[0m, ";
ok = false;
}
}
std::cout << "E=" <<avgH <<" +- "<<dispH<<", Lz="<<avgLz<<" +- "<<dispLz<<
(ok? "" : " \033[1;31m**\033[0m") << "\n";
return ok;
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
int commsize, commrank;
MPI_Comm_size(MPI_COMM_WORLD, &commsize);
MPI_Comm_rank(MPI_COMM_WORLD, &commrank);
const bool isRoot = commrank == 0;
const bool isLast = commrank == commsize - 1;
if (isRoot)
std::cout << "Initialization...\n" << std::endl;
RuntimeConfiguration conf(argc, argv);
// Compute my start and end time
const double timeStart = conf.timeSliceSize() * commrank;
const double timeEnd = conf.timeSliceSize() * (commrank + 1);
if (isRoot)
std::cout << "Running with:\n"
<< " - initial diffusion coefficient: " << conf.nu0() << "\n"
<< " - frequence of diffusion coefficient: " << conf.nufreq() << "\n"
<< " - advection velocity in x: " << conf.cx() << "\n"
<< " - advection velocity in y: " << conf.cy() << "\n"
<< " - advection velocity in z: " << conf.cz() << "\n"
<< " - spatial discretization step: " << conf.dx() << "\n"
<< " - endtime: " << conf.endTime() << "\n"
<< " - number of time slices: " << conf.timeSlices() << "\n"
<< " - time slice size: " << conf.timeSliceSize() << "\n"
<< " - CFL fine: " << conf.cflFine() << "\n"
<< " - CFL coarse: " << conf.cflCoarse() << "\n"
<< " - timestep size fine: " << conf.dtFine() << "\n"
<< " - timestep size coarse: " << conf.dtCoarse() << "\n"
<< " - timesteps per slice fine propagator: " << conf.timeStepsFinePerTimeSlice() << "\n"
<< " - timesteps per slice coarse propagator: " << conf.timeStepsCoarsePerTimeSlice() << "\n"
<< " - parareal iterations: " << conf.kmax() << "\n"
<< " - asynchronous communications: " << (conf.async() ? "Enabled" : "Disabled") << "\n"
<< " - intermediate fields in mat files: " << (conf.mat() ? "Yes" : "No") << "\n"
<< std::endl;
// Calculation domain and boundaries
IJKSize domain; domain.Init(conf.gridSize(), conf.gridSize(), conf.gridSize());
KBoundary kboundary; kboundary.Init(-convectionBoundaryLines, convectionBoundaryLines);
// Initialize fields
ConvectionField q, qinitial;
q.Init("q", domain, kboundary);
qinitial.Init("qinitial", domain, kboundary);
Convection convection(conf.gridSize(), conf.gridSize(), conf.gridSize(), conf.dx(), conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz());
// Initialize parareal
Parareal<Convection, ConvectionField> parareal(convection, qinitial, q, timeStart, conf, MPI_COMM_WORLD);
if (conf.mode() == ModeCompare)
{
// Measure time required by convection
const int tauSamples = 4;
double tauF = MPI_Wtime();
convection.DoRK4(qinitial, qinitial, 0., conf.dtFine(), tauSamples*conf.timeStepsFinePerTimeSlice());
SynchronizeCUDA();
tauF = MPI_Wtime() - tauF;
double tauG = MPI_Wtime();
convection.DoEuler(qinitial, qinitial, 0., conf.dtCoarse(), tauSamples*conf.timeStepsCoarsePerTimeSlice());
SynchronizeCUDA();
tauG = MPI_Wtime() - tauG;
const double tauRatio = tauG / tauF;
const double Nit_Np = static_cast<double>(conf.kmax()) / commsize;
const double maxSpeedup = 1. / (tauRatio * (1. + Nit_Np) + Nit_Np);
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
MPI_Barrier(MPI_COMM_WORLD);
double eserial = MPI_Wtime();
parareal.DoSerial();
eserial = MPI_Wtime() - eserial;
// Save reference
ConvectionField qreference = q;
SynchronizeHost(qreference);
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
MPI_Barrier(MPI_COMM_WORLD);
double eparallel = MPI_Wtime();
parareal.DoParallel();
eparallel = MPI_Wtime() - eparallel;
// Output
MPI_Barrier(MPI_COMM_WORLD);
if (isLast)
{
double e = computeErrorReference(q, qreference);
std::cout << "\n"
<< "Serial run time: " << eserial << "\n"
<< "Parallel run time: " << eparallel << "\n"
<< "Speedup: " << eserial / eparallel << "\n"
<< "Maximal speedup: " << maxSpeedup << "\n"
<< "Error at end: " << e << "\n"
<< std::endl;
MatFile matfile("result.mat");
matfile.addField("q", q);
matfile.addField("qreference", qreference);
}
}
else if (conf.mode() == ModeSerial)
{
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
double e = MPI_Wtime();
double energyStart = energy();
double deviceEnergyStart = deviceEnergy();
parareal.DoSerial();
e = MPI_Wtime() - e;
double energyEnd = energy();
double deviceEnergyEnd = deviceEnergy();
const double totDevice = totalEnergy(deviceEnergyStart, deviceEnergyEnd);
const double totNode = totalEnergy(energyStart, energyEnd) - totDevice;
const double totNetwork = e * powerNetwork;
const double totBlower = e * powerBlower;
const double totEnergy = totNode + totDevice + totNetwork + totBlower;
// Output
MPI_Barrier(MPI_COMM_WORLD);
if (isLast)
{
std::cout << "\n" << "Serial run time: " << e << "\n";
std::printf("Node energy : %8f J (%8.3e W/node)\n", totNode , totNode/e);
std::printf("Device energy : %8f J (%8.3e W/node)\n", totDevice , totDevice/e);
std::printf("Network energy: %8f J (%8.3e W/node)\n", totNetwork, totNetwork/e);
std::printf("Blower energy : %8f J (%8.3e W/node)\n", totBlower , totBlower/e);
std::printf("Total energy : %8f J (%8.3e W/node)\n", totEnergy , totEnergy/e);
std::cout << std::endl;
}
}
else if (conf.mode() == ModeParallel)
{
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
parareal.DoSerial();
std::cout << " -- The serial computation is done\n";
ConvectionField qreference = q;
// Run parallel
MPI_Barrier(MPI_COMM_WORLD);
double e = MPI_Wtime();
double energyStart = energy();
double deviceEnergyStart = deviceEnergy();
parareal.DoParallel();
MPI_Barrier(MPI_COMM_WORLD);
e = MPI_Wtime() - e;
std::cout << " -- The parallel computation is done\n";
double energyEnd = energy();
double deviceEnergyEnd = deviceEnergy();
const double totDevice = totalEnergy(deviceEnergyStart, deviceEnergyEnd, MPI_COMM_WORLD);
const double totNode = totalEnergy(energyStart, energyEnd, MPI_COMM_WORLD) - totDevice;
const double totNetwork = e * powerNetwork * commsize;
const double totBlower = e * powerBlower * commsize;
const double totEnergy = totNode + totDevice + totNetwork + totBlower;
// Compute error
double error = computeErrorReference(q, qreference);
// Output
MPI_Barrier(MPI_COMM_WORLD);
if (isLast)
{
const double fac = 1./e/commsize;
std::cout << std::endl;
std::printf("Parallel run time: %f s\n", e);
std::printf("Node energy : %8f J (%8.3e W/node)\n", totNode , fac*totNode);
std::printf("Device energy : %8f J (%8.3e W/node)\n", totDevice , fac*totDevice);
std::printf("Network energy: %8f J (%8.3e W/node)\n", totNetwork, fac*totNetwork);
std::printf("Blower energy : %8f J (%8.3e W/node)\n", totBlower , fac*totBlower);
std::printf("Total energy : %8f J (%8.3e W/node)\n", totEnergy , fac*totEnergy);
std::printf("Error of parareal: %.4e\n", error);
std::cout << std::endl;
}
}
else if (conf.mode() == ModeTiming)
{
// Fill initial solution
SynchronizeHost(qinitial);
fillQ(qinitial, conf.nu0(), conf.nufreq(), conf.cx(), conf.cy(), conf.cz(), 0., 0., 1., 0., 1., 0., 1.);
SynchronizeDevice(qinitial);
// Run serial
std::vector<double> times;
MPI_Barrier(MPI_COMM_WORLD);
parareal.DoTimedParallel(times);
// Gather on root
const int s = times.size();
std::vector<double> timesGlobal;
timesGlobal.resize(s * commsize);
MPI_Gather(×[0], s, MPI_DOUBLE, ×Global[0], s, MPI_DOUBLE, 0, MPI_COMM_WORLD);
// Output
if (isRoot)
{
std::cout << "\nTimes:\n";
for(int i = 0; i < s; ++i)
{
for(int p = 0; p < commsize; ++p)
{
std::cout << std::scientific << std::setprecision(6) << timesGlobal[p*s + i] << " ";
}
std::cout << "\n";
}
}
}
// Finalize
MPI_Finalize();
return 0;
}
void globalLeapFrogAccuracy(){
double h = 1;
int n = 3;
double t;
FILE * fp;
particle * particles = malloc(n * sizeof(particle));
double energy;
double currentEnergy;
double energyChange;
//lin3 global
fp = fopen("lin3global.dat","w");
initialize_linearx(particles, n);
energy = totalEnergy(particles, n);
for (h = 0.1;h>1E-7;h*=0.5){
initialize_linearx(particles, n);
for(t=0;t<0.1;t+=h){
//printf("leapfrogging at time t = %lf", t);
leapFrogStep(particles, n, h);
}
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
//equilateral global
fp = fopen("equilateralglobal.dat","w");
initialize_equilateral(particles, n);
energy = totalEnergy(particles, n);
for (h = 0.1;h>1E-7;h*=0.5){
initialize_equilateral(particles, n);
for(t=0;t<0.1;t+=h){
//printf("leapfrogging at time t = %lf", t);
leapFrogStep(particles, n, h);
}
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
printf("10 particles may take some time\n");
//linear 10 global
free(particles);
n = 10;
particles = malloc(n * sizeof(particle));
fp = fopen("lin10global.dat","w");
initialize_linearx(particles, n);
energy = totalEnergy(particles, n);
for (h = 0.1;h>1E-6;h*=0.5){
initialize_linearx(particles, n);
for(t=0;t<0.1;t+=h){
//printf("leapfrogging at time t = %lf", t);
leapFrogStep(particles, n, h);
}
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
//random 10 global
free(particles);
n = 10;
particles = malloc(n * sizeof(particle));
fp = fopen("random10global.dat","w");
initialize_2(particles, n);
energy = totalEnergy(particles, n);
for (h = 0.1;h>1E-6;h*=0.5){
initialize_2(particles, n);
for(t=0;t<0.1;t+=h){
//printf("leapfrogging at time t = %lf", t);
leapFrogStep(particles, n, h);
}
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
printf("20 particles may take noticeably more time\n");
//random 20 global
free(particles);
n = 20;
particles = malloc(n * sizeof(particle));
fp = fopen("random20global.dat","w");
initialize_3(particles, n);
//fprintParticles(stdout, particles, n);
energy = totalEnergy(particles, n);
for (h = 0.1;h>1E-6;h*=0.5){
initialize_3(particles, n);
for(t=0;t<0.1;t+=h){
//printf("leapfrogging at time t = %lf", t);
leapFrogStep(particles, n, h);
}
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
printf("40 particles may take noticeably more time\n");
//random 40 global
free(particles);
n = 40;
particles = malloc(n * sizeof(particle));
fp = fopen("random40global.dat","w");
initialize_3(particles, n);
//fprintParticles(stdout, particles, n);
energy = totalEnergy(particles, n);
for (h = 0.1;h>1E-6;h*=0.5){
initialize_3(particles, n);
for(t=0;t<0.1;t+=h){
//printf("leapfrogging at time t = %lf", t);
leapFrogStep(particles, n, h);
}
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
free(particles);
}
void localLeapFrogAccuracy(){
double h = 1;
int n = 3;
particle * particles = malloc(n * sizeof(particle));
FILE * fp;
double currentEnergy;
double energyChange;
double energy;
fp = fopen("lin3local1.dat","w");
initialize_linearx(particles, n);
energy = totalEnergy(particles, n);
printf("initial energy = %g\n", energy);
leapFrogStep(particles, n, 0.01);
currentEnergy = totalEnergy(particles, n);
printf("current energy = %g\n", currentEnergy);
fprintParticles(stdout, particles, n);
for (h = 1;h>1E-8;h/=2){
initialize_linearx(particles, n);
leapFrogStep(particles, n, h);
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
//
fp = fopen("lin3local10.dat","w");
initialize_linearx(particles, n);
energy = totalEnergy(particles, n);
for (h = 1;h>1E-8;h/=2){
initialize_linearx(particles, n);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
// 10 particles
free(particles);
n = 10;
particles = malloc(n * sizeof(particle));
fp = fopen("lin10local1.dat","w");
initialize_linearx(particles, n);
energy = totalEnergy(particles, n);
for (h = 1;h>1E-8;h/=2){
initialize_linearx(particles, n);
leapFrogStep(particles, n, h);
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
//
fp = fopen("lin10local10.dat","w");
initialize_linearx(particles, n);
energy = totalEnergy(particles, n);
for (h = 1;h>1E-8;h/=2){
initialize_linearx(particles, n);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
leapFrogStep(particles, n, h);
currentEnergy = totalEnergy(particles, n);
energyChange = fabs(currentEnergy - energy);
fprintf(fp, "%g\t%g\n", h, energyChange);
}
fclose(fp);
free(particles);
}
