/*! This function assigns a certain number of files to processors, such that
* each processor is exactly assigned to one file, and the number of cpus per
* file is as homogenous as possible. The number of files may at most be
* equal to the number of processors.
*/
void distribute_file(int nfiles, int firstfile, int firsttask, int lasttask, int *filenr, int *master,
int *last)
{
int ntask, filesleft, filesright, tasksleft, tasksright;
if(nfiles > 1)
{
ntask = lasttask - firsttask + 1;
filesleft = (((double) (ntask / 2)) / ntask) * nfiles;
if(filesleft <= 0)
filesleft = 1;
if(filesleft >= nfiles)
filesleft = nfiles - 1;
filesright = nfiles - filesleft;
tasksleft = ntask / 2;
tasksright = ntask - tasksleft;
distribute_file(filesleft, firstfile, firsttask, firsttask + tasksleft - 1, filenr, master, last);
distribute_file(filesright, firstfile + filesleft, firsttask + tasksleft, lasttask, filenr, master,
last);
}
else
{
if(ThisTask >= firsttask && ThisTask <= lasttask)
{
*filenr = firstfile;
*master = firsttask;
*last = lasttask;
}
}
}
/*! This function writes a snapshot of the particle distribution to one or
* several files using the selected file format. If NumFilesPerSnapshot>1,
* the snapshot is distributed onto several files, several of them can be
* written simultaneously (up to NumFilesWrittenInParallel). Each file
* contains data from a group of processors.
*/
void savepositions(int num)
{
double t0, t1;
char buf[500];
int i, j, *temp, n, filenr, gr, ngroups, masterTask, lastTask;
t0 = second();
if(ThisTask == 0)
printf("\nwriting snapshot file... \n");
#if defined(SFR) || defined(BLACK_HOLES)
rearrange_particle_sequence();
/* ensures that new tree will be constructed */
All.NumForcesSinceLastDomainDecomp = 1 + All.TreeDomainUpdateFrequency * All.TotNumPart;
#endif
if(NTask < All.NumFilesPerSnapshot)
{
if(ThisTask == 0)
printf("Fatal error.\nNumber of processors must be larger or equal than All.NumFilesPerSnapshot.\n");
endrun(0);
}
if(All.SnapFormat < 1 || All.SnapFormat > 3)
{
if(ThisTask == 0)
printf("Unsupported File-Format\n");
endrun(0);
}
#ifndef HAVE_HDF5
if(All.SnapFormat == 3)
{
if(ThisTask == 0)
printf("Code wasn't compiled with HDF5 support enabled!\n");
endrun(0);
}
#endif
/* determine global and local particle numbers */
for(n = 0; n < 6; n++)
n_type[n] = 0;
for(n = 0; n < NumPart; n++)
n_type[P[n].Type]++;
/* because ntot_type_all[] is of type `long long', we cannot do a simple
* MPI_Allreduce() to sum the total particle numbers
*/
temp = malloc(NTask * 6 * sizeof(int));
MPI_Allgather(n_type, 6, MPI_INT, temp, 6, MPI_INT, MPI_COMM_WORLD);
for(i = 0; i < 6; i++)
{
ntot_type_all[i] = 0;
for(j = 0; j < NTask; j++)
ntot_type_all[i] += temp[j * 6 + i];
}
free(temp);
/* assign processors to output files */
distribute_file(All.NumFilesPerSnapshot, 0, 0, NTask - 1, &filenr, &masterTask, &lastTask);
fill_Tab_IO_Labels();
if(All.NumFilesPerSnapshot > 1)
sprintf(buf, "%s%s_%03d.%d.g", All.OutputDir, All.SnapshotFileBase, num, filenr);
else
sprintf(buf, "%s%s_%03d.g", All.OutputDir, All.SnapshotFileBase, num);
ngroups = All.NumFilesPerSnapshot / All.NumFilesWrittenInParallel;
if((All.NumFilesPerSnapshot % All.NumFilesWrittenInParallel))
ngroups++;
for(gr = 0; gr < ngroups; gr++)
{
if((filenr / All.NumFilesWrittenInParallel) == gr) /* ok, it's this processor's turn */
write_file(buf, masterTask, lastTask);
MPI_Barrier(MPI_COMM_WORLD);
}
if(ThisTask == 0)
printf("done with snapshot.\n");
t1 = second();
All.CPU_Snapshot += timediff(t0, t1);
}
/*! This function reads initial conditions, in one of the three possible file
* formats currently supported by Gadget. Note: When a snapshot file is
* started from initial conditions (start-option 0), not all the information
* in the header is used, in particular, the STARTING TIME needs to be set in
* the parameterfile. Also, for gas particles, only the internal energy is
* read, the density and mean molecular weight will be recomputed by the
* code. When InitGasTemp>0 is given, the gas temperature will be initialzed
* to this value assuming a mean colecular weight either corresponding to
* complete neutrality, or full ionization.
*
* However, when the code is started with start-option 2, then all the this
* data in the snapshot files is preserved, i.e. this is also the way to
* resume a simulation from a snapshot file in case a regular restart file is
* not available.
*/
void read_ic(char *fname)
{
int i, num_files, rest_files, ngroups, gr, filenr, masterTask, lastTask, groupMaster;
double u_init;
char buf[500];
#ifndef ISOTHERM_EQS
double molecular_weight;
#endif
#ifdef SFR
double original_gas_mass, mass, masstot;
#endif
NumPart = 0;
N_gas = 0;
All.TotNumPart = 0;
num_files = find_files(fname);
rest_files = num_files;
fill_Tab_IO_Labels();
while(rest_files > NTask)
{
sprintf(buf, "%s.%d", fname, ThisTask + (rest_files - NTask));
if(All.ICFormat == 3)
sprintf(buf, "%s.%d.hdf5", fname, ThisTask + (rest_files - NTask));
ngroups = NTask / All.NumFilesWrittenInParallel;
if((NTask % All.NumFilesWrittenInParallel))
ngroups++;
groupMaster = (ThisTask / ngroups) * ngroups;
for(gr = 0; gr < ngroups; gr++)
{
if(ThisTask == (groupMaster + gr)) /* ok, it's this processor's turn */
read_file(buf, ThisTask, ThisTask);
MPI_Barrier(MPI_COMM_WORLD);
}
rest_files -= NTask;
}
if(rest_files > 0)
{
distribute_file(rest_files, 0, 0, NTask - 1, &filenr, &masterTask, &lastTask);
if(num_files > 1)
{
sprintf(buf, "%s.%d", fname, filenr);
if(All.ICFormat == 3)
sprintf(buf, "%s.%d.hdf5", fname, filenr);
}
else
{
sprintf(buf, "%s", fname);
if(All.ICFormat == 3)
sprintf(buf, "%s.hdf5", fname);
}
ngroups = rest_files / All.NumFilesWrittenInParallel;
if((rest_files % All.NumFilesWrittenInParallel))
ngroups++;
for(gr = 0; gr < ngroups; gr++)
{
if((filenr / All.NumFilesWrittenInParallel) == gr) /* ok, it's this processor's turn */
read_file(buf, masterTask, lastTask);
MPI_Barrier(MPI_COMM_WORLD);
}
}
/* this makes sure that masses are initialized in the case that the mass-block
is completely empty */
for(i = 0; i < NumPart; i++)
{
if(All.MassTable[P[i].Type] != 0)
P[i].Mass = All.MassTable[P[i].Type];
}
if(RestartFlag == 0)
{
#ifdef NEUTRINO_FLUID
/* Initial sound velocity = 3.6e-5 c (94.1 Omega_Nu * h^2)^-2 * (1+z)^2 */
u_init = (BOLTZMANN / PROTONMASS) * 10000.;
u_init *= All.UnitMass_in_g / All.UnitEnergy_in_cgs; /* unit conversion */
if (ThisTask == 0) printf("U_init 10000K : %f\n\n",u_init);
u_init = 0.2 * 2.02e-7 * (29979245800. / All.UnitVelocity_in_cm_per_s)
* (29979245800. / All.UnitVelocity_in_cm_per_s)
/ (94.1 * All.OmegaNeutrino * All.HubbleParam * All.HubbleParam)
/ (94.1 * All.OmegaNeutrino * All.HubbleParam * All.HubbleParam)
/ (All.Time * All.Time);
if (ThisTask == 0) printf("U_init Neutrinos : %f\n\n",u_init);
All.InitGasTemp = u_init;
for(i = 0; i < N_gas; i++)
{
SphP[i].Entropy = u_init;
/* Note: the coversion to entropy will be done in the function init(),
after the densities have been computed */
}
#else
if(All.InitGasTemp > 0)
{
u_init = (BOLTZMANN / PROTONMASS) * All.InitGasTemp;
u_init *= All.UnitMass_in_g / All.UnitEnergy_in_cgs; /* unit conversion */
#ifdef ISOTHERM_EQS
u_init *= 1.0;
#else
u_init *= (1.0 / GAMMA_MINUS1);
if(All.InitGasTemp > 1.0e4) /* assuming FULL ionization */
molecular_weight = 4 / (8 - 5 * (1 - HYDROGEN_MASSFRAC));
else /* assuming NEUTRAL GAS */
molecular_weight = 4 / (1 + 3 * HYDROGEN_MASSFRAC);
u_init /= molecular_weight;
#endif
for(i = 0; i < N_gas; i++)
{
if(SphP[i].Entropy == 0)
SphP[i].Entropy = u_init;
/* Note: the coversion to entropy will be done in the function init(),
after the densities have been computed */
}
}
#endif
}
for(i = 0; i < N_gas; i++)
SphP[i].Entropy = dmax(All.MinEgySpec, SphP[i].Entropy);
MPI_Barrier(MPI_COMM_WORLD);
if(ThisTask == 0)
{
printf("reading done.\n");
fflush(stdout);
}
if(ThisTask == 0)
{
printf("Total number of particles : %d%09d\n\n",
(int) (All.TotNumPart / 1000000000), (int) (All.TotNumPart % 1000000000));
fflush(stdout);
}
}
void read_ic(char *fname)
{
int i, num_files, rest_files, ngroups, gr, filenr, masterTask, lastTask, groupMaster;
double u_init, molecular_weight, dmax1, dmax2;
char buf[500];
CPU_Step[CPU_MISC] += measure_time();
#ifdef RESCALEVINI
if(ThisTask == 0 && RestartFlag == 0)
{
fprintf(stdout, "\nRescaling v_ini !\n\n");
fflush(stdout);
}
#endif
NumPart = 0;
N_gas = 0;
All.TotNumPart = 0;
num_files = find_files(fname);
#if defined(SAVE_HSML_IN_IC_ORDER) || defined(SUBFIND_RESHUFFLE_CATALOGUE)
NumPartPerFile = (long long *) mymalloc(num_files * sizeof(long long));
if(ThisTask == 0)
get_particle_numbers(fname, num_files);
MPI_Bcast(NumPartPerFile, num_files * sizeof(long long), MPI_BYTE, 0, MPI_COMM_WORLD);
#endif
rest_files = num_files;
while(rest_files > NTask)
{
sprintf(buf, "%s.%d", fname, ThisTask + (rest_files - NTask));
if(All.ICFormat == 3)
sprintf(buf, "%s.%d.hdf5", fname, ThisTask + (rest_files - NTask));
#if defined(SAVE_HSML_IN_IC_ORDER) || defined(SUBFIND_RESHUFFLE_CATALOGUE)
FileNr = ThisTask + (rest_files - NTask);
#endif
ngroups = NTask / All.NumFilesWrittenInParallel;
if((NTask % All.NumFilesWrittenInParallel))
ngroups++;
groupMaster = (ThisTask / ngroups) * ngroups;
for(gr = 0; gr < ngroups; gr++)
{
if(ThisTask == (groupMaster + gr)) /* ok, it's this processor's turn */
read_file(buf, ThisTask, ThisTask);
MPI_Barrier(MPI_COMM_WORLD);
}
rest_files -= NTask;
}
if(rest_files > 0)
{
distribute_file(rest_files, 0, 0, NTask - 1, &filenr, &masterTask, &lastTask);
if(num_files > 1)
{
sprintf(buf, "%s.%d", fname, filenr);
if(All.ICFormat == 3)
sprintf(buf, "%s.%d.hdf5", fname, filenr);
#if defined(SAVE_HSML_IN_IC_ORDER) || defined(SUBFIND_RESHUFFLE_CATALOGUE)
FileNr = filenr;
#endif
}
else
{
sprintf(buf, "%s", fname);
if(All.ICFormat == 3)
sprintf(buf, "%s.hdf5", fname);
#if defined(SAVE_HSML_IN_IC_ORDER) || defined(SUBFIND_RESHUFFLE_CATALOGUE)
FileNr = 0;
#endif
}
ngroups = rest_files / All.NumFilesWrittenInParallel;
if((rest_files % All.NumFilesWrittenInParallel))
ngroups++;
for(gr = 0; gr < ngroups; gr++)
{
if((filenr / All.NumFilesWrittenInParallel) == gr) /* ok, it's this processor's turn */
read_file(buf, masterTask, lastTask);
MPI_Barrier(MPI_COMM_WORLD);
}
}
#if defined(SUBFIND_RESHUFFLE_CATALOGUE)
subfind_reshuffle_free();
#endif
myfree_msg(CommBuffer, "CommBuffer");
if(header.flag_ic_info != FLAG_SECOND_ORDER_ICS)
{
/* this makes sure that masses are initialized in the case that the mass-block
is empty for this particle type */
for(i = 0; i < NumPart; i++)
{
if(All.MassTable[P[i].Type] != 0)
P[i].Mass = All.MassTable[P[i].Type];
}
}
#ifdef GENERATE_GAS_IN_ICS
int count, j;
double fac, d, a, b, rho;
if(RestartFlag == 0)
{
header.flag_entropy_instead_u = 0;
for(i = 0, count = 0; i < NumPart; i++)
if(P[i].Type == 1)
count++;
memmove(P + count, P, sizeof(struct particle_data) * NumPart);
NumPart += count;
N_gas += count;
if(N_gas > All.MaxPartSph)
{
printf("Task=%d ends up getting more SPH particles (%d) than allowed (%d)\n",
ThisTask, N_gas, All.MaxPartSph);
endrun(111);
}
fac = All.OmegaBaryon / All.Omega0;
rho = All.Omega0 * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G);
for(i = count, j = 0; i < NumPart; i++)
if(P[i].Type == 1)
{
P[j] = P[i];
d = pow(P[i].Mass / rho, 1.0 / 3);
a = 0.5 * All.OmegaBaryon / All.Omega0 * d;
b = 0.5 * (All.Omega0 - All.OmegaBaryon) / All.Omega0 * d;
P[j].Mass *= fac;
P[i].Mass *= (1 - fac);
P[j].Type = 0;
P[j].ID += 1000000000;
P[i].Pos[0] += a;
P[i].Pos[1] += a;
P[i].Pos[2] += a;
P[j].Pos[0] -= b;
P[j].Pos[1] -= b;
P[j].Pos[2] -= b;
j++;
}
All.MassTable[0] = fac * All.MassTable[1];
All.MassTable[1] *= (1 - fac);
}
#endif
#if defined(BLACK_HOLES) && defined(SWALLOWGAS)
if(RestartFlag == 0)
{
All.MassTable[5] = 0;
}
#endif
#ifdef SFR
if(RestartFlag == 0)
{
if(All.MassTable[4] == 0 && All.MassTable[0] > 0)
{
All.MassTable[0] = 0;
All.MassTable[4] = 0;
}
}
#endif
u_init = (1.0 / GAMMA_MINUS1) * (BOLTZMANN / PROTONMASS) * All.InitGasTemp;
u_init *= All.UnitMass_in_g / All.UnitEnergy_in_cgs; /* unit conversion */
if(All.InitGasTemp > 1.0e4) /* assuming FULL ionization */
molecular_weight = 4 / (8 - 5 * (1 - HYDROGEN_MASSFRAC));
else /* assuming NEUTRAL GAS */
molecular_weight = 4 / (1 + 3 * HYDROGEN_MASSFRAC);
u_init /= molecular_weight;
All.InitGasU = u_init;
if(RestartFlag == 0)
{
if(All.InitGasTemp > 0)
{
for(i = 0; i < N_gas; i++)
{
if(ThisTask == 0 && i == 0 && SphP[i].Entropy == 0)
printf("Initializing u from InitGasTemp !\n");
if(SphP[i].Entropy == 0)
SphP[i].Entropy = All.InitGasU;
/* Note: the coversion to entropy will be done in the function init(),
after the densities have been computed */
}
}
}
for(i = 0; i < N_gas; i++)
SphP[i].Entropy = DMAX(All.MinEgySpec, SphP[i].Entropy);
#ifdef EOS_DEGENERATE
for(i = 0; i < N_gas; i++)
SphP[i].u = 0;
#endif
MPI_Barrier(MPI_COMM_WORLD);
if(ThisTask == 0)
{
printf("reading done.\n");
fflush(stdout);
}
if(ThisTask == 0)
{
printf("Total number of particles : %d%09d\n\n",
(int) (All.TotNumPart / 1000000000), (int) (All.TotNumPart % 1000000000));
fflush(stdout);
}
CPU_Step[CPU_SNAPSHOT] += measure_time();
}