/*! This function finds the next synchronization point of the system (i.e. the
* earliest point of time any of the particles needs a force computation),
* and drifts the system to this point of time. If the system drifts over
* the desired time of a snapshot file, the function will drift to this
* moment, generate an output, and then resume the drift.
*/
void find_next_sync_point_and_drift(void)
{
int n, flag, *temp, i, nskip=20;
long long int min_glob, min;
double timeold;
double t0, t1;
int task_max, loc_max, tot_loc_max, list_loc_max[NTask], n_check;
double hubble_a, dt_raytrace=0, nh_local, nh_max, tot_nh_max, mass_max, tot_mass_max, ray_dist2, list_nh_max[NTask], list_mass_max[NTask];
#ifdef RAYTRACE_TG
double nu_min_H = 3.3e15;
double nu_min_He = 1.32e16;
double c_s = 16.5967; //soundspeed of 20,000K gas in km/s
#endif
if (All.ComovingIntegrationOn) { /* comoving variables */
hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time)
+ All.OmegaLambda;
hubble_a = All.Hubble * All.HubbleParam * sqrt(hubble_a);
}
else hubble_a = 1.0;
t0 = second();
timeold = All.Time;
/*SINK - must skip any accreted particles at the beginning of the SPH particle list*/
for(i = 0; i < NumPart; i++)
{
if(P[i].Type == 5 || P[i].ID >= 0)
break;
}
if(i == NumPart)
min = INT_MAX;
else
{
for(n = i+1, min = P[i].Ti_endstep; n < NumPart; n++)
{
if(P[n].Type == 0 && P[n].ID < 0) /*SINK*/
continue;
if(min > P[n].Ti_endstep)
min = P[n].Ti_endstep;
}
}
MPI_Allreduce(&min, &min_glob, 1, MPI_LONG, MPI_MIN, MPI_COMM_WORLD);
/* We check whether this is a full step where all particles are synchronized */
flag = 1;
for(n = 0; n < NumPart; n++)
{
if(P[n].Type == 0 && P[n].ID < 0) /*SINK*/
continue;
if(P[n].Ti_endstep > min_glob)
flag = 0;
}
MPI_Allreduce(&flag, &Flag_FullStep, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
#ifdef PMGRID
if(min_glob >= All.PM_Ti_endstep)
{
min_glob = All.PM_Ti_endstep;
Flag_FullStep = 1;
}
#endif
/* Determine 'NumForceUpdate', i.e. the number of particles on this processor that are going to be active */
for(n = 0, NumForceUpdate = 0; n < NumPart; n++)
{
if(P[n].Type == 0 && P[n].ID < 0) /*SINK*/
continue;
if(P[n].Ti_endstep == min_glob)
#ifdef SELECTIVE_NO_GRAVITY
if(!((1 << P[n].Type) & (SELECTIVE_NO_GRAVITY)))
#endif
NumForceUpdate++;
}
/* note: NumForcesSinceLastDomainDecomp has type "long long" */
temp = malloc(NTask * sizeof(int));
MPI_Allgather(&NumForceUpdate, 1, MPI_INT, temp, 1, MPI_INT, MPI_COMM_WORLD);
for(n = 0; n < NTask; n++)
All.NumForcesSinceLastDomainDecomp += temp[n];
free(temp);
t1 = second();
All.CPU_Predict += timediff(t0, t1);
tot_nh_max = nh_max = nh_local = tot_mass_max = mass_max = tot_loc_max = loc_max = task_max = 0;
for(n = 0; n < NTask; n++)
list_nh_max[n] = list_mass_max[n] = list_loc_max[n] = 0;
for(i = 0; i < N_gas; i++)
if(P[i].ID > 0 /*&& P[i].Mass / All.HubbleParam * 1.0e10 < All.RefinementMass*/)
{
nh_local = SphP[i].Density*All.UnitDensity_in_cgs*All.HubbleParam*All.HubbleParam/All.Time/All.Time/All.Time*HYDROGEN_MASSFRAC/PROTONMASS;
//if(nh_local > nh_max)
if(P[i].ID == 2931027)
{
nh_max = nh_local;
mass_max = P[i].Mass;
loc_max = i;
}
}
MPI_Allgather(&nh_max, 1, MPI_DOUBLE, &list_nh_max, 1, MPI_DOUBLE, MPI_COMM_WORLD);
MPI_Allgather(&mass_max, 1, MPI_DOUBLE, &list_mass_max, 1, MPI_DOUBLE, MPI_COMM_WORLD);
MPI_Allgather(&loc_max, 1, MPI_INT, &list_loc_max, 1, MPI_INT, MPI_COMM_WORLD);
for(n = 0; n < NTask; n++)
if(list_nh_max[n] > tot_nh_max)
{
tot_nh_max = list_nh_max[n];
tot_mass_max = list_mass_max[n];
tot_loc_max = list_loc_max[n];
task_max = n;
}
dt_raytrace = fmax(fmax((min_glob - All.Time_last) * All.Timebase_interval, All.MinSizeTimestep), All.Timebase_interval) / hubble_a * All.UnitTime_in_s;
All.t_s = All.t_s + dt_raytrace; //time in seconds
All.r_s = All.x_s * c_s * All.t_s / 3.08568025e16 / All.Time * All.HubbleParam; //shock radius in km (converted to comoving kpc)
All.n_s = All.alpha0/(4.0*3.14159*6.67e-8*pow(All.t_s,2))*HYDROGEN_MASSFRAC/PROTONMASS;
All.n_sink = All.alpha0/(4.0*3.14159*6.67e-8*pow((All.tacc - 300.*3.e7),2))*HYDROGEN_MASSFRAC/PROTONMASS;
//All.n_sink = 1.e-1;
All.Time_last = min_glob;
#ifdef RAYTRACE_TG
if(ThisTask == task_max)
ray_dist2 = (P[tot_loc_max].Pos[0] - All.BoxSize / 2.0) * (P[tot_loc_max].Pos[0] - All.BoxSize / 2.0) + (P[tot_loc_max].Pos[1] - All.BoxSize / 2.0) * (P[tot_loc_max].Pos[1] - All.BoxSize / 2.0) + (P[tot_loc_max].Pos[2] - All.BoxSize / 2.0) * (P[tot_loc_max].Pos[2] - All.BoxSize / 2.0);
MPI_Bcast(&ray_dist2, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
if(ThisTask == task_max)
{
if(All.NumCurrentTiStep % 100 == 0)
printf("Densest particle (ID %d) with nh = %g at x = %g, y = %g, z = %g and mass %g\n", P[tot_loc_max].ID, tot_nh_max, P[tot_loc_max].Pos[0], P[tot_loc_max].Pos[1], P[tot_loc_max].Pos[2], P[tot_loc_max].Mass / All.HubbleParam * 1.0e10);
for(n = 0; n < N_gas; n++)
if(P[n].ID == P[tot_loc_max].ID) //center ray at most dense particle
{
All.star_mass = P[n].Mass/All.HubbleParam/1.e-10; //sink mass in solar masses
printf("star_mass =%lg\n", All.star_mass);
//All.star_mass = 1.e-5;
//All.star_mass = 0;
//All.star_mass = 2.e1;
}
n_check = 140740;
if(All.NumCurrentTiStep == 0)
{
//All.mdot= 0.095*pow(All.star_mass, -0.814);
All.numtot = 155639;
All.alpha = alpha_calc(n_check);
All.mdot = mdot_calc(All.numtot);
//All.mdot = 1.e-7;
//All.mdot = 1.e6;
}
All.flag_sink = 0;
if(All.NumCurrentTiStep == 10 || All.NumCurrentTiStep == 1000 || All.t_s - All.t_s0 > 3.e7)
{
All.alpha = alpha_calc(n_check);
All.mdot = mdot_calc(All.numtot);
//All.mdot= 1.e-7;
//All.mdot= 1.e6;
All.numtot = All.numtot + All.sink_number_global;
All.t_s0 = All.t_s;
All.flag_sink = 1;
printf("All.numtot = %d, All.t_s0_sink = %lg, All.t_s0 = %lg\n", All.numtot, All.t_s0_sink, All.t_s0);
}
if(All.NumCurrentTiStep < 1)
All.lum_tot = lum_calc(1, All.star_mass, All.mdot, nu_min_H, 1.e-5);
if(All.NumCurrentTiStep % 100 == 0)
printf("run lum_tot = %lg, Teff = %lg, mdot = %lg\n", All.lum_tot, All.Teff, All.mdot);
if(All.ray_flag_sun == 3)
{
for(i=0; i<=6; i++)
{
All.heat_ion[i] = heat_ion_rates(i, All.lum_tot, All.Teff);
COOLR.heat_ion[i] = All.heat_ion[i];
if(All.NumCurrentTiStep % 10000 == 0)
printf("heat_ion %d = %lg\n", i, COOLR.heat_ion[i]);
}
}
}
MPI_Bcast(&All.star_mass, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.star_rad, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.alpha, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.numtot, 1, MPI_INT, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.mdot, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.t_s0, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.flag_sink, 1, MPI_INT, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.tacc, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&COOLR.heat_ion, 7, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
//make sure ray doesn't go outside of box (?)
//ARS adding condition that the LW radiation actually needs to be significant before computation time will be spent on the ray-tracing)
if(tot_nh_max >= 0.99*All.ray_crit_dens && ray.flag_continue == 0 && ray_dist2 < All.ray_r_max_sink * All.ray_r_max_sink)
{
ray.flag_start = 1;
if(ThisTask == task_max)
All.ray_center_ID = P[tot_loc_max].ID;
MPI_Bcast(&All.ray_center_ID, 1, MPI_INT, task_max, MPI_COMM_WORLD);
if(ThisTask == task_max && All.NumCurrentTiStep % 1000 == 0)
printf("Found starp (ID %d) at x = %g, y = %g, z = %g with mass %g\n", P[tot_loc_max].ID, P[tot_loc_max].Pos[0], P[tot_loc_max].Pos[1], P[tot_loc_max].Pos[2], P[tot_loc_max].Mass / All.HubbleParam * 1.0e10);
}
// ARS asks why we cannot have a SINK be a star particle (starp)?
if(tot_nh_max > 0.99*All.SinkCriticalDens && ray_dist2 < All.ray_r_max_sink * All.ray_r_max_sink)
{
if(ThisTask == task_max && All.NumCurrentTiStep % 100 == 0)
printf("Problem! A sink instead of a starp will form (ID %d) at x = %g, y = %g, z = %g with mass %g! Aborting...\n", P[tot_loc_max].ID, P[tot_loc_max].Pos[0], P[tot_loc_max].Pos[1], P[tot_loc_max].Pos[2], P[tot_loc_max].Mass / All.HubbleParam * 1.0e10);
//exit(0);
}
if(All.lum_tot < 1.e36 || All.star_mass < 1.0)
{
nskip = 100;
if(ThisTask == 0 && All.NumCurrentTiStep % 100 == 0)
printf("Let's not ray trace quite so often\n");
}
if(ray.flag_start == 1 && ray.flag_continue == 0 || (ray.flag_start == 1 && All.NumCurrentTiStep % nskip == 0) || (ray.flag_start == 1 && All.t_s - All.t_s0_acc > 3.e5) || All.NumCurrentTiStep < 2) //ARS asks: Where should the parentheses go?
{
All.t_s0_acc = All.t_s;
if(ThisTask == 0)
printf("Imma gonna ray trace so there.\n");
if(ray.flag_start == 1 && ray.flag_continue == 0)
{
All.Time_last_raytrace = All.Time;
dt_raytrace = fmax(All.Timebase_interval, All.MinSizeTimestep) / hubble_a * All.UnitTime_in_s;
}
else
dt_raytrace = fmax(fmax((min_glob - All.Time_last_raytrace) * All.Timebase_interval, All.MinSizeTimestep), All.Timebase_interval) / hubble_a * All.UnitTime_in_s;
if(ThisTask == task_max)
{
ray.Q_H_ion = lum_calc(0, All.star_mass, All.mdot, nu_min_H, dt_raytrace);
ray.Q_He_ion = lum_calc(0, All.star_mass, All.mdot, nu_min_He, dt_raytrace);
All.Q_LW = lum_calc(4, All.star_mass, All.mdot, nu_min_H, dt_raytrace);
All.lum_tot = lum_calc(1, All.star_mass, All.mdot, nu_min_H, dt_raytrace);
}
MPI_Bcast(&ray.Q_H_ion, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&ray.Q_He_ion, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.lum_tot, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
MPI_Bcast(&All.Teff, 1, MPI_DOUBLE, task_max, MPI_COMM_WORLD);
All.r_s = All.x_s * c_s * All.t_s / 3.08568025e16 / All.Time * All.HubbleParam; //shock radius in km (converted to comoving kpc)
All.n_s = All.alpha0/(4.0*3.14159*6.67e-8*pow(All.t_s,2))*HYDROGEN_MASSFRAC/PROTONMASS;
All.n_sink = All.alpha0/(4.0*3.14159*6.67e-8*pow((All.tacc - 300.*3.e7),2))*HYDROGEN_MASSFRAC/PROTONMASS;
All.n_hii = All.alpha0/(4.0*3.14159*6.67e-8*pow((All.tacc - 1600.*3.e7),2))*HYDROGEN_MASSFRAC/PROTONMASS;
if(All.n_hii < 2.35e6) All.n_hii = 2.35e6;
//All.n_hii = 1.e3;
if(ThisTask == 0)
printf("t_s = %lg, tacc = %lg, r_s = %lg, n_s = %lg n_sink = %lg, n_hii = %lg\n", All.t_s, All.tacc, All.r_s, All.n_s, All.n_sink, All.n_hii);
//if(All.lum_tot > 0.0)
raytrace_TG(dt_raytrace);
All.Time_last_raytrace = min_glob;
}
#endif
while(min_glob >= All.Ti_nextoutput && All.Ti_nextoutput >= 0)
{
#ifdef CHEMCOOL
All.NeedAbundancesForOutput = 1;
#endif
move_particles(All.Ti_Current, All.Ti_nextoutput);
All.Ti_Current = All.Ti_nextoutput;
if(All.ComovingIntegrationOn)
All.Time = All.TimeBegin * exp(All.Ti_Current * All.Timebase_interval);
else
All.Time = All.TimeBegin + All.Ti_Current * All.Timebase_interval;
#ifdef OUTPUTPOTENTIAL
All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
domain_Decomposition();
compute_potential();
#endif
if(All.NumCurrentTiStep > 20)
savepositions(All.SnapshotFileCount++); /* write snapshot file */
#ifdef CHEMCOOL
All.NeedAbundancesForOutput = 0;
#endif
All.Ti_nextoutput = find_next_outputtime(All.Ti_nextoutput + 1);
#ifdef CHEMCOOL
All.Ti_nextnextoutput = find_next_outputtime(All.Ti_nextoutput + 1);
#endif
}
move_particles(All.Ti_Current, min_glob);
All.Ti_Current = min_glob;
if(All.ComovingIntegrationOn)
All.Time = All.TimeBegin * exp(All.Ti_Current * All.Timebase_interval);
else
All.Time = All.TimeBegin + All.Ti_Current * All.Timebase_interval;
All.TimeStep = All.Time - timeold;
}
/*! This function performs the initial set-up of the simulation. First, the
* parameterfile is set, then routines for setting units, reading
* ICs/restart-files are called, auxialiary memory is allocated, etc.
*/
void begrun(void)
{
struct global_data_all_processes all;
if(ThisTask == 0)
{
printf("\nThis is Gadget, version `%s'.\n", GADGETVERSION);
printf("\nRunning on %d processors.\n", NTask);
}
read_parameter_file(ParameterFile); /* ... read in parameters for this run */
allocate_commbuffers(); /* ... allocate buffer-memory for particle
exchange during force computation */
set_units();
#if defined(PERIODIC) && (!defined(PMGRID) || defined(FORCETEST))
ewald_init();
#endif
open_outputfiles();
random_generator = gsl_rng_alloc(gsl_rng_ranlxd1);
gsl_rng_set(random_generator, 42); /* start-up seed */
#ifdef PMGRID
long_range_init();
#endif
All.TimeLastRestartFile = CPUThisRun;
if(RestartFlag == 0 || RestartFlag == 2)
{
set_random_numbers();
init(); /* ... read in initial model */
}
else
{
all = All; /* save global variables. (will be read from restart file) */
restart(RestartFlag); /* ... read restart file. Note: This also resets
all variables in the struct `All'.
However, during the run, some variables in the parameter
file are allowed to be changed, if desired. These need to
copied in the way below.
Note: All.PartAllocFactor is treated in restart() separately.
*/
All.MinSizeTimestep = all.MinSizeTimestep;
All.MaxSizeTimestep = all.MaxSizeTimestep;
All.BufferSize = all.BufferSize;
All.BunchSizeForce = all.BunchSizeForce;
All.BunchSizeDensity = all.BunchSizeDensity;
All.BunchSizeHydro = all.BunchSizeHydro;
All.BunchSizeDomain = all.BunchSizeDomain;
All.TimeLimitCPU = all.TimeLimitCPU;
All.ResubmitOn = all.ResubmitOn;
All.TimeBetSnapshot = all.TimeBetSnapshot;
All.TimeBetStatistics = all.TimeBetStatistics;
All.CpuTimeBetRestartFile = all.CpuTimeBetRestartFile;
All.ErrTolIntAccuracy = all.ErrTolIntAccuracy;
All.MaxRMSDisplacementFac = all.MaxRMSDisplacementFac;
All.ErrTolForceAcc = all.ErrTolForceAcc;
All.TypeOfTimestepCriterion = all.TypeOfTimestepCriterion;
All.TypeOfOpeningCriterion = all.TypeOfOpeningCriterion;
All.NumFilesWrittenInParallel = all.NumFilesWrittenInParallel;
All.TreeDomainUpdateFrequency = all.TreeDomainUpdateFrequency;
All.SnapFormat = all.SnapFormat;
All.NumFilesPerSnapshot = all.NumFilesPerSnapshot;
All.MaxNumNgbDeviation = all.MaxNumNgbDeviation;
All.ArtBulkViscConst = all.ArtBulkViscConst;
All.OutputListOn = all.OutputListOn;
All.CourantFac = all.CourantFac;
All.OutputListLength = all.OutputListLength;
memcpy(All.OutputListTimes, all.OutputListTimes, sizeof(double) * All.OutputListLength);
strcpy(All.ResubmitCommand, all.ResubmitCommand);
strcpy(All.OutputListFilename, all.OutputListFilename);
strcpy(All.OutputDir, all.OutputDir);
strcpy(All.RestartFile, all.RestartFile);
strcpy(All.EnergyFile, all.EnergyFile);
strcpy(All.InfoFile, all.InfoFile);
strcpy(All.CpuFile, all.CpuFile);
strcpy(All.TimingsFile, all.TimingsFile);
strcpy(All.SnapshotFileBase, all.SnapshotFileBase);
if(All.TimeMax != all.TimeMax)
readjust_timebase(All.TimeMax, all.TimeMax);
}
#ifdef PMGRID
long_range_init_regionsize();
#endif
if(All.ComovingIntegrationOn)
init_drift_table();
if(RestartFlag == 2)
All.Ti_nextoutput = find_next_outputtime(All.Ti_Current + 1);
else
All.Ti_nextoutput = find_next_outputtime(All.Ti_Current);
All.TimeLastRestartFile = CPUThisRun;
}
/*! This function finds the next synchronization point of the system (i.e. the
* earliest point of time any of the particles needs a force computation),
* and drifts the system to this point of time. If the system drifts over
* the desired time of a snapshot file, the function will drift to this
* moment, generate an output, and then resume the drift.
*/
void find_next_sync_point_and_drift(void)
{
int n, min, min_glob, flag, *temp;
double timeold;
double t0, t1;
t0 = second();
timeold = All.Time;
for(n = 1, min = P[0].Ti_endstep; n < NumPart; n++)
if(min > P[n].Ti_endstep)
min = P[n].Ti_endstep;
MPI_Allreduce(&min, &min_glob, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
/* We check whether this is a full step where all particles are synchronized */
flag = 1;
for(n = 0; n < NumPart; n++)
if(P[n].Ti_endstep > min_glob)
flag = 0;
MPI_Allreduce(&flag, &Flag_FullStep, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
#ifdef PMGRID
if(min_glob >= All.PM_Ti_endstep)
{
min_glob = All.PM_Ti_endstep;
Flag_FullStep = 1;
}
#endif
/* Determine 'NumForceUpdate', i.e. the number of particles on this processor that are going to be active */
for(n = 0, NumForceUpdate = 0; n < NumPart; n++)
{
if(P[n].Ti_endstep == min_glob)
#ifdef SELECTIVE_NO_GRAVITY
if(!((1 << P[n].Type) & (SELECTIVE_NO_GRAVITY)))
#endif
NumForceUpdate++;
}
/* note: NumForcesSinceLastDomainDecomp has type "long long" */
temp = malloc(NTask * sizeof(int));
MPI_Allgather(&NumForceUpdate, 1, MPI_INT, temp, 1, MPI_INT, MPI_COMM_WORLD);
for(n = 0; n < NTask; n++)
All.NumForcesSinceLastDomainDecomp += temp[n];
free(temp);
t1 = second();
All.CPU_Predict += timediff(t0, t1);
while(min_glob >= All.Ti_nextoutput && All.Ti_nextoutput >= 0)
{
move_particles(All.Ti_Current, All.Ti_nextoutput);
All.Ti_Current = All.Ti_nextoutput;
if(All.ComovingIntegrationOn)
All.Time = All.TimeBegin * exp(All.Ti_Current * All.Timebase_interval);
else
All.Time = All.TimeBegin + All.Ti_Current * All.Timebase_interval;
#ifdef OUTPUTPOTENTIAL
All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
domain_Decomposition();
compute_potential();
#endif
savepositions(All.SnapshotFileCount++); /* write snapshot file */
All.Ti_nextoutput = find_next_outputtime(All.Ti_nextoutput + 1);
}
move_particles(All.Ti_Current, min_glob);
All.Ti_Current = min_glob;
if(All.ComovingIntegrationOn)
All.Time = All.TimeBegin * exp(All.Ti_Current * All.Timebase_interval);
else
All.Time = All.TimeBegin + All.Ti_Current * All.Timebase_interval;
All.TimeStep = All.Time - timeold;
}
