/*! This function writes a snapshot of the particle distribution to one or * several files using the selected file format. If NumFilesPerSnapshot>1, * the snapshot is distributed onto several files, several of them can be * written simultaneously (up to NumFilesWrittenInParallel). Each file * contains data from a group of processors. */ void savepositions(int num) { double t0, t1; char buf[500]; int i, j, *temp, n, filenr, gr, ngroups, masterTask, lastTask; t0 = second(); if(ThisTask == 0) printf("\nwriting snapshot file... \n"); #if defined(SFR) || defined(BLACK_HOLES) rearrange_particle_sequence(); /* ensures that new tree will be constructed */ All.NumForcesSinceLastDomainDecomp = 1 + All.TreeDomainUpdateFrequency * All.TotNumPart; #endif if(NTask < All.NumFilesPerSnapshot) { if(ThisTask == 0) printf("Fatal error.\nNumber of processors must be larger or equal than All.NumFilesPerSnapshot.\n"); endrun(0); } if(All.SnapFormat < 1 || All.SnapFormat > 3) { if(ThisTask == 0) printf("Unsupported File-Format\n"); endrun(0); } #ifndef HAVE_HDF5 if(All.SnapFormat == 3) { if(ThisTask == 0) printf("Code wasn't compiled with HDF5 support enabled!\n"); endrun(0); } #endif /* determine global and local particle numbers */ for(n = 0; n < 6; n++) n_type[n] = 0; for(n = 0; n < NumPart; n++) n_type[P[n].Type]++; /* because ntot_type_all[] is of type `long long', we cannot do a simple * MPI_Allreduce() to sum the total particle numbers */ temp = malloc(NTask * 6 * sizeof(int)); MPI_Allgather(n_type, 6, MPI_INT, temp, 6, MPI_INT, MPI_COMM_WORLD); for(i = 0; i < 6; i++) { ntot_type_all[i] = 0; for(j = 0; j < NTask; j++) ntot_type_all[i] += temp[j * 6 + i]; } free(temp); /* assign processors to output files */ distribute_file(All.NumFilesPerSnapshot, 0, 0, NTask - 1, &filenr, &masterTask, &lastTask); fill_Tab_IO_Labels(); if(All.NumFilesPerSnapshot > 1) sprintf(buf, "%s%s_%03d.%d.g", All.OutputDir, All.SnapshotFileBase, num, filenr); else sprintf(buf, "%s%s_%03d.g", All.OutputDir, All.SnapshotFileBase, num); ngroups = All.NumFilesPerSnapshot / All.NumFilesWrittenInParallel; if((All.NumFilesPerSnapshot % All.NumFilesWrittenInParallel)) ngroups++; for(gr = 0; gr < ngroups; gr++) { if((filenr / All.NumFilesWrittenInParallel) == gr) /* ok, it's this processor's turn */ write_file(buf, masterTask, lastTask); MPI_Barrier(MPI_COMM_WORLD); } if(ThisTask == 0) printf("done with snapshot.\n"); t1 = second(); All.CPU_Snapshot += timediff(t0, t1); }
/* This function computes the gravitational potential for ALL the particles. * It expects that the particles are predicted to the current time. */ void compute_potential(void) { int i; #ifndef NOGRAVITY long long ntot, ntotleft; int j, k, level, sendTask, recvTask; int ndone; int maxfill, ngrp, place, nexport; int *nsend, *noffset, *nsend_local, *nbuffer, *ndonelist, *numlist; double fac; double t0, t1, tstart, tend; MPI_Status status; double r2; t0 = second(); if(All.ComovingIntegrationOn) set_softenings(); if(ThisTask == 0) { printf("Start computation of potential for all particles...\n"); fflush(stdout); } #ifdef ISOTHERM for(i = 0; i < NumPart; i++) { for(k = 0, r2 = 0; k < 3; k++) r2 += P[i].Pos[k] * P[i].Pos[k]; P[i].Potential = -2 * ISOTHERM * ISOTHERM * (1 + log(ISOTHERM / sqrt(r2))); } return; #endif tstart = second(); if(TreeReconstructFlag) { if(ThisTask == 0) printf("Tree construction.\n"); #if defined(SFR) || defined(BLACK_HOLES) rearrange_particle_sequence(); #endif force_treebuild(); TreeReconstructFlag = 0; if(ThisTask == 0) printf("Tree construction done.\n"); } tend = second(); All.CPU_TreeConstruction += timediff(tstart, tend); numlist = malloc(NTask * sizeof(int) * NTask); MPI_Allgather(&NumPart, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD); for(i = 0, ntot = 0; i < NTask; i++) ntot += numlist[i]; free(numlist); noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */ nbuffer = malloc(sizeof(int) * NTask); nsend_local = malloc(sizeof(int) * NTask); nsend = malloc(sizeof(int) * NTask * NTask); ndonelist = malloc(sizeof(int) * NTask); i = 0; /* beginn with this index */ ntotleft = ntot; /* particles left for all tasks together */ while(ntotleft > 0) { for(j = 0; j < NTask; j++) nsend_local[j] = 0; /* do local particles and prepare export list */ for(nexport = 0, ndone = 0; i < NumPart && nexport < All.BunchSizeForce - NTask; i++) { ndone++; for(j = 0; j < NTask; j++) Exportflag[j] = 0; #ifndef PMGRID force_treeevaluate_potential(i, 0); #else force_treeevaluate_potential_shortrange(i, 0); #endif for(j = 0; j < NTask; j++) { if(Exportflag[j]) { for(k = 0; k < 3; k++) GravDataGet[nexport].u.Pos[k] = P[i].Pos[k]; #ifdef UNEQUALSOFTENINGS GravDataGet[nexport].v.Type = P[i].Type; #endif GravDataGet[nexport].w.OldAcc = P[i].OldAcc; GravDataIndexTable[nexport].Task = j; GravDataIndexTable[nexport].Index = i; GravDataIndexTable[nexport].SortIndex = nexport; nexport++; nsend_local[j]++; } } } qsort(GravDataIndexTable, nexport, sizeof(struct gravdata_index), grav_tree_compare_key); for(j = 0; j < nexport; j++) GravDataIn[j] = GravDataGet[GravDataIndexTable[j].SortIndex]; for(j = 1, noffset[0] = 0; j < NTask; j++) noffset[j] = noffset[j - 1] + nsend_local[j - 1]; MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD); /* now do the particles that need to be exported */ for(level = 1; level < (1 << PTask); level++) { for(j = 0; j < NTask; j++) nbuffer[j] = 0; for(ngrp = level; ngrp < (1 << PTask); ngrp++) { maxfill = 0; for(j = 0; j < NTask; j++) { if((j ^ ngrp) < NTask) if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j]) maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j]; } if(maxfill >= All.BunchSizeForce) break; sendTask = ThisTask; recvTask = ThisTask ^ ngrp; if(recvTask < NTask) { if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0) { /* get the particles */ MPI_Sendrecv(&GravDataIn[noffset[recvTask]], nsend_local[recvTask] * sizeof(struct gravdata_in), MPI_BYTE, recvTask, TAG_POTENTIAL_A, &GravDataGet[nbuffer[ThisTask]], nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in), MPI_BYTE, recvTask, TAG_POTENTIAL_A, MPI_COMM_WORLD, &status); } } for(j = 0; j < NTask; j++) if((j ^ ngrp) < NTask) nbuffer[j] += nsend[(j ^ ngrp) * NTask + j]; } for(j = 0; j < nbuffer[ThisTask]; j++) { #ifndef PMGRID force_treeevaluate_potential(j, 1); #else force_treeevaluate_potential_shortrange(j, 1); #endif } /* get the result */ for(j = 0; j < NTask; j++) nbuffer[j] = 0; for(ngrp = level; ngrp < (1 << PTask); ngrp++) { maxfill = 0; for(j = 0; j < NTask; j++) { if((j ^ ngrp) < NTask) if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j]) maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j]; } if(maxfill >= All.BunchSizeForce) break; sendTask = ThisTask; recvTask = ThisTask ^ ngrp; if(recvTask < NTask) { if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0) { /* send the results */ MPI_Sendrecv(&GravDataResult[nbuffer[ThisTask]], nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in), MPI_BYTE, recvTask, TAG_POTENTIAL_B, &GravDataOut[noffset[recvTask]], nsend_local[recvTask] * sizeof(struct gravdata_in), MPI_BYTE, recvTask, TAG_POTENTIAL_B, MPI_COMM_WORLD, &status); /* add the result to the particles */ for(j = 0; j < nsend_local[recvTask]; j++) { place = GravDataIndexTable[noffset[recvTask] + j].Index; P[place].Potential += GravDataOut[j + noffset[recvTask]].v.Potential; } } } for(j = 0; j < NTask; j++) if((j ^ ngrp) < NTask) nbuffer[j] += nsend[(j ^ ngrp) * NTask + j]; } level = ngrp - 1; } MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD); for(j = 0; j < NTask; j++) ntotleft -= ndonelist[j]; } free(ndonelist); free(nsend); free(nsend_local); free(nbuffer); free(noffset); /* add correction to exclude self-potential */ for(i = 0; i < NumPart; i++) { /* remove self-potential */ P[i].Potential += P[i].Mass / All.SofteningTable[P[i].Type]; if(All.ComovingIntegrationOn) if(All.PeriodicBoundariesOn) P[i].Potential -= 2.8372975 * pow(P[i].Mass, 2.0 / 3) * pow(All.Omega0 * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G), 1.0 / 3); } /* multiply with the gravitational constant */ for(i = 0; i < NumPart; i++) P[i].Potential *= All.G; #ifdef PMGRID #ifdef PERIODIC pmpotential_periodic(); #ifdef PLACEHIGHRESREGION pmpotential_nonperiodic(1); #endif #else pmpotential_nonperiodic(0); #ifdef PLACEHIGHRESREGION pmpotential_nonperiodic(1); #endif #endif #endif if(All.ComovingIntegrationOn) { #ifndef PERIODIC fac = -0.5 * All.Omega0 * All.Hubble * All.Hubble; for(i = 0; i < NumPart; i++) { for(k = 0, r2 = 0; k < 3; k++) r2 += P[i].Pos[k] * P[i].Pos[k]; P[i].Potential += fac * r2; } #endif } else { fac = -0.5 * All.OmegaLambda * All.Hubble * All.Hubble; if(fac != 0) { for(i = 0; i < NumPart; i++) { for(k = 0, r2 = 0; k < 3; k++) r2 += P[i].Pos[k] * P[i].Pos[k]; P[i].Potential += fac * r2; } } } if(ThisTask == 0) { printf("potential done.\n"); fflush(stdout); } t1 = second(); All.CPU_Potential += timediff(t0, t1); #else for(i = 0; i < NumPart; i++) P[i].Potential = 0; #endif }
/*! This function computes the gravitational potential for ALL the particles. * First, the (short-range) tree potential is computed, and then, if needed, * the long range PM potential is added. */ void compute_potential(void) { int i; #ifndef NOGRAVITY int j, k, ret, sendTask, recvTask; int ndone, ndone_flag, dummy; int ngrp, place, nexport, nimport; double fac; MPI_Status status; double r2; if(All.ComovingIntegrationOn) set_softenings(); if(ThisTask == 0) { printf("Start computation of potential for all particles...\n"); fflush(stdout); } CPU_Step[CPU_MISC] += measure_time(); if(TreeReconstructFlag) { if(ThisTask == 0) printf("Tree construction.\n"); CPU_Step[CPU_MISC] += measure_time(); #if defined(SFR) || defined(BLACK_HOLES) rearrange_particle_sequence(); #endif force_treebuild(NumPart, NULL); CPU_Step[CPU_TREEBUILD] += measure_time(); TreeReconstructFlag = 0; if(ThisTask == 0) printf("Tree construction done.\n"); } /* allocate buffers to arrange communication */ All.BunchSize = (int) ((All.BufferSize * 1024 * 1024) / (sizeof(struct data_index) + sizeof(struct data_nodelist) + sizeof(struct gravdata_in) + sizeof(struct potdata_out) + sizemax(sizeof(struct gravdata_in), sizeof(struct potdata_out)))); DataIndexTable = (struct data_index *) mymalloc(All.BunchSize * sizeof(struct data_index)); DataNodeList = (struct data_nodelist *) mymalloc(All.BunchSize * sizeof(struct data_nodelist)); for(i = 0; i < NumPart; i++) if(P[i].Ti_current != All.Ti_Current) drift_particle(i, All.Ti_Current); i = 0; /* beginn with this index */ do { for(j = 0; j < NTask; j++) { Send_count[j] = 0; Exportflag[j] = -1; } /* do local particles and prepare export list */ for(nexport = 0; i < NumPart; i++) { #ifndef PMGRID ret = force_treeevaluate_potential(i, 0, &nexport, Send_count); #else ret = force_treeevaluate_potential_shortrange(i, 0, &nexport, Send_count); #endif if(ret < 0) break; /* export buffer has filled up */ } #ifdef MYSORT mysort_dataindex(DataIndexTable, nexport, sizeof(struct data_index), data_index_compare); #else qsort(DataIndexTable, nexport, sizeof(struct data_index), data_index_compare); #endif MPI_Allgather(Send_count, NTask, MPI_INT, Sendcount_matrix, NTask, MPI_INT, MPI_COMM_WORLD); for(j = 0, nimport = 0, Recv_offset[0] = 0, Send_offset[0] = 0; j < NTask; j++) { Recv_count[j] = Sendcount_matrix[j * NTask + ThisTask]; nimport += Recv_count[j]; if(j > 0) { Send_offset[j] = Send_offset[j - 1] + Send_count[j - 1]; Recv_offset[j] = Recv_offset[j - 1] + Recv_count[j - 1]; } } GravDataGet = (struct gravdata_in *) mymalloc(nimport * sizeof(struct gravdata_in)); GravDataIn = (struct gravdata_in *) mymalloc(nexport * sizeof(struct gravdata_in)); /* prepare particle data for export */ for(j = 0; j < nexport; j++) { place = DataIndexTable[j].Index; for(k = 0; k < 3; k++) GravDataIn[j].Pos[k] = P[place].Pos[k]; #ifdef UNEQUALSOFTENINGS GravDataIn[j].Type = P[place].Type; #ifdef ADAPTIVE_GRAVSOFT_FORGAS if(P[place].Type == 0) GravDataIn[j].Soft = SphP[place].Hsml; #endif #endif GravDataIn[j].OldAcc = P[place].OldAcc; for(k = 0; k < NODELISTLENGTH; k++) GravDataIn[j].NodeList[k] = DataNodeList[DataIndexTable[j].IndexGet].NodeList[k]; } /* exchange particle data */ for(ngrp = 1; ngrp < (1 << PTask); ngrp++) { sendTask = ThisTask; recvTask = ThisTask ^ ngrp; if(recvTask < NTask) { if(Send_count[recvTask] > 0 || Recv_count[recvTask] > 0) { /* get the particles */ MPI_Sendrecv(&GravDataIn[Send_offset[recvTask]], Send_count[recvTask] * sizeof(struct gravdata_in), MPI_BYTE, recvTask, TAG_POTENTIAL_A, &GravDataGet[Recv_offset[recvTask]], Recv_count[recvTask] * sizeof(struct gravdata_in), MPI_BYTE, recvTask, TAG_POTENTIAL_A, MPI_COMM_WORLD, &status); } } } myfree(GravDataIn); PotDataResult = (struct potdata_out *) mymalloc(nimport * sizeof(struct potdata_out)); PotDataOut = (struct potdata_out *) mymalloc(nexport * sizeof(struct potdata_out)); /* now do the particles that were sent to us */ for(j = 0; j < nimport; j++) { #ifndef PMGRID force_treeevaluate_potential(j, 1, &dummy, &dummy); #else force_treeevaluate_potential_shortrange(j, 1, &dummy, &dummy); #endif } if(i >= NumPart) ndone_flag = 1; else ndone_flag = 0; MPI_Allreduce(&ndone_flag, &ndone, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); /* get the result */ for(ngrp = 1; ngrp < (1 << PTask); ngrp++) { sendTask = ThisTask; recvTask = ThisTask ^ ngrp; if(recvTask < NTask) { if(Send_count[recvTask] > 0 || Recv_count[recvTask] > 0) { /* send the results */ MPI_Sendrecv(&PotDataResult[Recv_offset[recvTask]], Recv_count[recvTask] * sizeof(struct potdata_out), MPI_BYTE, recvTask, TAG_POTENTIAL_B, &PotDataOut[Send_offset[recvTask]], Send_count[recvTask] * sizeof(struct potdata_out), MPI_BYTE, recvTask, TAG_POTENTIAL_B, MPI_COMM_WORLD, &status); } } } /* add the results to the local particles */ for(j = 0; j < nexport; j++) { place = DataIndexTable[j].Index; P[place].p.dPotential += PotDataOut[j].Potential; } myfree(PotDataOut); myfree(PotDataResult); myfree(GravDataGet); } while(ndone < NTask); myfree(DataNodeList); myfree(DataIndexTable); /* add correction to exclude self-potential */ for(i = 0; i < NumPart; i++) { #ifdef FLTROUNDOFFREDUCTION P[i].p.Potential = FLT(P[i].p.dPotential); #endif /* remove self-potential */ P[i].p.Potential += P[i].Mass / All.SofteningTable[P[i].Type]; if(All.ComovingIntegrationOn) if(All.PeriodicBoundariesOn) P[i].p.Potential -= 2.8372975 * pow(P[i].Mass, 2.0 / 3) * pow(All.Omega0 * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G), 1.0 / 3); } /* multiply with the gravitational constant */ for(i = 0; i < NumPart; i++) P[i].p.Potential *= All.G; #ifdef PMGRID #ifdef PERIODIC pmpotential_periodic(); #ifdef PLACEHIGHRESREGION i = pmpotential_nonperiodic(1); if(i == 1) /* this is returned if a particle lied outside allowed range */ { pm_init_regionsize(); pm_setup_nonperiodic_kernel(); i = pmpotential_nonperiodic(1); /* try again */ } if(i == 1) endrun(88686); #endif #else i = pmpotential_nonperiodic(0); if(i == 1) /* this is returned if a particle lied outside allowed range */ { pm_init_regionsize(); pm_setup_nonperiodic_kernel(); i = pmpotential_nonperiodic(0); /* try again */ } if(i == 1) endrun(88687); #ifdef PLACEHIGHRESREGION i = pmpotential_nonperiodic(1); if(i == 1) /* this is returned if a particle lied outside allowed range */ { pm_init_regionsize(); i = pmpotential_nonperiodic(1); } if(i != 0) endrun(88688); #endif #endif #endif if(All.ComovingIntegrationOn) { #ifndef PERIODIC fac = -0.5 * All.Omega0 * All.Hubble * All.Hubble; for(i = 0; i < NumPart; i++) { for(k = 0, r2 = 0; k < 3; k++) r2 += P[i].Pos[k] * P[i].Pos[k]; P[i].p.Potential += fac * r2; } #endif } else { fac = -0.5 * All.OmegaLambda * All.Hubble * All.Hubble; if(fac != 0) { for(i = 0; i < NumPart; i++) { for(k = 0, r2 = 0; k < 3; k++) r2 += P[i].Pos[k] * P[i].Pos[k]; P[i].p.Potential += fac * r2; } } } if(ThisTask == 0) { printf("potential done.\n"); fflush(stdout); } #else for(i = 0; i < NumPart; i++) P[i].Potential = 0; #endif CPU_Step[CPU_POTENTIAL] += measure_time(); }
/* This function reads or writes the restart files. * Each processor writes its own restart file, with the * I/O being done in parallel. To avoid congestion of the disks * you can tell the program to restrict the number of files * that are simultaneously written to NumFilesWrittenInParallel. * * If modus>0 the restart()-routine reads, * if modus==0 it writes a restart file. */ void restart(int modus) { char buf[200], buf_bak[200], buf_mv[500]; double save_PartAllocFactor, save_TreeAllocFactor; int i, nprocgroup, masterTask, groupTask, old_MaxPart, old_MaxNodes; struct global_data_all_processes all_task0; #if defined(SFR) || defined(BLACK_HOLES) #ifdef NO_TREEDATA_IN_RESTART if(modus == 0) { rearrange_particle_sequence(); All.NumForcesSinceLastDomainDecomp = 1 + All.TreeDomainUpdateFrequency * All.TotNumPart; /* ensures that new tree will be constructed */ } #endif #endif sprintf(buf, "%s%s.%d", All.OutputDir, All.RestartFile, ThisTask); sprintf(buf_bak, "%s%s.%d.bak", All.OutputDir, All.RestartFile, ThisTask); sprintf(buf_mv, "mv %s %s", buf, buf_bak); if((NTask < All.NumFilesWrittenInParallel)) { printf ("Fatal error.\nNumber of processors must be a smaller or equal than `NumFilesWrittenInParallel'.\n"); endrun(2131); } nprocgroup = NTask / All.NumFilesWrittenInParallel; if((NTask % All.NumFilesWrittenInParallel)) { nprocgroup++; } masterTask = (ThisTask / nprocgroup) * nprocgroup; for(groupTask = 0; groupTask < nprocgroup; groupTask++) { if(ThisTask == (masterTask + groupTask)) /* ok, it's this processor's turn */ { if(modus) { if(!(fd = fopen(buf, "r"))) { printf("Restart file '%s' not found.\n", buf); endrun(7870); } } else { system(buf_mv); /* move old restart files to .bak files */ if(!(fd = fopen(buf, "w"))) { printf("Restart file '%s' cannot be opened.\n", buf); endrun(7878); } } save_PartAllocFactor = All.PartAllocFactor; save_TreeAllocFactor = All.TreeAllocFactor; /* common data */ byten(&All, sizeof(struct global_data_all_processes), modus); if(ThisTask == 0 && modus > 0) all_task0 = All; if(modus > 0 && groupTask == 0) /* read */ { MPI_Bcast(&all_task0, sizeof(struct global_data_all_processes), MPI_BYTE, 0, MPI_COMM_WORLD); } old_MaxPart = All.MaxPart; old_MaxNodes = All.TreeAllocFactor * All.MaxPart; if(modus) /* read */ { if(All.PartAllocFactor != save_PartAllocFactor) { All.PartAllocFactor = save_PartAllocFactor; All.MaxPart = All.PartAllocFactor * (All.TotNumPart / NTask); All.MaxPartSph = All.PartAllocFactor * (All.TotN_gas / NTask); #ifdef INHOMOG_GASDISTR_HINT All.MaxPartSph = All.MaxPart; #endif save_PartAllocFactor = -1; } if(All.TreeAllocFactor != save_TreeAllocFactor) { All.TreeAllocFactor = save_TreeAllocFactor; save_TreeAllocFactor = -1; } if(all_task0.Time != All.Time) { printf("The restart file on task=%d is not consistent with the one on task=0\n", ThisTask); fflush(stdout); endrun(16); } allocate_memory(); } in(&NumPart, modus); if(NumPart > All.MaxPart) { printf ("it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n", NumPart / (((double) All.TotNumPart) / NTask)); printf("fatal error\n"); endrun(22); } /* Particle data */ byten(&P[0], NumPart * sizeof(struct particle_data), modus); in(&N_gas, modus); if(N_gas > 0) { if(N_gas > All.MaxPartSph) { printf ("SPH: it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n", N_gas / (((double) All.TotN_gas) / NTask)); printf("fatal error\n"); endrun(222); } /* Sph-Particle data */ byten(&SphP[0], N_gas * sizeof(struct sph_particle_data), modus); } /* write state of random number generator */ byten(gsl_rng_state(random_generator), gsl_rng_size(random_generator), modus); #ifndef NO_TREEDATA_IN_RESTART /* now store relevant data for tree */ #ifdef SFR in(&Stars_converted, modus); #endif if(modus) /* read */ { ngb_treeallocate(MAX_NGB); force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart); } in(&Numnodestree, modus); if(Numnodestree > MaxNodes) { printf ("Tree storage: it seems you have reduced(!) 'PartAllocFactor' below the value needed to load the restart file (task=%d). " "Numnodestree=%d MaxNodes=%d\n", ThisTask, Numnodestree, MaxNodes); endrun(221); } byten(Nodes_base, Numnodestree * sizeof(struct NODE), modus); byten(Extnodes_base, Numnodestree * sizeof(struct extNODE), modus); byten(Father, NumPart * sizeof(int), modus); byten(Nextnode, NumPart * sizeof(int), modus); byten(Nextnode + All.MaxPart, MAXTOPNODES * sizeof(int), modus); byten(DomainStartList, NTask * sizeof(int), modus); byten(DomainEndList, NTask * sizeof(int), modus); byten(DomainTask, MAXTOPNODES * sizeof(int), modus); byten(DomainNodeIndex, MAXTOPNODES * sizeof(int), modus); byten(DomainTreeNodeLen, MAXTOPNODES * sizeof(FLOAT), modus); byten(DomainHmax, MAXTOPNODES * sizeof(FLOAT), modus); byten(DomainMoment, MAXTOPNODES * sizeof(struct DomainNODE), modus); byten(DomainCorner, 3 * sizeof(double), modus); byten(DomainCenter, 3 * sizeof(double), modus); byten(&DomainLen, sizeof(double), modus); byten(&DomainFac, sizeof(double), modus); byten(&DomainMyStart, sizeof(int), modus); byten(&DomainMyLast, sizeof(int), modus); if(modus) /* read */ if(All.PartAllocFactor != save_PartAllocFactor || All.TreeAllocFactor != save_TreeAllocFactor) { for(i = 0; i < NumPart; i++) Father[i] += (All.MaxPart - old_MaxPart); for(i = 0; i < NumPart; i++) if(Nextnode[i] >= old_MaxPart) { if(Nextnode[i] >= old_MaxPart + old_MaxNodes) Nextnode[i] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxPart); else Nextnode[i] += (All.MaxPart - old_MaxPart); } for(i = 0; i < Numnodestree; i++) { if(Nodes_base[i].u.d.sibling >= old_MaxPart) { if(Nodes_base[i].u.d.sibling >= old_MaxPart + old_MaxNodes) Nodes_base[i].u.d.sibling += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes); else Nodes_base[i].u.d.sibling += (All.MaxPart - old_MaxPart); } if(Nodes_base[i].u.d.father >= old_MaxPart) { if(Nodes_base[i].u.d.father >= old_MaxPart + old_MaxNodes) Nodes_base[i].u.d.father += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes); else Nodes_base[i].u.d.father += (All.MaxPart - old_MaxPart); } if(Nodes_base[i].u.d.nextnode >= old_MaxPart) { if(Nodes_base[i].u.d.nextnode >= old_MaxPart + old_MaxNodes) Nodes_base[i].u.d.nextnode += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes); else Nodes_base[i].u.d.nextnode += (All.MaxPart - old_MaxPart); } } for(i = 0; i < MAXTOPNODES; i++) if(Nextnode[i + All.MaxPart] >= old_MaxPart) { if(Nextnode[i + All.MaxPart] >= old_MaxPart + old_MaxNodes) Nextnode[i + All.MaxPart] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes); else Nextnode[i + All.MaxPart] += (All.MaxPart - old_MaxPart); } for(i = 0; i < MAXTOPNODES; i++) if(DomainNodeIndex[i] >= old_MaxPart) { if(DomainNodeIndex[i] >= old_MaxPart + old_MaxNodes) DomainNodeIndex[i] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes); else DomainNodeIndex[i] += (All.MaxPart - old_MaxPart); } } #endif fclose(fd); } else /* wait inside the group */ { if(modus > 0 && groupTask == 0) /* read */ { MPI_Bcast(&all_task0, sizeof(struct global_data_all_processes), MPI_BYTE, 0, MPI_COMM_WORLD); } } MPI_Barrier(MPI_COMM_WORLD); } }