void subfind_find_linkngb(void)
{
long long ntot;
int i, j, ndone, ndone_flag, npleft, dummy, iter = 0, save_DesNumNgb;
MyFloat *Left, *Right;
char *Todo;
int ngrp, recvTask, place, nexport, nimport;
double t0, t1;
if(ThisTask == 0)
printf("Start find_linkngb (%d particles on task=%d)\n", NumPartGroup, ThisTask);
save_DesNumNgb = All.DesNumNgb;
All.DesNumNgb = All.DesLinkNgb; /* for simplicity, reset this value */
/* allocate buffers to arrange communication */
Ngblist = (int *) mymalloc("Ngblist", NumPartGroup * sizeof(int));
Dist2list = (double *) mymalloc("Dist2list", NumPartGroup * sizeof(double));
All.BunchSize =
(int) ((All.BufferSize * 1024 * 1024) / (sizeof(struct data_index) + sizeof(struct data_nodelist) +
sizeof(struct linkngbdata_in) + sizeof(struct linkngbdata_out) +
sizemax(sizeof(struct linkngbdata_in),
sizeof(struct linkngbdata_out))));
DataIndexTable =
(struct data_index *) mymalloc("DataIndexTable", All.BunchSize * sizeof(struct data_index));
DataNodeList =
(struct data_nodelist *) mymalloc("DataNodeList", All.BunchSize * sizeof(struct data_nodelist));
Left = mymalloc("Left", sizeof(MyFloat) * NumPartGroup);
Right = mymalloc("Right", sizeof(MyFloat) * NumPartGroup);
Todo = mymalloc("Todo", sizeof(char) * NumPartGroup);
for(i = 0; i < NumPartGroup; i++)
{
Left[i] = Right[i] = 0;
Todo[i] = 1;
}
/* we will repeat the whole thing for those particles where we didn't find enough neighbours */
do
{
t0 = second();
i = 0; /* begin with this index */
do
{
for(j = 0; j < NTask; j++)
{
Send_count[j] = 0;
Exportflag[j] = -1;
}
/* do local particles and prepare export list */
for(nexport = 0; i < NumPartGroup; i++)
{
if(Todo[i])
{
if(subfind_linkngb_evaluate(i, 0, &nexport, Send_count) < 0)
break;
}
}
#ifdef OMP_SORT
omp_qsort(DataIndexTable, nexport, sizeof(struct data_index), data_index_compare);
#else
qsort(DataIndexTable, nexport, sizeof(struct data_index), data_index_compare);
#endif
MPI_Alltoall(Send_count, 1, MPI_INT, Recv_count, 1, MPI_INT, MPI_COMM_WORLD);
for(j = 0, nimport = 0, Recv_offset[0] = 0, Send_offset[0] = 0; j < NTask; j++)
{
nimport += Recv_count[j];
if(j > 0)
{
Send_offset[j] = Send_offset[j - 1] + Send_count[j - 1];
Recv_offset[j] = Recv_offset[j - 1] + Recv_count[j - 1];
}
}
LinkngbDataGet =
(struct linkngbdata_in *) mymalloc(" LinkngbDataGet",
nimport * sizeof(struct linkngbdata_in));
LinkngbDataIn =
(struct linkngbdata_in *) mymalloc(" LinkngbDataIn",
nexport * sizeof(struct linkngbdata_in));
/* prepare particle data for export */
for(j = 0; j < nexport; j++)
{
place = DataIndexTable[j].Index;
LinkngbDataIn[j].Pos[0] = P[place].Pos[0];
LinkngbDataIn[j].Pos[1] = P[place].Pos[1];
LinkngbDataIn[j].Pos[2] = P[place].Pos[2];
LinkngbDataIn[j].DM_Hsml = P[place].DM_Hsml;
memcpy(LinkngbDataIn[j].NodeList,
DataNodeList[DataIndexTable[j].IndexGet].NodeList, NODELISTLENGTH * sizeof(int));
}
/* exchange particle data */
for(ngrp = 1; ngrp < (1 << PTask); ngrp++)
{
recvTask = ThisTask ^ ngrp;
if(recvTask < NTask)
{
if(Send_count[recvTask] > 0 || Recv_count[recvTask] > 0)
{
/* get the particles */
MPI_Sendrecv(&LinkngbDataIn[Send_offset[recvTask]],
Send_count[recvTask] * sizeof(struct linkngbdata_in), MPI_BYTE,
recvTask, TAG_DENS_A,
&LinkngbDataGet[Recv_offset[recvTask]],
Recv_count[recvTask] * sizeof(struct linkngbdata_in), MPI_BYTE,
recvTask, TAG_DENS_A, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
}
}
myfree(LinkngbDataIn);
LinkngbDataResult =
(struct linkngbdata_out *) mymalloc(" LinkngbDataResult",
nimport * sizeof(struct linkngbdata_out));
LinkngbDataOut =
(struct linkngbdata_out *) mymalloc(" LinkngbDataOut",
nexport * sizeof(struct linkngbdata_out));
/* now do the particles that were sent to us */
for(j = 0; j < nimport; j++)
subfind_linkngb_evaluate(j, 1, &dummy, &dummy);
if(i >= NumPartGroup)
ndone_flag = 1;
else
ndone_flag = 0;
MPI_Allreduce(&ndone_flag, &ndone, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
/* get the result */
for(ngrp = 1; ngrp < (1 << PTask); ngrp++)
{
recvTask = ThisTask ^ ngrp;
if(recvTask < NTask)
{
if(Send_count[recvTask] > 0 || Recv_count[recvTask] > 0)
{
/* send the results */
MPI_Sendrecv(&LinkngbDataResult[Recv_offset[recvTask]],
Recv_count[recvTask] * sizeof(struct linkngbdata_out),
MPI_BYTE, recvTask, TAG_DENS_B,
&LinkngbDataOut[Send_offset[recvTask]],
Send_count[recvTask] * sizeof(struct linkngbdata_out),
MPI_BYTE, recvTask, TAG_DENS_B, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
}
}
/* add the result to the local particles */
for(j = 0; j < nexport; j++)
{
place = DataIndexTable[j].Index;
P[place].DM_NumNgb += LinkngbDataOut[j].Ngb;
}
myfree(LinkngbDataOut);
myfree(LinkngbDataResult);
myfree(LinkngbDataGet);
}
while(ndone < NTask);
/* do final operations on results */
for(i = 0, npleft = 0; i < NumPartGroup; i++)
{
/* now check whether we had enough neighbours */
if(Todo[i])
{
if(P[i].DM_NumNgb != All.DesLinkNgb &&
((Right[i] - Left[i]) > 1.0e-3 * Left[i] || Left[i] == 0 || Right[i] == 0))
{
/* need to redo this particle */
npleft++;
if(P[i].DM_NumNgb < All.DesLinkNgb)
Left[i] = DMAX(P[i].DM_Hsml, Left[i]);
else
{
if(Right[i] != 0)
{
if(P[i].DM_Hsml < Right[i])
Right[i] = P[i].DM_Hsml;
}
else
Right[i] = P[i].DM_Hsml;
}
if(iter >= MAXITER - 10)
{
printf
("i=%d task=%d ID=%d DM_Hsml=%g Left=%g Right=%g Ngbs=%g Right-Left=%g\n pos=(%g|%g|%g)\n",
i, ThisTask, (int) P[i].ID, P[i].DM_Hsml, Left[i], Right[i],
(double) P[i].DM_NumNgb, Right[i] - Left[i], P[i].Pos[0], P[i].Pos[1], P[i].Pos[2]);
fflush(stdout);
}
if(Right[i] > 0 && Left[i] > 0)
P[i].DM_Hsml = pow(0.5 * (pow(Left[i], 3) + pow(Right[i], 3)), 1.0 / 3);
else
{
if(Right[i] == 0 && Left[i] == 0)
endrun(8189); /* can't occur */
if(Right[i] == 0 && Left[i] > 0)
P[i].DM_Hsml *= 1.26;
if(Right[i] > 0 && Left[i] == 0)
P[i].DM_Hsml /= 1.26;
}
}
else
Todo[i] = 0;
}
}
sumup_large_ints(1, &npleft, &ntot);
t1 = second();
if(ntot > 0)
{
iter++;
if(iter > 0 && ThisTask == 0)
{
printf("find linkngb iteration %d: need to repeat for %d%09d particles. (took %g sec)\n", iter,
(int) (ntot / 1000000000), (int) (ntot % 1000000000), timediff(t0, t1));
fflush(stdout);
}
if(iter > MAXITER)
{
printf("failed to converge in neighbour iteration in density()\n");
fflush(stdout);
endrun(1155);
}
}
}
while(ntot > 0);
myfree(Todo);
myfree(Right);
myfree(Left);
myfree(DataNodeList);
myfree(DataIndexTable);
myfree(Dist2list);
myfree(Ngblist);
All.DesNumNgb = save_DesNumNgb; /* restore it */
}
/* This function updates the weights for SN before exploding. This is
* necessary due to the fact that gas particles neighbours of a given star
* could have been transformed into stars and they need to be taken off the
* neighbour list for the exploding star.
*/
void cs_update_weights(void)
{
MyFloat *Left, *Right;
int i, j, ndone, ndone_flag, npleft, dummy, iter = 0;
int ngrp, sendTask, recvTask, place, nexport, nimport;
long long ntot;
double dmax1, dmax2;
double desnumngb;
if(ThisTask == 0)
{
printf("... start update weights phase = %d ...\n", Flag_phase);
fflush(stdout);
}
Left = (MyFloat *) mymalloc(NumPart * sizeof(MyFloat));
Right = (MyFloat *) mymalloc(NumPart * sizeof(MyFloat));
for(i = FirstActiveParticle; i >= 0; i = NextActiveParticle[i])
{
if(P[i].Type == 6 || P[i].Type == 7)
{
Left[i] = Right[i] = 0;
}
}
/* allocate buffers to arrange communication */
Ngblist = (int *) mymalloc(NumPart * sizeof(int));
R2ngblist = (double *) mymalloc(NumPart * sizeof(double));
All.BunchSize =
(int) ((All.BufferSize * 1024 * 1024) / (sizeof(struct data_index) + sizeof(struct data_nodelist) +
sizeof(struct updateweight_in) +
sizeof(struct updateweight_out) +
sizemax(sizeof(struct updateweight_in),
sizeof(struct updateweight_out))));
DataIndexTable = (struct data_index *) mymalloc(All.BunchSize * sizeof(struct data_index));
DataNodeList = (struct data_nodelist *) mymalloc(All.BunchSize * sizeof(struct data_nodelist));
desnumngb = All.DesNumNgb;
/* we will repeat the whole thing for those particles where we didn't find enough neighbours */
do
{
i = FirstActiveParticle; /* begin with this index */
do
{
for(j = 0; j < NTask; j++)
{
Send_count[j] = 0;
Exportflag[j] = -1;
}
/* do local particles and prepare export list */
for(nexport = 0; i >= 0; i = NextActiveParticle[i])
{
if((P[i].Type == 6 || P[i].Type == 7) && P[i].TimeBin >= 0)
{
if(cs_update_weight_evaluate(i, 0, &nexport, Send_count) < 0)
break;
}
}
#ifdef MYSORT
mysort_dataindex(DataIndexTable, nexport, sizeof(struct data_index), data_index_compare);
#else
qsort(DataIndexTable, nexport, sizeof(struct data_index), data_index_compare);
#endif
MPI_Allgather(Send_count, NTask, MPI_INT, Sendcount_matrix, NTask, MPI_INT, MPI_COMM_WORLD);
for(j = 0, nimport = 0, Recv_offset[0] = 0, Send_offset[0] = 0; j < NTask; j++)
{
Recv_count[j] = Sendcount_matrix[j * NTask + ThisTask];
nimport += Recv_count[j];
if(j > 0)
{
Send_offset[j] = Send_offset[j - 1] + Send_count[j - 1];
Recv_offset[j] = Recv_offset[j - 1] + Recv_count[j - 1];
}
}
UpdateweightGet = (struct updateweight_in *) mymalloc(nimport * sizeof(struct updateweight_in));
UpdateweightIn = (struct updateweight_in *) mymalloc(nexport * sizeof(struct updateweight_in));
/* prepare particle data for export */
for(j = 0; j < nexport; j++)
{
place = DataIndexTable[j].Index;
UpdateweightIn[j].Pos[0] = P[place].Pos[0];
UpdateweightIn[j].Pos[1] = P[place].Pos[1];
UpdateweightIn[j].Pos[2] = P[place].Pos[2];
UpdateweightIn[j].Hsml = PPP[place].Hsml;
memcpy(UpdateweightIn[j].NodeList,
DataNodeList[DataIndexTable[j].IndexGet].NodeList, NODELISTLENGTH * sizeof(int));
}
/* exchange particle data */
for(ngrp = 1; ngrp < (1 << PTask); ngrp++)
{
sendTask = ThisTask;
recvTask = ThisTask ^ ngrp;
if(recvTask < NTask)
{
if(Send_count[recvTask] > 0 || Recv_count[recvTask] > 0)
{
/* get the particles */
MPI_Sendrecv(&UpdateweightIn[Send_offset[recvTask]],
Send_count[recvTask] * sizeof(struct updateweight_in), MPI_BYTE,
recvTask, TAG_DENS_A,
&UpdateweightGet[Recv_offset[recvTask]],
Recv_count[recvTask] * sizeof(struct updateweight_in), MPI_BYTE,
recvTask, TAG_DENS_A, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
}
}
myfree(UpdateweightIn);
UpdateweightResult =
(struct updateweight_out *) mymalloc(nimport * sizeof(struct updateweight_out));
UpdateweightOut = (struct updateweight_out *) mymalloc(nexport * sizeof(struct updateweight_out));
/* now do the particles that were sent to us */
for(j = 0; j < nimport; j++)
cs_update_weight_evaluate(j, 1, &dummy, &dummy);
if(i < 0)
ndone_flag = 1;
else
ndone_flag = 0;
MPI_Allreduce(&ndone_flag, &ndone, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
/* get the result */
for(ngrp = 1; ngrp < (1 << PTask); ngrp++)
{
sendTask = ThisTask;
recvTask = ThisTask ^ ngrp;
if(recvTask < NTask)
{
if(Send_count[recvTask] > 0 || Recv_count[recvTask] > 0)
{
/* send the results */
MPI_Sendrecv(&UpdateweightResult[Recv_offset[recvTask]],
Recv_count[recvTask] * sizeof(struct updateweight_out),
MPI_BYTE, recvTask, TAG_DENS_B,
&UpdateweightOut[Send_offset[recvTask]],
Send_count[recvTask] * sizeof(struct updateweight_out),
MPI_BYTE, recvTask, TAG_DENS_B, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
}
}
/* add the result to the local particles */
for(j = 0; j < nexport; j++)
{
place = DataIndexTable[j].Index;
PPP[place].n.dNumNgb += UpdateweightOut[j].Ngb;
}
myfree(UpdateweightOut);
myfree(UpdateweightResult);
myfree(UpdateweightGet);
}
while(ndone < NTask);
/* do final operations on results */
for(i = FirstActiveParticle, npleft = 0; i >= 0; i = NextActiveParticle[i])
{
if(P[i].Type == 6 || P[i].Type == 7)
{
#ifdef FLTROUNDOFFREDUCTION
PPP[i].n.NumNgb = FLT(PPP[i].n.dNumNgb);
#endif
/* now check whether we had enough neighbours */
if(PPP[i].n.NumNgb < (desnumngb - All.MaxNumNgbDeviation) ||
(PPP[i].n.NumNgb > (desnumngb + All.MaxNumNgbDeviation)
&& PPP[i].Hsml > (1.01 * All.MinGasHsml)))
{
/* need to redo this particle */
npleft++;
if(Left[i] > 0 && Right[i] > 0)
if((Right[i] - Left[i]) < 1.0e-3 * Left[i])
{
/* this one should be ok */
npleft--;
P[i].TimeBin = -P[i].TimeBin - 1; /* Mark as inactive */
continue;
}
if(PPP[i].n.NumNgb < (desnumngb - All.MaxNumNgbDeviation))
Left[i] = DMAX(PPP[i].Hsml, Left[i]);
else
{
if(Right[i] != 0)
{
if(PPP[i].Hsml < Right[i])
Right[i] = PPP[i].Hsml;
}
else
Right[i] = PPP[i].Hsml;
}
if(iter >= MAXITER - 10)
{
printf
("i=%d task=%d ID=%d Hsml=%g Left=%g Right=%g Ngbs=%g Right-Left=%g\n pos=(%g|%g|%g)\n",
i, ThisTask, (int) P[i].ID, PPP[i].Hsml, Left[i], Right[i],
(float) PPP[i].n.NumNgb, Right[i] - Left[i], P[i].Pos[0], P[i].Pos[1], P[i].Pos[2]);
fflush(stdout);
}
if(Right[i] > 0 && Left[i] > 0)
PPP[i].Hsml = pow(0.5 * (pow(Left[i], 3) + pow(Right[i], 3)), 1.0 / 3);
else
{
if(Right[i] == 0 && Left[i] == 0)
endrun(8188); /* can't occur */
if(Right[i] == 0 && Left[i] > 0)
{
PPP[i].Hsml *= 1.26;
}
if(Right[i] > 0 && Left[i] == 0)
{
PPP[i].Hsml /= 1.26;
}
}
if(PPP[i].Hsml < All.MinGasHsml)
PPP[i].Hsml = All.MinGasHsml;
}
else
P[i].TimeBin = -P[i].TimeBin - 1; /* Mark as inactive */
/* CECILIA */
if(iter == MAXITER)
{
int old_type;
old_type = P[i].Type;
P[i].Type = 4; /* no SN mark any more */
PPP[i].n.NumNgb = 0;
printf("part=%d of type=%d was assigned NumNgb=%g and type=%d\n", i, old_type,
PPP[i].n.NumNgb, P[i].Type);
}
}
}
sumup_large_ints(1, &npleft, &ntot);
if(ntot > 0)
{
iter++;
if(iter > 0 && ThisTask == 0)
{
printf("ngb iteration %d: need to repeat for %d%09d particles.\n", iter,
(int) (ntot / 1000000000), (int) (ntot % 1000000000));
fflush(stdout);
}
if(iter > MAXITER)
{
#ifndef CS_FEEDBACK
printf("failed to converge in neighbour iteration in update_weights \n");
fflush(stdout);
endrun(1155);
#else
/* CECILIA */
if(Flag_phase == 2) /* HOT */
{
printf("Not enough hot neighbours for energy/metal distribution part=%d Type=%d\n", i,
P[i].Type);
fflush(stdout);
break;
/* endrun(1156); */
}
else
{
printf("Not enough cold neighbours for energy/metal distribution part=%d Type=%d\n", i,
P[i].Type);
fflush(stdout);
break;
}
#endif
}
}
}
while(ntot > 0);
myfree(DataNodeList);
myfree(DataIndexTable);
myfree(R2ngblist);
myfree(Ngblist);
myfree(Right);
myfree(Left);
/* mark as active again */
for(i = FirstActiveParticle; i >= 0; i = NextActiveParticle[i])
if(P[i].TimeBin < 0)
P[i].TimeBin = -P[i].TimeBin - 1;
/* collect some timing information */
if(ThisTask == 0)
{
printf("... update weights phase = %d done...\n", Flag_phase);
fflush(stdout);
}
}
/* reshift positions to center of mass */
void SCF_do_center_of_mass_correction(double fac_rad, double start_rad, double fac_part, int max_iter)
{
int i, k, n;
MyDouble pos[3], pos_all[3];
int num, num_all;
int iter=0;
double rad, max_rad=start_rad;
if (ThisTask==0)
printf("SCF center of mass correction...\n");
for(n = 0; n < 6; n++)
SCF_n_type[n] = 0;
for(n = 0; n < NumPart; n++)
SCF_n_type[P[n].Type]++;
sumup_large_ints(6, SCF_n_type, SCF_ntot_type_all);
if (ThisTask==0)
printf("total number of DM particles=%lld\n", SCF_ntot_type_all[1]);
while(iter < max_iter)
{
num=0;
for(k = 0; k < 3; k++)
{
pos[k]=0.0;
pos_all[k]=0.0;
}
for(i = 0; i < NumPart; i++)
{
rad = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
/* consider only DM particles of SCF coefficients */
if (P[i].Type != 1 || rad > max_rad)
continue;
for(k = 0; k < 3; k++)
pos[k]+=P[i].Pos[k];
num++;
}
MPI_Allreduce(&pos[0], &pos_all[0], 3, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(&num, &num_all, 3, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
#ifdef DEBUG
if (ThisTask==0)
{
printf("temp. center of mass = (%g|%g|%g) iter=%d num_all=%d\n", pos_all[0]/num_all, pos_all[1]/num_all, pos_all[2]/num_all, iter, num_all);
printf("done.\n");
}
#endif
for(i = 0; i < NumPart; i++)
{
for(k = 0; k < 3; k++)
P[i].Pos[k]-=pos_all[k]/num_all;
}
if (num_all < fac_part*SCF_ntot_type_all[1])
break;
max_rad*=fac_rad;
iter++;
}
if (ThisTask==0)
{
printf("SCF center of mass = (%g|%g|%g) iter=%d\n", pos_all[0]/num_all, pos_all[1]/num_all, pos_all[2]/num_all, iter);
printf("done.\n");
}
}
