void cells_resort_particles(int global_flag)
{
CELL_TRACE(fprintf(stderr, "%d: entering cells_resort_particles %d\n", this_node, global_flag));
invalidate_ghosts();
particle_invalidate_part_node();
n_verlet_updates++;
switch (cell_structure.type) {
case CELL_STRUCTURE_LAYERED:
layered_exchange_and_sort_particles(global_flag);
break;
case CELL_STRUCTURE_NSQUARE:
nsq_balance_particles(global_flag);
break;
case CELL_STRUCTURE_DOMDEC:
dd_exchange_and_sort_particles(global_flag);
break;
}
#ifdef ADDITIONAL_CHECKS
/* at the end of the day, everything should be consistent again */
check_particle_consistency();
#endif
ghost_communicator(&cell_structure.ghost_cells_comm);
ghost_communicator(&cell_structure.exchange_ghosts_comm);
resort_particles = 0;
rebuild_verletlist = 1;
on_resort_particles();
CELL_TRACE(dump_particle_ordering());
CELL_TRACE(fprintf(stderr, "%d: leaving cells_resort_particles\n", this_node));
}
void nsq_balance_particles()
{
int i, n, surplus, s_node, tmp, lack, l_node, transfer;
int pp = cells_get_n_particles();
int *ppnode = malloc(n_nodes*sizeof(int));
/* minimal difference between node shares */
int minshare = n_total_particles/n_nodes;
int maxshare = minshare + 1;
CELL_TRACE(fprintf(stderr, "%d: nsq_balance_particles: load %d-%d\n", this_node, minshare, maxshare));
MPI_Allgather(&pp, 1, MPI_INT, ppnode, 1, MPI_INT, MPI_COMM_WORLD);
for (;;) {
/* find node with most excessive particles */
surplus = -1;
s_node = -1;
for (n = 0; n < n_nodes; n++) {
tmp = ppnode[n] - minshare;
CELL_TRACE(fprintf(stderr, "%d: nsq_balance_particles: node %d has %d\n", this_node, n, ppnode[n]));
if (tmp > surplus) {
surplus = tmp;
s_node = n;
}
}
CELL_TRACE(fprintf(stderr, "%d: nsq_balance_particles: excess %d on node %d\n", this_node, surplus, s_node));
/* find node with most lacking particles */
lack = -1;
l_node = -1;
for (n = 0; n < n_nodes; n++) {
tmp = maxshare - ppnode[n];
if (tmp > lack) {
lack = tmp;
l_node = n;
}
}
CELL_TRACE(fprintf(stderr, "%d: nsq_balance_particles: lack %d on node %d\n", this_node, lack, l_node));
/* should not happen: minshare or maxshare wrong or more likely,
the algorithm */
if (s_node == -1 || l_node == -1) {
fprintf(stderr, "%d: Particle load balancing failed\n", this_node);
break;
}
/* exit if all nodes load is withing min and max share */
if (lack <= 1 && surplus <= 1)
break;
transfer = lack < surplus ? lack : surplus;
if (s_node == this_node) {
ParticleList send_buf;
init_particlelist(&send_buf);
realloc_particlelist(&send_buf, send_buf.n = transfer);
for (i = 0; i < transfer; i++) {
memcpy(&send_buf.part[i], &local->part[--local->n], sizeof(Particle));
}
realloc_particlelist(local, local->n);
update_local_particles(local);
send_particles(&send_buf, l_node);
#ifdef ADDITIONAL_CHECKS
check_particle_consistency();
#endif
}
else if (l_node == this_node) {
recv_particles(local, s_node);
#ifdef ADDITIONAL_CHECKS
check_particle_consistency();
#endif
}
ppnode[s_node] -= transfer;
ppnode[l_node] += transfer;
}
CELL_TRACE(fprintf(stderr, "%d: nsq_balance_particles: done\n", this_node));
free(ppnode);
}
