static void lb_migrate_points(const par::communicator& comm, list_type& pl, LBData& lbd)
{
const unsigned int num_proc = comm.size();
const unsigned int dim = point_type::dim;
// Sendcounts should be already filled
// Perform alltoall for recvcounts from sendcounts
lbd.recvcounts.resize(num_proc);
comm.alltoall(lbd.sendcounts.data(), 1, lbd.recvcounts.data(), 1);
// Resize the send buffer
lbd.send.resize(pl.size() * dim);
// Pack up the send buffer
{
unsigned int i = 0;
for (const point_type& p : pl)
for (unsigned int d=0; d<dim; d++, i++)
lbd.send[i] = p[d];
}
// Offsets in memory (for alltoallv)
lbd.sdispls[0] = 0;
for (unsigned int i=1; i<num_proc; i++)
lbd.sdispls[i] = lbd.sdispls[i-1] + lbd.sendcounts[i-1];
lbd.rdispls[0] = 0;
for (unsigned int i=1; i<num_proc; i++)
lbd.rdispls[i] = lbd.rdispls[i-1] + lbd.recvcounts[i-1];
// Create receive buffer
const int total_recv = std::accumulate(lbd.recvcounts.begin(), lbd.recvcounts.end(), 0);
lbd.recv.resize(total_recv);
// Communicate points (alltoallv)
comm.alltoallv(lbd.send.data(), lbd.sendcounts.data(), lbd.sdispls.data(),
lbd.recv.data(), lbd.recvcounts.data(), lbd.rdispls.data());
// Clear out current points and reload from the receive buffer
pl.clear();
for (unsigned int i=0; i<lbd.recv.size(); i+=dim)
pl.emplace_back(point_type(lbd.recv[i], lbd.recv[i+1]));
}
static void rcb1d_recurse(const par::communicator& comm, unsigned int dimen,
list_type& pl, LBData& lbd)
{
const unsigned int my_rank = comm.rank();
const unsigned int num_proc = comm.size();
// First: sort list
sort_point2d_list(dimen, pl);
if (num_proc == 1) return;
lbd.change_comm(comm, pl.size());
const list_type::size_type orig_num_point = lbd.size[my_rank];
const unsigned int dim = point_type::dim;
const list_type::size_type glob_num_point =
std::accumulate(lbd.size.begin(), lbd.size.end(), 0);
lbd.cut.resize(num_proc);
lbd.bin.resize(num_proc);
const real ratio = static_cast<real>((num_proc+1) / 2) / num_proc;
list_type::size_type this_offset = ratio * orig_num_point;
const unsigned int rank_midpt = ratio * num_proc;
{
real cut = 0;
int use = 0;
if (orig_num_point > 0) {
const point_type& p = *std::next(pl.begin(), this_offset);
cut = p[dimen];
use = 1;
}
comm.allgather(&cut, 1, lbd.cut.data(), 1);
comm.allgather(&use, 1, lbd.bin.data(), 1);
}
// std::cout << "this_offset = " << this_offset << std::endl;
// std::cout << "ratio = " << ratio << std::endl;
// std::cout << "cuts: ";
// for (auto& p : lbd.cut) std::cout << p << " ";
// std::cout << std::endl;
// std::cout << "use: ";
// for (auto& p : lbd.bin) std::cout << p << " ";
// std::cout << std::endl;
real midpt = 0.;
// std::cout << "my_rank = " << my_rank << std::endl;
// std::cout << "num_proc = " << num_proc << std::endl;
for (unsigned int i=0; i<num_proc; i++)
if (lbd.bin[i] != 0) // include this point
midpt += lbd.cut[i] *
static_cast<real>(lbd.size[i]) /
static_cast<real>(glob_num_point);
list_type other;
list_type::iterator pit = std::find_if(pl.begin(), pl.end(),
[midpt,dimen](const point_type& p)
{ return p[dimen] >= midpt; });
list_type::size_type offset;
if (my_rank < rank_midpt) {
other.splice(other.begin(), pl, pit, pl.end());
offset = pl.size();
}
else {
other.splice(other.begin(), pl, pl.begin(), pit);
offset = other.size();
}
// std::cout << "pl: ";
// for (auto& p : pl) std::cout << p << " ";
// std::cout << std::endl;
// std::cout << "other: ";
// for (auto& p : other) std::cout << p << " ";
// std::cout << std::endl;
// std::cout << "offset = " << offset << std::endl;
lbd.send.resize(other.size() * dim);
{
list_type::iterator it=other.begin();
list_type::size_type i = 0;
while (it != other.end()) {
const point_type& p = *it;
for (unsigned int d=0; d<dim; d++, i++)
lbd.send[i] = p[d];
other.erase(it++);
}
}
// std::cout << "send: ";
// for (auto& p : lbd.send) std::cout << p << " ";
// std::cout << std::endl;
// std::cout << "midpt = " << midpt << std::endl;
// std::cout << "rank_midpt = " << rank_midpt << std::endl;
// std::cout << "this_offset = " << this_offset << std::endl;
// Pack up points
lbd.sendcounts.resize(num_proc);
// Zero out
std::fill(lbd.sendcounts.begin(), lbd.sendcounts.end(), 0);
if (my_rank < rank_midpt) {
// Left side
for (unsigned int i=offset; i<orig_num_point; i++) {
const unsigned int inc = (i-offset) % (num_proc - rank_midpt);
lbd.sendcounts[rank_midpt + inc] += dim;
}
}
else {
// Right side
for (unsigned int i=0; i<offset; i++) {
lbd.sendcounts[i % rank_midpt] += dim;
}
}
// std::cout << "sendcounts: ";
// for (auto& p : lbd.sendcounts) std::cout << p << " ";
// std::cout << std::endl;
// alltoall recvcounts
comm.alltoall(lbd.sendcounts.data(), 1, lbd.recvcounts.data(), 1);
// std::cout << "recvcounts: ";
// for (auto& p : lbd.recvcounts) std::cout << p << " ";
// std::cout << std::endl;
// Offsets in memory (for alltoallv)
lbd.sdispls[0] = 0;
for (unsigned int i=1; i<num_proc; i++)
lbd.sdispls[i] = lbd.sdispls[i-1] + lbd.sendcounts[i-1];
lbd.rdispls[0] = 0;
for (unsigned int i=1; i<num_proc; i++)
lbd.rdispls[i] = lbd.rdispls[i-1] + lbd.recvcounts[i-1];
// Create receive buffer
const int total_recv = std::accumulate(lbd.recvcounts.begin(), lbd.recvcounts.end(), 0);
lbd.recv.resize(total_recv);
// Communicate points (alltoallv)
comm.alltoallv(lbd.send.data(), lbd.sendcounts.data(), lbd.sdispls.data(),
lbd.recv.data(), lbd.recvcounts.data(), lbd.rdispls.data());
// Clear out current points and reload from the receive buffer
for (unsigned int i=0; i<lbd.recv.size(); i+=dim)
pl.emplace_back(point_type(lbd.recv[i], lbd.recv[i+1]));
// std::cout << "points: ";
// for (auto& p : pl) std::cout << p << " ";
// std::cout << std::endl;
// std::cout << "--------------" << std::endl;
par::communicator halfcomm(comm, my_rank < rank_midpt);
rcb1d_recurse(halfcomm, dimen, pl, lbd);
}
