static void lb_migrate_points(const par::communicator& comm, list_type& pl, LBData& lbd) { const unsigned int num_proc = comm.size(); const unsigned int dim = point_type::dim; // Sendcounts should be already filled // Perform alltoall for recvcounts from sendcounts lbd.recvcounts.resize(num_proc); comm.alltoall(lbd.sendcounts.data(), 1, lbd.recvcounts.data(), 1); // Resize the send buffer lbd.send.resize(pl.size() * dim); // Pack up the send buffer { unsigned int i = 0; for (const point_type& p : pl) for (unsigned int d=0; d<dim; d++, i++) lbd.send[i] = p[d]; } // Offsets in memory (for alltoallv) lbd.sdispls[0] = 0; for (unsigned int i=1; i<num_proc; i++) lbd.sdispls[i] = lbd.sdispls[i-1] + lbd.sendcounts[i-1]; lbd.rdispls[0] = 0; for (unsigned int i=1; i<num_proc; i++) lbd.rdispls[i] = lbd.rdispls[i-1] + lbd.recvcounts[i-1]; // Create receive buffer const int total_recv = std::accumulate(lbd.recvcounts.begin(), lbd.recvcounts.end(), 0); lbd.recv.resize(total_recv); // Communicate points (alltoallv) comm.alltoallv(lbd.send.data(), lbd.sendcounts.data(), lbd.sdispls.data(), lbd.recv.data(), lbd.recvcounts.data(), lbd.rdispls.data()); // Clear out current points and reload from the receive buffer pl.clear(); for (unsigned int i=0; i<lbd.recv.size(); i+=dim) pl.emplace_back(point_type(lbd.recv[i], lbd.recv[i+1])); }
static void rcb1d_recurse(const par::communicator& comm, unsigned int dimen, list_type& pl, LBData& lbd) { const unsigned int my_rank = comm.rank(); const unsigned int num_proc = comm.size(); // First: sort list sort_point2d_list(dimen, pl); if (num_proc == 1) return; lbd.change_comm(comm, pl.size()); const list_type::size_type orig_num_point = lbd.size[my_rank]; const unsigned int dim = point_type::dim; const list_type::size_type glob_num_point = std::accumulate(lbd.size.begin(), lbd.size.end(), 0); lbd.cut.resize(num_proc); lbd.bin.resize(num_proc); const real ratio = static_cast<real>((num_proc+1) / 2) / num_proc; list_type::size_type this_offset = ratio * orig_num_point; const unsigned int rank_midpt = ratio * num_proc; { real cut = 0; int use = 0; if (orig_num_point > 0) { const point_type& p = *std::next(pl.begin(), this_offset); cut = p[dimen]; use = 1; } comm.allgather(&cut, 1, lbd.cut.data(), 1); comm.allgather(&use, 1, lbd.bin.data(), 1); } // std::cout << "this_offset = " << this_offset << std::endl; // std::cout << "ratio = " << ratio << std::endl; // std::cout << "cuts: "; // for (auto& p : lbd.cut) std::cout << p << " "; // std::cout << std::endl; // std::cout << "use: "; // for (auto& p : lbd.bin) std::cout << p << " "; // std::cout << std::endl; real midpt = 0.; // std::cout << "my_rank = " << my_rank << std::endl; // std::cout << "num_proc = " << num_proc << std::endl; for (unsigned int i=0; i<num_proc; i++) if (lbd.bin[i] != 0) // include this point midpt += lbd.cut[i] * static_cast<real>(lbd.size[i]) / static_cast<real>(glob_num_point); list_type other; list_type::iterator pit = std::find_if(pl.begin(), pl.end(), [midpt,dimen](const point_type& p) { return p[dimen] >= midpt; }); list_type::size_type offset; if (my_rank < rank_midpt) { other.splice(other.begin(), pl, pit, pl.end()); offset = pl.size(); } else { other.splice(other.begin(), pl, pl.begin(), pit); offset = other.size(); } // std::cout << "pl: "; // for (auto& p : pl) std::cout << p << " "; // std::cout << std::endl; // std::cout << "other: "; // for (auto& p : other) std::cout << p << " "; // std::cout << std::endl; // std::cout << "offset = " << offset << std::endl; lbd.send.resize(other.size() * dim); { list_type::iterator it=other.begin(); list_type::size_type i = 0; while (it != other.end()) { const point_type& p = *it; for (unsigned int d=0; d<dim; d++, i++) lbd.send[i] = p[d]; other.erase(it++); } } // std::cout << "send: "; // for (auto& p : lbd.send) std::cout << p << " "; // std::cout << std::endl; // std::cout << "midpt = " << midpt << std::endl; // std::cout << "rank_midpt = " << rank_midpt << std::endl; // std::cout << "this_offset = " << this_offset << std::endl; // Pack up points lbd.sendcounts.resize(num_proc); // Zero out std::fill(lbd.sendcounts.begin(), lbd.sendcounts.end(), 0); if (my_rank < rank_midpt) { // Left side for (unsigned int i=offset; i<orig_num_point; i++) { const unsigned int inc = (i-offset) % (num_proc - rank_midpt); lbd.sendcounts[rank_midpt + inc] += dim; } } else { // Right side for (unsigned int i=0; i<offset; i++) { lbd.sendcounts[i % rank_midpt] += dim; } } // std::cout << "sendcounts: "; // for (auto& p : lbd.sendcounts) std::cout << p << " "; // std::cout << std::endl; // alltoall recvcounts comm.alltoall(lbd.sendcounts.data(), 1, lbd.recvcounts.data(), 1); // std::cout << "recvcounts: "; // for (auto& p : lbd.recvcounts) std::cout << p << " "; // std::cout << std::endl; // Offsets in memory (for alltoallv) lbd.sdispls[0] = 0; for (unsigned int i=1; i<num_proc; i++) lbd.sdispls[i] = lbd.sdispls[i-1] + lbd.sendcounts[i-1]; lbd.rdispls[0] = 0; for (unsigned int i=1; i<num_proc; i++) lbd.rdispls[i] = lbd.rdispls[i-1] + lbd.recvcounts[i-1]; // Create receive buffer const int total_recv = std::accumulate(lbd.recvcounts.begin(), lbd.recvcounts.end(), 0); lbd.recv.resize(total_recv); // Communicate points (alltoallv) comm.alltoallv(lbd.send.data(), lbd.sendcounts.data(), lbd.sdispls.data(), lbd.recv.data(), lbd.recvcounts.data(), lbd.rdispls.data()); // Clear out current points and reload from the receive buffer for (unsigned int i=0; i<lbd.recv.size(); i+=dim) pl.emplace_back(point_type(lbd.recv[i], lbd.recv[i+1])); // std::cout << "points: "; // for (auto& p : pl) std::cout << p << " "; // std::cout << std::endl; // std::cout << "--------------" << std::endl; par::communicator halfcomm(comm, my_rank < rank_midpt); rcb1d_recurse(halfcomm, dimen, pl, lbd); }