// create and return the custom MPI data type
MPI_Datatype get_mpi_datatype()
{
if (!is_committed)
{
#if defined(MPI_VERSION) && MPI_VERSION >= 2
BOOST_MPI_CHECK_RESULT(MPI_Type_create_struct,
(
addresses.size(),
boost::serialization::detail::get_data(lengths),
boost::serialization::detail::get_data(addresses),
boost::serialization::detail::get_data(types),
&datatype_
));
#else
BOOST_MPI_CHECK_RESULT(MPI_Type_struct,
(
addresses.size(),
boost::serialization::detail::get_data(lengths),
boost::serialization::detail::get_data(addresses),
boost::serialization::detail::get_data(types),
&datatype_
));
#endif
BOOST_MPI_CHECK_RESULT(MPI_Type_commit,(&datatype_));
is_committed = true;
}
return datatype_;
}
mpi_datatype_primitive(void const* orig)
: is_committed(false),
origin()
{
#if defined(MPI_VERSION) && MPI_VERSION >= 2
BOOST_MPI_CHECK_RESULT(MPI_Get_address,(const_cast<void*>(orig), &origin));
#else
BOOST_MPI_CHECK_RESULT(MPI_Address,(const_cast<void*>(orig), &origin));
#endif
}
void save_impl(void const * p, MPI_Datatype t, int l)
{
BOOST_ASSERT ( !is_committed );
// store address, type and length
MPI_Aint a;
#if defined(MPI_VERSION) && MPI_VERSION >= 2
BOOST_MPI_CHECK_RESULT(MPI_Get_address,(const_cast<void*>(p), &a));
#else
BOOST_MPI_CHECK_RESULT(MPI_Address,(const_cast<void*>(p), &a));
#endif
addresses.push_back(a-origin);
types.push_back(t);
lengths.push_back(l);
}
bool communicator::has_cartesian_topology() const
{
int status;
BOOST_MPI_CHECK_RESULT(MPI_Topo_test, ((MPI_Comm)*this, &status));
return status == MPI_CART;
}
communicator::communicator(const MPI_Comm& comm, comm_create_kind kind)
{
if (comm == MPI_COMM_NULL)
/* MPI_COMM_NULL indicates that the communicator is not usable. */
return;
switch (kind) {
case comm_duplicate:
{
MPI_Comm newcomm;
BOOST_MPI_CHECK_RESULT(MPI_Comm_dup, (comm, &newcomm));
comm_ptr.reset(new MPI_Comm(newcomm), comm_free());
MPI_Errhandler_set(newcomm, MPI_ERRORS_RETURN);
break;
}
case comm_take_ownership:
comm_ptr.reset(new MPI_Comm(comm), comm_free());
break;
case comm_attach:
comm_ptr.reset(new MPI_Comm(comm));
break;
}
}
bool operator==(const communicator& comm1, const communicator& comm2)
{
int result;
BOOST_MPI_CHECK_RESULT(MPI_Comm_compare,
((MPI_Comm)comm1, (MPI_Comm)comm2, &result));
return result == MPI_IDENT;
}
int
cartesian_communicator::ndims() const {
int n = -1;
BOOST_MPI_CHECK_RESULT(MPI_Cartdim_get,
(MPI_Comm(*this), &n));
return n;
}
cartesian_communicator::cartesian_communicator(const communicator& comm,
const cartesian_topology& topology,
bool reorder )
: communicator(MPI_COMM_NULL, comm_attach)
{
std::vector<int> dims(topology.size());
std::vector<int> periodic(topology.size());
int tsz = topology.size();
for(int i = 0; i < tsz; ++i) {
dims[i] = topology[i].size;
periodic[i] = topology[i].periodic;
}
// Fill the gaps, if any
if (std::count(dims.begin(), dims.end(), 0) > 0) {
cartesian_dimensions(comm, dims);
}
MPI_Comm newcomm;
BOOST_MPI_CHECK_RESULT(MPI_Cart_create,
((MPI_Comm)comm, dims.size(),
c_data(dims), c_data(periodic),
int(reorder), &newcomm));
if(newcomm != MPI_COMM_NULL) {
comm_ptr.reset(new MPI_Comm(newcomm), comm_free());
}
}
std::vector<int>
cartesian_communicator::coordinates(int rk) const {
std::vector<int> cbuf(ndims());
BOOST_MPI_CHECK_RESULT(MPI_Cart_coords,
(MPI_Comm(*this), rk, cbuf.size(), c_data(cbuf) ));
return cbuf;
}
communicator communicator::split(int color, int key) const
{
MPI_Comm newcomm;
BOOST_MPI_CHECK_RESULT(MPI_Comm_split,
(MPI_Comm(*this), color, key, &newcomm));
return communicator(newcomm, comm_take_ownership);
}
status communicator::probe(int source, int tag) const
{
status stat;
BOOST_MPI_CHECK_RESULT(MPI_Probe,
(source, tag, MPI_Comm(*this), &stat.m_status));
return stat;
}
communicator::communicator(const communicator& comm,
const boost::mpi::group& subgroup)
{
MPI_Comm newcomm;
BOOST_MPI_CHECK_RESULT(MPI_Comm_create,
((MPI_Comm)comm, (MPI_Group)subgroup, &newcomm));
comm_ptr.reset(new MPI_Comm(newcomm), comm_free());
}
std::pair<int, int>
cartesian_communicator::shifted_ranks(int dim, int disp) const {
std::pair<int, int> r(-1,-1);
assert(0 <= dim && dim < ndims());
BOOST_MPI_CHECK_RESULT(MPI_Cart_shift,
(MPI_Comm(*this), dim, disp, &(r.first), &(r.second)));
return r;
}
status communicator::recv(int source, int tag) const
{
status stat;
BOOST_MPI_CHECK_RESULT(MPI_Recv,
(MPI_BOTTOM, 0, MPI_PACKED,
source, tag, MPI_Comm(*this), &stat.m_status));
return stat;
}
request communicator::irecv(int source, int tag) const
{
request req;
BOOST_MPI_CHECK_RESULT(MPI_Irecv,
(MPI_BOTTOM, 0, MPI_PACKED,
source, tag, MPI_Comm(*this), &req.m_requests[0]));
return req;
}
request communicator::isend(int dest, int tag) const
{
request req;
BOOST_MPI_CHECK_RESULT(MPI_Isend,
(MPI_BOTTOM, 0, MPI_PACKED,
dest, tag, MPI_Comm(*this), &req.m_requests[0]));
return req;
}
request communicator::isend<content>(int dest, int tag, const content& c) const
{
request req;
BOOST_MPI_CHECK_RESULT(MPI_Isend,
(MPI_BOTTOM, 1, c.get_mpi_datatype(),
dest, tag, MPI_Comm(*this), &req.m_requests[0]));
return req;
}
optional<graph_communicator> communicator::as_graph_communicator() const
{
int status;
BOOST_MPI_CHECK_RESULT(MPI_Topo_test, ((MPI_Comm)*this, &status));
if (status == MPI_GRAPH)
return graph_communicator(comm_ptr);
else
return optional<graph_communicator>();
}
explicit user_op(Op& op)
{
BOOST_MPI_CHECK_RESULT(MPI_Op_create,
(&user_op<Op, T>::perform,
is_commutative<Op, T>::value,
&mpi_op));
op_ptr = &op;
}
status communicator::probe(int source, int tag) const
{
typedef optional<status> result_type;
status stat;
BOOST_MPI_CHECK_RESULT(MPI_Probe,
(source, tag, MPI_Comm(*this), &stat.m_status));
return stat;
}
void
all_gather_impl(const communicator& comm, const T* in_values, int n,
T* out_values, mpl::true_)
{
MPI_Datatype type = get_mpi_datatype<T>(*in_values);
BOOST_MPI_CHECK_RESULT(MPI_Allgather,
(const_cast<T*>(in_values), n, type,
out_values, n, type, comm));
}
status
communicator::recv<const content>(int source, int tag, const content& c) const
{
status stat;
BOOST_MPI_CHECK_RESULT(MPI_Recv,
(MPI_BOTTOM, 1, c.get_mpi_datatype(),
source, tag, MPI_Comm(*this), &stat.m_status));
return stat;
}
void
scatter_impl(const communicator& comm, const T* in_values, T* out_values,
int n, int root, mpl::true_)
{
MPI_Datatype type = get_mpi_datatype<T>(*in_values);
BOOST_MPI_CHECK_RESULT(MPI_Scatter,
(const_cast<T*>(in_values), n, type,
out_values, n, type, root, comm));
}
int
cartesian_communicator::rank(const std::vector<int>& coords ) const {
int r = -1;
assert(int(coords.size()) == ndims());
BOOST_MPI_CHECK_RESULT(MPI_Cart_rank,
(MPI_Comm(*this), c_data(const_cast<std::vector<int>&>(coords)),
&r));
return r;
}
optional<intercommunicator> communicator::as_intercommunicator() const
{
int flag;
BOOST_MPI_CHECK_RESULT(MPI_Comm_test_inter, ((MPI_Comm)*this, &flag));
if (flag)
return intercommunicator(comm_ptr);
else
return optional<intercommunicator>();
}
void
scan_impl(const communicator& comm, const T* in_values, int n, T* out_values,
Op op, mpl::true_ /*is_mpi_op*/, mpl::true_ /*is_mpi_datatype*/)
{
BOOST_MPI_CHECK_RESULT(MPI_Scan,
(const_cast<T*>(in_values), out_values, n,
boost::mpi::get_mpi_datatype<T>(*in_values),
(is_mpi_op<Op, T>::op()), comm));
}
void
broadcast_impl(const communicator& comm, T* values, int n, int root,
mpl::true_)
{
BOOST_MPI_CHECK_RESULT(MPI_Bcast,
(values, n,
boost::mpi::get_mpi_datatype<T>(*values),
root, MPI_Comm(comm)));
}
void
reduce_impl(const communicator& comm, const T* in_values, int n, Op op,
int root, mpl::true_ /*is_mpi_op*/, mpl::true_/*is_mpi_datatype*/)
{
BOOST_MPI_CHECK_RESULT(MPI_Reduce,
(const_cast<T*>(in_values), 0, n,
boost::mpi::get_mpi_datatype<T>(*in_values),
(is_mpi_op<Op, T>::op()), root, comm));
}
void
all_gather_impl(const communicator& comm, const T* in_values, int n,
T* out_values, int const* sizes, int const* skips, mpl::false_)
{
int nproc = comm.size();
// first, gather all size, these size can be different for
// each process
packed_oarchive oa(comm);
for (int i = 0; i < n; ++i) {
oa << in_values[i];
}
std::vector<int> oasizes(nproc);
int oasize = oa.size();
BOOST_MPI_CHECK_RESULT(MPI_Allgather,
(&oasize, 1, MPI_INTEGER,
c_data(oasizes), 1, MPI_INTEGER,
MPI_Comm(comm)));
// Gather the archives, which can be of different sizes, so
// we need to use allgatherv.
// Every thing is contiguous, so the offsets can be
// deduced from the collected sizes.
std::vector<int> offsets(nproc);
sizes2offsets(oasizes, offsets);
packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0));
BOOST_MPI_CHECK_RESULT(MPI_Allgatherv,
(const_cast<void*>(oa.address()), int(oa.size()), MPI_BYTE,
c_data(recv_buffer), c_data(oasizes), c_data(offsets), MPI_BYTE,
MPI_Comm(comm)));
for (int src = 0; src < nproc; ++src) {
int nb = sizes ? sizes[src] : n;
int skip = skips ? skips[src] : 0;
std::advance(out_values, skip);
if (src == comm.rank()) { // this is our local data
for (int i = 0; i < nb; ++i) {
*out_values++ = *in_values++;
}
} else {
packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]);
for (int i = 0; i < nb; ++i) {
ia >> *out_values++;
}
}
}
}
request
communicator::irecv<const content>(int source, int tag,
const content& c) const
{
request req;
BOOST_MPI_CHECK_RESULT(MPI_Irecv,
(MPI_BOTTOM, 1, c.get_mpi_datatype(),
source, tag, MPI_Comm(*this), &req.m_requests[0]));
return req;
}
