Chris@16: // Copyright (C) 2005, 2006 Douglas Gregor.
Chris@16: 
Chris@16: // Use, modification and distribution is subject to the Boost Software
Chris@16: // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
Chris@16: // http://www.boost.org/LICENSE_1_0.txt)
Chris@16: 
Chris@16: // Message Passing Interface 1.1 -- Section 4.8. All-to-all
Chris@16: #ifndef BOOST_MPI_ALL_TO_ALL_HPP
Chris@16: #define BOOST_MPI_ALL_TO_ALL_HPP
Chris@16: 
Chris@16: #include <boost/mpi/exception.hpp>
Chris@16: #include <boost/mpi/datatype.hpp>
Chris@16: #include <vector>
Chris@16: #include <boost/mpi/packed_oarchive.hpp>
Chris@16: #include <boost/mpi/packed_iarchive.hpp>
Chris@16: #include <boost/mpi/communicator.hpp>
Chris@16: #include <boost/mpi/environment.hpp>
Chris@16: #include <boost/assert.hpp>
Chris@16: #include <boost/mpi/collectives_fwd.hpp>
Chris@16: #include <boost/mpi/allocator.hpp>
Chris@16: 
Chris@16: namespace boost { namespace mpi {
Chris@16:           
Chris@16: namespace detail {
Chris@16:   // We're performaing an all-to-all with a type that has an
Chris@16:   // associated MPI datatype, so we'll use MPI_Alltoall to do all of
Chris@16:   // the work.
Chris@16:   template<typename T>
Chris@16:   void
Chris@16:   all_to_all_impl(const communicator& comm, const T* in_values, int n, 
Chris@16:                   T* out_values, mpl::true_)
Chris@16:   {
Chris@16:     MPI_Datatype type = get_mpi_datatype<T>(*in_values);
Chris@16:     BOOST_MPI_CHECK_RESULT(MPI_Alltoall,
Chris@16:                            (const_cast<T*>(in_values), n, type,
Chris@16:                             out_values, n, type, comm));
Chris@16:   }
Chris@16: 
Chris@16:   // We're performing an all-to-all with a type that does not have an
Chris@16:   // associated MPI datatype, so we'll need to serialize
Chris@16:   // it. Unfortunately, this means that we cannot use MPI_Alltoall, so
Chris@16:   // we'll just have to send individual messages to the other
Chris@16:   // processes.
Chris@16:   template<typename T>
Chris@16:   void
Chris@16:   all_to_all_impl(const communicator& comm, const T* in_values, int n,
Chris@16:                   T* out_values, mpl::false_)
Chris@16:   {
Chris@16:     int size = comm.size();
Chris@16:     int rank = comm.rank();
Chris@16: 
Chris@16:     // The amount of data to be sent to each process
Chris@16:     std::vector<int> send_sizes(size);
Chris@16: 
Chris@16:     // The displacements for each outgoing value.
Chris@16:     std::vector<int> send_disps(size);
Chris@16: 
Chris@16:     // The buffer that will store all of the outgoing values
Chris@16:     std::vector<char, allocator<char> > outgoing;
Chris@16: 
Chris@16:     // Pack the buffer with all of the outgoing values.
Chris@16:     for (int dest = 0; dest < size; ++dest) {
Chris@16:       // Keep track of the displacements
Chris@16:       send_disps[dest] = outgoing.size();
Chris@16: 
Chris@16:       // Our own value will never be transmitted, so don't pack it.
Chris@16:       if (dest != rank) {
Chris@16:         packed_oarchive oa(comm, outgoing);
Chris@16:         for (int i = 0; i < n; ++i)
Chris@16:           oa << in_values[dest * n + i];
Chris@16:       }
Chris@16: 
Chris@16:       // Keep track of the sizes
Chris@16:       send_sizes[dest] = outgoing.size() - send_disps[dest];
Chris@16:     }
Chris@16: 
Chris@16:     // Determine how much data each process will receive.
Chris@16:     std::vector<int> recv_sizes(size);
Chris@16:     all_to_all(comm, send_sizes, recv_sizes);
Chris@16: 
Chris@16:     // Prepare a buffer to receive the incoming data.
Chris@16:     std::vector<int> recv_disps(size);
Chris@16:     int sum = 0;
Chris@16:     for (int src = 0; src < size; ++src) {
Chris@16:       recv_disps[src] = sum;
Chris@16:       sum += recv_sizes[src];
Chris@16:     }
Chris@16:     std::vector<char, allocator<char> > incoming(sum > 0? sum : 1);
Chris@16: 
Chris@16:     // Make sure we don't try to reference an empty vector
Chris@16:     if (outgoing.empty())
Chris@16:       outgoing.push_back(0);
Chris@16: 
Chris@16:     // Transmit the actual data
Chris@16:     BOOST_MPI_CHECK_RESULT(MPI_Alltoallv,
Chris@16:                            (&outgoing[0], &send_sizes[0],
Chris@16:                             &send_disps[0], MPI_PACKED,
Chris@16:                             &incoming[0], &recv_sizes[0],
Chris@16:                             &recv_disps[0], MPI_PACKED,
Chris@16:                             comm));
Chris@16: 
Chris@16:     // Deserialize data from the iarchive
Chris@16:     for (int src = 0; src < size; ++src) {
Chris@16:       if (src == rank) 
Chris@16:         std::copy(in_values + src * n, in_values + (src + 1) * n, 
Chris@16:                   out_values + src * n);
Chris@16:       else {
Chris@16:         packed_iarchive ia(comm, incoming, boost::archive::no_header,
Chris@16:                            recv_disps[src]);
Chris@16:         for (int i = 0; i < n; ++i)
Chris@16:           ia >> out_values[src * n + i];
Chris@16:       }
Chris@16:     }
Chris@16:   }
Chris@16: } // end namespace detail
Chris@16: 
Chris@16: template<typename T>
Chris@16: inline void
Chris@16: all_to_all(const communicator& comm, const T* in_values, T* out_values)
Chris@16: {
Chris@16:   detail::all_to_all_impl(comm, in_values, 1, out_values, is_mpi_datatype<T>());
Chris@16: }
Chris@16: 
Chris@16: template<typename T>
Chris@16: void
Chris@16: all_to_all(const communicator& comm, const std::vector<T>& in_values,
Chris@16:            std::vector<T>& out_values)
Chris@16: {
Chris@16:   BOOST_ASSERT((int)in_values.size() == comm.size());
Chris@16:   out_values.resize(comm.size());
Chris@16:   ::boost::mpi::all_to_all(comm, &in_values[0], &out_values[0]);
Chris@16: }
Chris@16: 
Chris@16: template<typename T>
Chris@16: inline void
Chris@16: all_to_all(const communicator& comm, const T* in_values, int n, T* out_values)
Chris@16: {
Chris@16:   detail::all_to_all_impl(comm, in_values, n, out_values, is_mpi_datatype<T>());
Chris@16: }
Chris@16: 
Chris@16: template<typename T>
Chris@16: void
Chris@16: all_to_all(const communicator& comm, const std::vector<T>& in_values, int n,
Chris@16:            std::vector<T>& out_values)
Chris@16: {
Chris@16:   BOOST_ASSERT((int)in_values.size() == comm.size() * n);
Chris@16:   out_values.resize(comm.size() * n);
Chris@16:   ::boost::mpi::all_to_all(comm, &in_values[0], n, &out_values[0]);
Chris@16: }
Chris@16: 
Chris@16: } } // end namespace boost::mpi
Chris@16: 
Chris@16: #endif // BOOST_MPI_ALL_TO_ALL_HPP