BaseLib::MPI Namespace Reference

Classes
class	AnotherMPIRankThrew
struct	Mpi
struct	Setup

Functions
template<typename T>
constexpr MPI_Datatype	mpiType ()
template<typename T>
static std::vector< T >	allgather (T const &value, Mpi const &mpi)
template<typename T>
static std::vector< T >	allgather (std::vector< T > const &vector, Mpi const &mpi)
template<typename T>
static T	allreduce (T const &value, MPI_Op const &mpi_op, Mpi const &mpi)
template<typename T>
static std::vector< T >	allreduce (std::vector< T > const &vector, MPI_Op const &mpi_op, Mpi const &mpi)
template<typename T>
static void	allreduceInplace (std::vector< T > &vector, MPI_Op const &mpi_op, Mpi const &mpi)
template<typename T>
static std::vector< int >	allgatherv (std::span< T > const send_buffer, std::vector< std::remove_const_t< T > > &receive_buffer, Mpi const &mpi)
static bool	anyOf (bool const val, Mpi const &mpi=Mpi{OGS_COMM_WORLD})
static bool	allOf (bool const val, Mpi const &mpi=Mpi{OGS_COMM_WORLD})
template<typename Exception>
void	allRanksThrowOrNone (std::exception_ptr const &exception, auto &&warning_callback)
template<typename Exception>
void	allRanksThrowOrNone (std::exception_ptr const &exception)
	Same as above but using the standard WARN() function for warnings.

Variables
MPI_Comm	OGS_COMM_WORLD = MPI_COMM_WORLD

Function Documentation

allgather() [1/2]

template<typename T>

std::vector< T > BaseLib::MPI::allgather ( std::vector< T > const & vector, Mpi const & mpi )

static

Definition at line 98 of file MPI.h.

{
    std::size_t const size = vector.size();
    // Flat in memory over all ranks;
    std::vector<T> result(mpi.size * size);
 
    MPI_Allgather(vector.data(), size, mpiType<T>(), result.data(), size,
                  mpiType<T>(), mpi.communicator);
 
    return result;
}

References BaseLib::MPI::Mpi::communicator, mpiType(), and BaseLib::MPI::Mpi::size.

allgather() [2/2]

template<typename T>

std::vector< T > BaseLib::MPI::allgather ( T const & value, Mpi const & mpi )

static

Definition at line 87 of file MPI.h.

{
    std::vector<T> result(mpi.size);
 
    MPI_Allgather(&value, 1, mpiType<T>(), result.data(), 1, mpiType<T>(),
                  mpi.communicator);
 
    return result;
}

References BaseLib::MPI::Mpi::communicator, mpiType(), and BaseLib::MPI::Mpi::size.

Referenced by allgatherv(), MeshLib::computeNumberOfRegularBaseNodesAtRank(), MeshLib::computeNumberOfRegularHigherOrderNodesAtRank(), MeshLib::computeRegularBaseNodeGlobalNodeIDsOfSubDomainPartition(), MeshLib::IO::getPartitionInfo(), MeshLib::IO::getTopologyType(), and MeshLib::IO::NodePartitionedMeshReader::newMesh().

allgatherv()

template<typename T>

std::vector< int > BaseLib::MPI::allgatherv ( std::span< T > const send_buffer, std::vector< std::remove_const_t< T > > & receive_buffer, Mpi const & mpi )

static

Allgather for variable data. Returns offsets in the receive buffer. The receive buffer is resized to accommodate the gathered data.

Definition at line 147 of file MPI.h.

{
    // Determine the number of elements to send
    int const size = static_cast<int>(send_buffer.size());
 
    // Gather sizes from all ranks
    std::vector<int> const sizes = allgather(size, mpi);
 
    // Compute offsets based on counts
    std::vector<int> const offsets = BaseLib::sizesToOffsets(sizes);
 
    // Resize receive buffer to hold all gathered data
    receive_buffer.resize(offsets.back());
 
    MPI_Allgatherv(send_buffer.data(), size, mpiType<T>(),
                   receive_buffer.data(), sizes.data(), offsets.data(),
                   mpiType<T>(), mpi.communicator);
 
    return offsets;
}

References allgather(), BaseLib::MPI::Mpi::communicator, mpiType(), and BaseLib::sizesToOffsets().

Referenced by MeshLib::computeGhostBaseNodeGlobalNodeIDsOfSubDomainPartition(), and MathLib::PETScVector::getGlobalVector().

allOf()

bool BaseLib::MPI::allOf ( bool const val, Mpi const & mpi = Mpi{OGS_COMM_WORLD} )

inlinestatic

The reduction is implemented transparently for with and without MPI. In the latter case the input value is returned.

Definition at line 190 of file MPI.h.

                                             {OGS_COMM_WORLD}
#endif
)
{
    return !anyOf(!val
#ifdef USE_PETSC
                  ,
                  mpi
#endif
    );
}

References OGS_COMM_WORLD.

allRanksThrowOrNone() [1/2]

template<typename Exception>

void BaseLib::MPI::allRanksThrowOrNone

(

std::exception_ptr const &

exception

)

Same as above but using the standard WARN() function for warnings.

Definition at line 283 of file MPI.h.

{
    auto warning_callback =
        []<typename... Args>(fmt::format_string<Args...> fmt, Args&&... args)
    { WARN(fmt, std::forward<Args>(args)...); };
 
    allRanksThrowOrNone<Exception>(exception, warning_callback);
}

References allRanksThrowOrNone(), and WARN().

allRanksThrowOrNone() [2/2]

template<typename Exception>

void BaseLib::MPI::allRanksThrowOrNone	(	std::exception_ptr const &	exception,
		auto &&	warning_callback )

This function throws if and only if the passed exception is non-null on any MPI rank.

The thrown exception is:

• the passed one on any MPI rank where the passed exception is non-null
• AnotherMPIRankThrew<Exception> on all other MPI ranks

This function also checks if the thrown exception is derived from Exception.

Definition at line 236 of file MPI.h.

{
    // std::exception would lead to duplicate catch clauses below and would not
    // work together with the current implementation of AnotherMPIRankThrew.
    static_assert(!std::is_same_v<Exception, std::exception>);
 
    bool const exception_was_thrown = anyOf(exception != nullptr);
 
    [[unlikely]] if (exception_was_thrown)
    {
        if (exception)
        {
            try
            {
                std::rethrow_exception(exception);
            }
            catch (Exception const&)
            {
                // OK. Argument exception is derived from class Exception.
                throw;
            }
            catch (std::exception const& e)
            {
                warning_callback(
                    "An exception was thrown on this MPI rank, but it's not "
                    "derived from {}, but rather of type {}",
                    BaseLib::typeToString<Exception>(),
                    BaseLib::demangle(
                        typeid(e).name() /* demangle the runtime type of e */));
                throw;
            }
            catch (...)
            {
                warning_callback(
                    "An exception was thrown on this MPI rank, but it's not "
                    "derived from std::exception.");
                throw;
            }
        }
 
        throw AnotherMPIRankThrew<Exception>{};
    }
}

References anyOf(), BaseLib::demangle(), and BaseLib::typeToString().

Referenced by allRanksThrowOrNone(), ProcessLib::AssemblyMixin< Process >::assemble(), and ProcessLib::AssemblyMixin< Process >::assembleWithJacobian().

allreduce() [1/2]

template<typename T>

std::vector< T > BaseLib::MPI::allreduce ( std::vector< T > const & vector, MPI_Op const & mpi_op, Mpi const & mpi )

static

Definition at line 120 of file MPI.h.

{
    std::size_t const size = vector.size();
    std::vector<T> result(vector.size());
 
    MPI_Allreduce(vector.data(), result.data(), size, mpiType<T>(), mpi_op,
                  mpi.communicator);
    return result;
}

References BaseLib::MPI::Mpi::communicator, and mpiType().

allreduce() [2/2]

template<typename T>

T BaseLib::MPI::allreduce ( T const & value, MPI_Op const & mpi_op, Mpi const & mpi )

static

Definition at line 111 of file MPI.h.

{
    T result{};
 
    MPI_Allreduce(&value, &result, 1, mpiType<T>(), mpi_op, mpi.communicator);
    return result;
}

References BaseLib::MPI::Mpi::communicator, and mpiType().

Referenced by MeshGeoToolsLib::BoundaryElementsAtPoint::BoundaryElementsAtPoint(), MeshLib::getNumberOfGlobalNodes(), NumLib::anonymous_namespace{DOFTableUtil.cpp}::norm1(), NumLib::anonymous_namespace{DOFTableUtil.cpp}::norm2(), NumLib::anonymous_namespace{DOFTableUtil.cpp}::normInfinity(), ProcessLib::PhaseField::PhaseFieldProcess< DisplacementDim >::postNonLinearSolverConcreteProcess(), ProcessLib::PhaseField::PhaseFieldProcess< DisplacementDim >::postTimestepConcreteProcess(), and MeshLib::IO::NodePartitionedMeshReader::readProperties().

allreduceInplace()

template<typename T>

void BaseLib::MPI::allreduceInplace ( std::vector< T > & vector, MPI_Op const & mpi_op, Mpi const & mpi )

static

Definition at line 132 of file MPI.h.

{
    MPI_Allreduce(MPI_IN_PLACE,
                  vector.data(),
                  vector.size(),
                  mpiType<T>(),
                  mpi_op,
                  mpi.communicator);
}

References BaseLib::MPI::Mpi::communicator, and mpiType().

Referenced by MeshLib::computeGhostBaseNodeGlobalNodeIDsOfSubDomainPartition().

anyOf()

bool BaseLib::MPI::anyOf ( bool const val, Mpi const & mpi = Mpi{OGS_COMM_WORLD} )

inlinestatic

The reduction is implemented transparently for with and without MPI. In the latter case the input value is returned.

Definition at line 174 of file MPI.h.

                                             {OGS_COMM_WORLD}
#endif
)
{
#ifdef USE_PETSC
    return allreduce(val, MPI_LOR, mpi);
#else
    return val;
#endif
}

References OGS_COMM_WORLD.

Referenced by allRanksThrowOrNone(), and NumLib::NonlinearSolver< NonlinearSolverTag::Newton >::solve().

mpiType()

template<typename T>

MPI_Datatype BaseLib::MPI::mpiType ( )

constexpr

Definition at line 53 of file MPI.h.

{
    using U = std::remove_const_t<T>;
    if constexpr (std::is_same_v<U, bool>)
    {
        return MPI_C_BOOL;
    }
    if constexpr (std::is_same_v<U, char>)
    {
        return MPI_CHAR;
    }
    if constexpr (std::is_same_v<U, double>)
    {
        return MPI_DOUBLE;
    }
    if constexpr (std::is_same_v<U, float>)
    {
        return MPI_FLOAT;
    }
    if constexpr (std::is_same_v<U, int>)
    {
        return MPI_INT;
    }
    if constexpr (std::is_same_v<U, std::size_t>)
    {
        return MPI_UNSIGNED_LONG;
    }
    if constexpr (std::is_same_v<U, unsigned int>)
    {
        return MPI_UNSIGNED;
    }
}

Referenced by allgather(), allgather(), allgatherv(), allreduce(), allreduce(), and allreduceInplace().

Variable Documentation

OGS_COMM_WORLD

MPI_Comm BaseLib::MPI::OGS_COMM_WORLD = MPI_COMM_WORLD

Definition at line 9 of file MPI.cpp.

Referenced by ApplicationsLib::detail::LinearSolverLibrarySetupPETSc::LinearSolverLibrarySetupPETSc(), BaseLib::MPI::Mpi::Mpi(), MeshLib::IO::PVDFile::addVTUFile(), allOf(), anyOf(), MeshLib::IO::getCommunicator(), initializeLogger(), BaseLib::initOGSLogger(), MeshLib::IO::isFileManager(), ChemistryLib::PhreeqcIOData::specifyFileName(), and MeshLib::IO::VtuInterface::writeVTU().