From 49f46bb0b936268bf55e2b8b1452f212f6379dc2 Mon Sep 17 00:00:00 2001 From: Alexander Hampel Date: Wed, 10 Mar 2021 18:33:20 -0500 Subject: [PATCH 1/9] added layer to use triqs without MPI and created no mpi tests --- c++/mpi/mpi.hpp | 135 +++++++++++++++++++++++++++++++--------- test/c++/CMakeLists.txt | 34 +++++++++- 2 files changed, 138 insertions(+), 31 deletions(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index 96ec39f5..02f04a1a 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -43,37 +43,70 @@ namespace mpi { }; // ------------------------------------------------------------ + + /// MPI env state variable + enum class MPI_ENV { + Unchecked, + False, + True + }; + + inline MPI_ENV check_mpi_env() { + if (std::getenv("OMPI_COMM_WORLD_RANK") != nullptr or std::getenv("PMI_RANK") != nullptr) + return MPI_ENV::True; + else + return MPI_ENV::False; + } /// The communicator. Todo : add more constructors. class communicator { MPI_Comm _com = MPI_COMM_WORLD; + + inline static MPI_ENV _mpi_env = MPI_ENV::Unchecked; public: - communicator() = default; + communicator() { + if (_mpi_env == MPI_ENV::Unchecked) { + _mpi_env = check_mpi_env(); + //std::cout << "MPI environment (1: False / 2: True):" << int(_mpi_env) << "\n"; + } + } communicator(MPI_Comm c) : _com(c) {} [[nodiscard]] MPI_Comm get() const noexcept { return _com; } [[nodiscard]] int rank() const { - int num; - MPI_Comm_rank(_com, &num); - return num; + if (_mpi_env == MPI_ENV::True) { + int num; + MPI_Comm_rank(_com, &num); + return num; + } else + return 0; } [[nodiscard]] int size() const { - int num; - MPI_Comm_size(_com, &num); - return num; - } + if (_mpi_env == MPI_ENV::True) { + int num; + MPI_Comm_size(_com, &num); + return num; + } else + return 1; + } [[nodiscard]] communicator split(int color, int key = 0) const { - communicator c; - MPI_Comm_split(_com, color, key, &c._com); - return c; + if (_mpi_env == MPI_ENV::True) { + communicator c; + MPI_Comm_split(_com, color, key, &c._com); + return c; + } else +//TODO split should not be done without MPI? + return 0; } - void abort(int error_code) { MPI_Abort(_com, error_code); } + void abort(int error_code) { + if (_mpi_env == MPI_ENV::True) { MPI_Abort(_com, error_code); } + } #ifdef BOOST_MPI_HPP // Conversion to and from boost communicator, Keep for backward compatibility @@ -81,7 +114,9 @@ namespace mpi { inline communicator(boost::mpi::communicator c) : _com(c) {} #endif - void barrier() const { MPI_Barrier(_com); } + void barrier() const { + if (_mpi_env == MPI_ENV::True) { MPI_Barrier(_com); } + } }; // ---------------------------------------- @@ -116,45 +151,78 @@ namespace mpi { template [[gnu::always_inline]] inline decltype(auto) broadcast(T &&x, communicator c = {}, int root = 0) { - return mpi_broadcast(std::forward(x), c, root); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return mpi_broadcast(std::forward(x), c, root); + } else + return decltype(mpi_broadcast(std::forward(x), c, root))(std::forward(x)); } + template [[gnu::always_inline]] inline decltype(auto) reduce(T &&x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { - return mpi_reduce(std::forward(x), c, root, all, op); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return mpi_reduce(std::forward(x), c, root, all, op); + } else + return decltype(mpi_reduce(std::forward(x), c, root, all, op))(std::forward(x)); } + template [[gnu::always_inline]] inline void reduce_in_place(T &&x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { - return mpi_reduce_in_place(std::forward(x), c, root, all, op); + if (mpi::check_mpi_env() == MPI_ENV::True) { return mpi_reduce_in_place(std::forward(x), c, root, all, op); } } + template [[gnu::always_inline]] inline decltype(auto) scatter(T &&x, mpi::communicator c = {}, int root = 0) { - return mpi_scatter(std::forward(x), c, root); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return mpi_scatter(std::forward(x), c, root); + } else + return decltype(mpi_scatter(std::forward(x), c, root))(std::forward(x)); } + template [[gnu::always_inline]] inline decltype(auto) gather(T &&x, mpi::communicator c = {}, int root = 0, bool all = false) { - return mpi_gather(std::forward(x), c, root, all); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return mpi_gather(std::forward(x), c, root, all); + } else + return decltype(mpi_gather(std::forward(x), c, root, all))(std::forward(x)); } + template [[gnu::always_inline]] inline decltype(auto) all_reduce(T &&x, communicator c = {}, MPI_Op op = MPI_SUM) { - return reduce(std::forward(x), c, 0, true, op); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return reduce(std::forward(x), c, 0, true, op); + } else + return decltype(reduce(std::forward(x), c, 0, true, op))(std::forward(x)); } + template [[gnu::always_inline]] inline void all_reduce_in_place(T &&x, communicator c = {}, MPI_Op op = MPI_SUM) { - return reduce_in_place(std::forward(x), c, 0, true, op); + if (mpi::check_mpi_env() == MPI_ENV::True) { return reduce_in_place(std::forward(x), c, 0, true, op); } } + template [[gnu::always_inline]] inline decltype(auto) all_gather(T &&x, communicator c = {}) { - return gather(std::forward(x), c, 0, true); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return gather(std::forward(x), c, 0, true); + } else + return decltype(gather(std::forward(x), c, 0, true))(std::forward(x)); } + template [[gnu::always_inline]] [[deprecated("mpi_all_reduce is deprecated, please use mpi::all_reduce instead")]] inline decltype(auto) mpi_all_reduce(T &&x, communicator c = {}, MPI_Op op = MPI_SUM) { - return reduce(std::forward(x), c, 0, true, op); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return reduce(std::forward(x), c, 0, true, op); + } else + return decltype(reduce(std::forward(x), c, 0, true, op))(std::forward(x)); } + template [[gnu::always_inline]] [[deprecated("mpi_all_gather is deprecated, please use mpi::all_gather instead")]] inline decltype(auto) mpi_all_gather(T &&x, communicator c = {}) { - return gather(std::forward(x), c, 0, true); + if (mpi::check_mpi_env() == MPI_ENV::True) { + return gather(std::forward(x), c, 0, true); + } else + return decltype(gather(std::forward(x), c, 0, true))(std::forward(x)); } /* ----------------------------------------------------------- @@ -330,11 +398,20 @@ namespace mpi { MPI_Allreduce(MPI_IN_PLACE, &a, 1, mpi_type::get(), op, c.get()); } -#define MPI_TEST_MAIN \ - int main(int argc, char **argv) { \ - mpi::environment env(argc, argv); \ - ::testing::InitGoogleTest(&argc, argv); \ - return RUN_ALL_TESTS(); \ - } +#define MPI_TEST_MAIN \ + int main(int argc, char **argv) { \ +\ + ::testing::InitGoogleTest(&argc, argv); \ + if (mpi::check_mpi_env() == mpi::MPI_ENV::True) { \ + mpi::environment env(argc, argv); \ + std::cout << "MPI environment detected" \ + << "\n"; \ + return RUN_ALL_TESTS(); \ + } \ + else { \ + return RUN_ALL_TESTS(); \ + } \ + \ + } } // namespace mpi diff --git a/test/c++/CMakeLists.txt b/test/c++/CMakeLists.txt index 0304b2f9..eedd15e3 100644 --- a/test/c++/CMakeLists.txt +++ b/test/c++/CMakeLists.txt @@ -7,6 +7,11 @@ endforeach() # List of all tests file(GLOB_RECURSE all_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) +# List of all no mpi tests +file(GLOB_RECURSE nompi_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) +# remove custom mpi test, as this one explicitly uses MPI +list(REMOVE_ITEM nompi_tests mpi_custom.cpp) + # ========= OpenMP Dependency ========== find_package(OpenMP REQUIRED COMPONENTS CXX) @@ -23,8 +28,7 @@ foreach(test ${all_tests}) target_link_libraries(${test_name} ${PROJECT_NAME}::${PROJECT_NAME}_c openmp ${PROJECT_NAME}_warnings gtest_main) set_property(TARGET ${test_name} PROPERTY RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}) set(test_bin ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}/${test_name}) - add_test(NAME ${test_name}_np1 COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${test_bin} ${MPIEXEC_POSTFLAGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}) - add_test(NAME ${test_name}_np2 COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ${test_bin} ${MPIEXEC_POSTFLAGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}) + add_test(NAME ${test_name}_np2 COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${test_bin} ${MPIEXEC_POSTFLAGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}) add_test(NAME ${test_name}_np4 COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${test_bin} ${MPIEXEC_POSTFLAGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}) # Run clang-tidy if found if(CLANG_TIDY_EXECUTABLE) @@ -45,3 +49,29 @@ foreach(test ${all_tests}) ) endif() endforeach() + +# now the no mpi tests +foreach(test ${nompi_tests}) + get_filename_component(test_name ${test} NAME_WE) + get_filename_component(test_dir ${test} DIRECTORY) + set(test_bin ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}/${test_name}) + add_test(NAME ${test_name}_nompi COMMAND ${test_bin} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${test_dir}) + # Run clang-tidy if found + if(CLANG_TIDY_EXECUTABLE) + set_target_properties(${test_name} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXECUTABLE}") + endif() + # Run cppcheck if found + if(CPPCHECK_EXECUTABLE) + add_custom_command( + TARGET ${test_name} + COMMAND ${CPPCHECK_EXECUTABLE} + --enable=warning,style,performance,portability + --std=c++17 + --template=gcc + --verbose + --force + --quiet + ${CMAKE_CURRENT_SOURCE_DIR}/${test} + ) + endif() +endforeach() From d289030eb0e77cc9736983fd6a6d3440f6cf0921 Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Fri, 12 Mar 2021 17:26:42 -0500 Subject: [PATCH 2/9] [cmake] Do not run mpi_monitor as nompi test --- test/c++/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/c++/CMakeLists.txt b/test/c++/CMakeLists.txt index eedd15e3..8074ffb7 100644 --- a/test/c++/CMakeLists.txt +++ b/test/c++/CMakeLists.txt @@ -10,7 +10,7 @@ file(GLOB_RECURSE all_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) # List of all no mpi tests file(GLOB_RECURSE nompi_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) # remove custom mpi test, as this one explicitly uses MPI -list(REMOVE_ITEM nompi_tests mpi_custom.cpp) +list(REMOVE_ITEM nompi_tests mpi_custom.cpp mpi_monitor.cpp) # ========= OpenMP Dependency ========== From 28b98627e3329012c2ed8cc3ca50ad88e75df0d0 Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Mon, 15 Mar 2021 15:00:47 -0400 Subject: [PATCH 3/9] Minor formatting changes --- c++/mpi/mpi.hpp | 53 ++++++++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index 02f04a1a..c343d964 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -43,13 +43,9 @@ namespace mpi { }; // ------------------------------------------------------------ - + /// MPI env state variable - enum class MPI_ENV { - Unchecked, - False, - True - }; + enum class MPI_ENV { Unchecked, False, True }; inline MPI_ENV check_mpi_env() { if (std::getenv("OMPI_COMM_WORLD_RANK") != nullptr or std::getenv("PMI_RANK") != nullptr) @@ -61,7 +57,7 @@ namespace mpi { /// The communicator. Todo : add more constructors. class communicator { MPI_Comm _com = MPI_COMM_WORLD; - + inline static MPI_ENV _mpi_env = MPI_ENV::Unchecked; public: @@ -92,16 +88,16 @@ namespace mpi { return num; } else return 1; - } + } [[nodiscard]] communicator split(int color, int key = 0) const { if (_mpi_env == MPI_ENV::True) { - communicator c; - MPI_Comm_split(_com, color, key, &c._com); - return c; + communicator c; + MPI_Comm_split(_com, color, key, &c._com); + return c; } else -//TODO split should not be done without MPI? - return 0; + //TODO split should not be done without MPI? + return 0; } void abort(int error_code) { @@ -164,12 +160,12 @@ namespace mpi { } else return decltype(mpi_reduce(std::forward(x), c, root, all, op))(std::forward(x)); } - + template [[gnu::always_inline]] inline void reduce_in_place(T &&x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { if (mpi::check_mpi_env() == MPI_ENV::True) { return mpi_reduce_in_place(std::forward(x), c, root, all, op); } } - + template [[gnu::always_inline]] inline decltype(auto) scatter(T &&x, mpi::communicator c = {}, int root = 0) { if (mpi::check_mpi_env() == MPI_ENV::True) { @@ -177,7 +173,7 @@ namespace mpi { } else return decltype(mpi_scatter(std::forward(x), c, root))(std::forward(x)); } - + template [[gnu::always_inline]] inline decltype(auto) gather(T &&x, mpi::communicator c = {}, int root = 0, bool all = false) { if (mpi::check_mpi_env() == MPI_ENV::True) { @@ -398,20 +394,15 @@ namespace mpi { MPI_Allreduce(MPI_IN_PLACE, &a, 1, mpi_type::get(), op, c.get()); } -#define MPI_TEST_MAIN \ - int main(int argc, char **argv) { \ -\ - ::testing::InitGoogleTest(&argc, argv); \ - if (mpi::check_mpi_env() == mpi::MPI_ENV::True) { \ - mpi::environment env(argc, argv); \ - std::cout << "MPI environment detected" \ - << "\n"; \ - return RUN_ALL_TESTS(); \ - } \ - else { \ - return RUN_ALL_TESTS(); \ - } \ - \ - } +#define MPI_TEST_MAIN \ + int main(int argc, char **argv) { \ + ::testing::InitGoogleTest(&argc, argv); \ + if (mpi::check_mpi_env() == mpi::MPI_ENV::True) { \ + mpi::environment env(argc, argv); \ + std::cout << "MPI environment detected\n"; \ + return RUN_ALL_TESTS(); \ + } else \ + return RUN_ALL_TESTS(); \ + } } // namespace mpi From c61368a9a7cc46092e619d16fbc8a86d6b614788 Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Mon, 15 Mar 2021 15:08:25 -0400 Subject: [PATCH 4/9] Fix generic mpi operations for vector/lazy types, Protect in_place operations against rvalues and const --- c++/mpi/mpi.hpp | 93 +++++++++++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 34 deletions(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index c343d964..629c1c4a 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -146,79 +146,104 @@ namespace mpi { // ---------------------------------------- template - [[gnu::always_inline]] inline decltype(auto) broadcast(T &&x, communicator c = {}, int root = 0) { - if (mpi::check_mpi_env() == MPI_ENV::True) { - return mpi_broadcast(std::forward(x), c, root); - } else - return decltype(mpi_broadcast(std::forward(x), c, root))(std::forward(x)); + [[gnu::always_inline]] inline void broadcast(T &x, communicator c = {}, int root = 0) { + static_assert(not std::is_const_v, "mpi::broadcast cannot be called on const objects"); + if (mpi::check_mpi_env() == MPI_ENV::True) mpi_broadcast(x, c, root); } - + + namespace details { + + template + inline constexpr bool is_std_vector = false; + + template + inline constexpr bool is_std_vector> = true; + + template + T convert(V v) { + if constexpr (is_std_vector) { + T res; + res.reserve(v.size()); + for (auto &x : v) res.emplace_back(std::move(x)); + return res; + } else + return std::move(v); + } + } // namespace details + template [[gnu::always_inline]] inline decltype(auto) reduce(T &&x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { - if (mpi::check_mpi_env() == MPI_ENV::True) { + using r_t = decltype(mpi_reduce(std::forward(x), c, root, all, op)); + + if constexpr (is_mpi_lazy) { + return mpi_reduce(std::forward(x), c, root, all, op); + } else { + if (mpi::check_mpi_env() == MPI_ENV::True) return mpi_reduce(std::forward(x), c, root, all, op); - } else - return decltype(mpi_reduce(std::forward(x), c, root, all, op))(std::forward(x)); + else + return details::convert(std::forward(x)); + } } template - [[gnu::always_inline]] inline void reduce_in_place(T &&x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { - if (mpi::check_mpi_env() == MPI_ENV::True) { return mpi_reduce_in_place(std::forward(x), c, root, all, op); } + [[gnu::always_inline]] inline void reduce_in_place(T &x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { + static_assert(not std::is_const_v, "In-place mpi functions cannot be called on const objects"); + if (mpi::check_mpi_env() == MPI_ENV::True) mpi_reduce_in_place(x, c, root, all, op); } template [[gnu::always_inline]] inline decltype(auto) scatter(T &&x, mpi::communicator c = {}, int root = 0) { - if (mpi::check_mpi_env() == MPI_ENV::True) { + using r_t = decltype(mpi_scatter(std::forward(x), c, root)); + + if constexpr (is_mpi_lazy) { + return mpi_scatter(std::forward(x), c, root); + } else { + if (mpi::check_mpi_env() == MPI_ENV::True) return mpi_scatter(std::forward(x), c, root); - } else - return decltype(mpi_scatter(std::forward(x), c, root))(std::forward(x)); + else + return details::convert(std::forward(x)); + } } template [[gnu::always_inline]] inline decltype(auto) gather(T &&x, mpi::communicator c = {}, int root = 0, bool all = false) { - if (mpi::check_mpi_env() == MPI_ENV::True) { + using r_t = decltype(mpi_gather(std::forward(x), c, root, all)); + + if constexpr (is_mpi_lazy) { return mpi_gather(std::forward(x), c, root, all); - } else - return decltype(mpi_gather(std::forward(x), c, root, all))(std::forward(x)); + } else { + if (mpi::check_mpi_env() == MPI_ENV::True) + return mpi_gather(std::forward(x), c, root, all); + else + return details::convert(std::forward(x)); + } } template [[gnu::always_inline]] inline decltype(auto) all_reduce(T &&x, communicator c = {}, MPI_Op op = MPI_SUM) { - if (mpi::check_mpi_env() == MPI_ENV::True) { - return reduce(std::forward(x), c, 0, true, op); - } else - return decltype(reduce(std::forward(x), c, 0, true, op))(std::forward(x)); + return reduce(std::forward(x), c, 0, true, op); } template [[gnu::always_inline]] inline void all_reduce_in_place(T &&x, communicator c = {}, MPI_Op op = MPI_SUM) { - if (mpi::check_mpi_env() == MPI_ENV::True) { return reduce_in_place(std::forward(x), c, 0, true, op); } + reduce_in_place(std::forward(x), c, 0, true, op); } template [[gnu::always_inline]] inline decltype(auto) all_gather(T &&x, communicator c = {}) { - if (mpi::check_mpi_env() == MPI_ENV::True) { - return gather(std::forward(x), c, 0, true); - } else - return decltype(gather(std::forward(x), c, 0, true))(std::forward(x)); + return gather(std::forward(x), c, 0, true); } template [[gnu::always_inline]] [[deprecated("mpi_all_reduce is deprecated, please use mpi::all_reduce instead")]] inline decltype(auto) mpi_all_reduce(T &&x, communicator c = {}, MPI_Op op = MPI_SUM) { - if (mpi::check_mpi_env() == MPI_ENV::True) { - return reduce(std::forward(x), c, 0, true, op); - } else - return decltype(reduce(std::forward(x), c, 0, true, op))(std::forward(x)); + return reduce(std::forward(x), c, 0, true, op); } template [[gnu::always_inline]] [[deprecated("mpi_all_gather is deprecated, please use mpi::all_gather instead")]] inline decltype(auto) mpi_all_gather(T &&x, communicator c = {}) { - if (mpi::check_mpi_env() == MPI_ENV::True) { - return gather(std::forward(x), c, 0, true); - } else - return decltype(gather(std::forward(x), c, 0, true))(std::forward(x)); + return gather(std::forward(x), c, 0, true); } /* ----------------------------------------------------------- From ed92ce1cd63a00ecf95e0bb82227a1b5b84f2fd0 Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Mon, 15 Mar 2021 15:30:13 -0400 Subject: [PATCH 5/9] Simplify check for mpi env vars, provide single static mpi::has_env --- c++/mpi/mpi.hpp | 47 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index 629c1c4a..36a5e604 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -31,6 +31,13 @@ namespace mpi { // ------------------------------------------------------------ + static const bool has_env = []() { + if (std::getenv("OMPI_COMM_WORLD_RANK") != nullptr or std::getenv("PMI_RANK") != nullptr) + return true; + else + return false; + }(); + /// Environment must be initialized in C++ struct environment { @@ -44,36 +51,20 @@ namespace mpi { // ------------------------------------------------------------ - /// MPI env state variable - enum class MPI_ENV { Unchecked, False, True }; - - inline MPI_ENV check_mpi_env() { - if (std::getenv("OMPI_COMM_WORLD_RANK") != nullptr or std::getenv("PMI_RANK") != nullptr) - return MPI_ENV::True; - else - return MPI_ENV::False; - } /// The communicator. Todo : add more constructors. class communicator { MPI_Comm _com = MPI_COMM_WORLD; - inline static MPI_ENV _mpi_env = MPI_ENV::Unchecked; - public: - communicator() { - if (_mpi_env == MPI_ENV::Unchecked) { - _mpi_env = check_mpi_env(); - //std::cout << "MPI environment (1: False / 2: True):" << int(_mpi_env) << "\n"; - } - } + communicator() = default; communicator(MPI_Comm c) : _com(c) {} [[nodiscard]] MPI_Comm get() const noexcept { return _com; } [[nodiscard]] int rank() const { - if (_mpi_env == MPI_ENV::True) { + if (has_env) { int num; MPI_Comm_rank(_com, &num); return num; @@ -82,7 +73,7 @@ namespace mpi { } [[nodiscard]] int size() const { - if (_mpi_env == MPI_ENV::True) { + if (has_env) { int num; MPI_Comm_size(_com, &num); return num; @@ -91,7 +82,7 @@ namespace mpi { } [[nodiscard]] communicator split(int color, int key = 0) const { - if (_mpi_env == MPI_ENV::True) { + if (has_env) { communicator c; MPI_Comm_split(_com, color, key, &c._com); return c; @@ -101,7 +92,7 @@ namespace mpi { } void abort(int error_code) { - if (_mpi_env == MPI_ENV::True) { MPI_Abort(_com, error_code); } + if (has_env) { MPI_Abort(_com, error_code); } } #ifdef BOOST_MPI_HPP @@ -111,7 +102,7 @@ namespace mpi { #endif void barrier() const { - if (_mpi_env == MPI_ENV::True) { MPI_Barrier(_com); } + if (has_env) { MPI_Barrier(_com); } } }; @@ -148,7 +139,7 @@ namespace mpi { template [[gnu::always_inline]] inline void broadcast(T &x, communicator c = {}, int root = 0) { static_assert(not std::is_const_v, "mpi::broadcast cannot be called on const objects"); - if (mpi::check_mpi_env() == MPI_ENV::True) mpi_broadcast(x, c, root); + if (has_env) mpi_broadcast(x, c, root); } namespace details { @@ -178,7 +169,7 @@ namespace mpi { if constexpr (is_mpi_lazy) { return mpi_reduce(std::forward(x), c, root, all, op); } else { - if (mpi::check_mpi_env() == MPI_ENV::True) + if (has_env) return mpi_reduce(std::forward(x), c, root, all, op); else return details::convert(std::forward(x)); @@ -188,7 +179,7 @@ namespace mpi { template [[gnu::always_inline]] inline void reduce_in_place(T &x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { static_assert(not std::is_const_v, "In-place mpi functions cannot be called on const objects"); - if (mpi::check_mpi_env() == MPI_ENV::True) mpi_reduce_in_place(x, c, root, all, op); + if (has_env) mpi_reduce_in_place(x, c, root, all, op); } template @@ -198,7 +189,7 @@ namespace mpi { if constexpr (is_mpi_lazy) { return mpi_scatter(std::forward(x), c, root); } else { - if (mpi::check_mpi_env() == MPI_ENV::True) + if (has_env) return mpi_scatter(std::forward(x), c, root); else return details::convert(std::forward(x)); @@ -212,7 +203,7 @@ namespace mpi { if constexpr (is_mpi_lazy) { return mpi_gather(std::forward(x), c, root, all); } else { - if (mpi::check_mpi_env() == MPI_ENV::True) + if (has_env) return mpi_gather(std::forward(x), c, root, all); else return details::convert(std::forward(x)); @@ -422,7 +413,7 @@ namespace mpi { #define MPI_TEST_MAIN \ int main(int argc, char **argv) { \ ::testing::InitGoogleTest(&argc, argv); \ - if (mpi::check_mpi_env() == mpi::MPI_ENV::True) { \ + if (mpi::has_env) { \ mpi::environment env(argc, argv); \ std::cout << "MPI environment detected\n"; \ return RUN_ALL_TESTS(); \ From cf48eb7ff40c94e54c46161cf48627ef9fec42e2 Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Mon, 15 Mar 2021 16:56:00 -0400 Subject: [PATCH 6/9] Minor formatting changes --- c++/mpi/mpi.hpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index 36a5e604..ac1e2c92 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -51,7 +51,6 @@ namespace mpi { // ------------------------------------------------------------ - /// The communicator. Todo : add more constructors. class communicator { MPI_Comm _com = MPI_COMM_WORLD; @@ -126,12 +125,6 @@ namespace mpi { MPI_Op op{}; }; - template - inline constexpr bool is_mpi_lazy = false; - - template - inline constexpr bool is_mpi_lazy> = true; - // ---------------------------------------- // ------- general functions ------- // ---------------------------------------- @@ -144,6 +137,12 @@ namespace mpi { namespace details { + template + inline constexpr bool is_mpi_lazy = false; + + template + inline constexpr bool is_mpi_lazy> = true; + template inline constexpr bool is_std_vector = false; @@ -166,7 +165,7 @@ namespace mpi { [[gnu::always_inline]] inline decltype(auto) reduce(T &&x, communicator c = {}, int root = 0, bool all = false, MPI_Op op = MPI_SUM) { using r_t = decltype(mpi_reduce(std::forward(x), c, root, all, op)); - if constexpr (is_mpi_lazy) { + if constexpr (details::is_mpi_lazy) { return mpi_reduce(std::forward(x), c, root, all, op); } else { if (has_env) @@ -186,7 +185,7 @@ namespace mpi { [[gnu::always_inline]] inline decltype(auto) scatter(T &&x, mpi::communicator c = {}, int root = 0) { using r_t = decltype(mpi_scatter(std::forward(x), c, root)); - if constexpr (is_mpi_lazy) { + if constexpr (details::is_mpi_lazy) { return mpi_scatter(std::forward(x), c, root); } else { if (has_env) @@ -200,7 +199,7 @@ namespace mpi { [[gnu::always_inline]] inline decltype(auto) gather(T &&x, mpi::communicator c = {}, int root = 0, bool all = false) { using r_t = decltype(mpi_gather(std::forward(x), c, root, all)); - if constexpr (is_mpi_lazy) { + if constexpr (details::is_mpi_lazy) { return mpi_gather(std::forward(x), c, root, all); } else { if (has_env) From 910f7aab8e1e2689a73e78971b33fd28337faa26 Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Mon, 15 Mar 2021 16:56:58 -0400 Subject: [PATCH 7/9] Make sure to std::abort in mpi abort when not in an mpi env --- c++/mpi/mpi.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index ac1e2c92..bd1fe6b1 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -91,7 +91,10 @@ namespace mpi { } void abort(int error_code) { - if (has_env) { MPI_Abort(_com, error_code); } + if (has_env) + MPI_Abort(_com, error_code); + else + std::abort(); } #ifdef BOOST_MPI_HPP From 1e2e6ff3dcb92a34388b036028a4d52c272a567d Mon Sep 17 00:00:00 2001 From: Nils Wentzell Date: Mon, 15 Mar 2021 16:58:19 -0400 Subject: [PATCH 8/9] mpi::details::convert should be recursive, trigger explicit conversion in else branch --- c++/mpi/mpi.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index bd1fe6b1..941d1d46 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -157,10 +157,10 @@ namespace mpi { if constexpr (is_std_vector) { T res; res.reserve(v.size()); - for (auto &x : v) res.emplace_back(std::move(x)); + for (auto &x : v) res.emplace_back(convert(std::move(x))); return res; } else - return std::move(v); + return T{std::move(v)}; } } // namespace details From d068e4bc99d06e71a134d1195c9b774c4cc6a26d Mon Sep 17 00:00:00 2001 From: Alexander Hampel Date: Fri, 19 Mar 2021 14:24:48 -0400 Subject: [PATCH 9/9] added a few comments to the source code --- c++/mpi/mpi.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/c++/mpi/mpi.hpp b/c++/mpi/mpi.hpp index 941d1d46..6603be65 100644 --- a/c++/mpi/mpi.hpp +++ b/c++/mpi/mpi.hpp @@ -31,6 +31,10 @@ namespace mpi { // ------------------------------------------------------------ + /* helper function to check for MPI runtime environment + * covers at the moment OpenMPI, MPICH, and intelmpi + * as cray uses MPICH under the hood it should work as well + */ static const bool has_env = []() { if (std::getenv("OMPI_COMM_WORLD_RANK") != nullptr or std::getenv("PMI_RANK") != nullptr) return true; @@ -191,6 +195,7 @@ namespace mpi { if constexpr (details::is_mpi_lazy) { return mpi_scatter(std::forward(x), c, root); } else { + // if it does not have a mpi lazy type, check manually if triqs is run with MPI if (has_env) return mpi_scatter(std::forward(x), c, root); else @@ -205,6 +210,7 @@ namespace mpi { if constexpr (details::is_mpi_lazy) { return mpi_gather(std::forward(x), c, root, all); } else { + // if it does not have a mpi lazy type, check manually if triqs is run with MPI if (has_env) return mpi_gather(std::forward(x), c, root, all); else