From 6167b915ff78ab459683fc763bd16ffa618f8f5c Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Thu, 13 Jun 2024 23:32:21 +0200 Subject: [PATCH 01/13] feat(init/fini): add basic initialization and finalization KokkosComm should now be initialized (and finalized) these new functions. Users now must _not_ initialize/finalize MPI, nor Kokkos by themselves. --- src/KokkosComm.hpp | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index ae4b5cb2..3998da3a 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -16,6 +16,7 @@ #pragma once +#include "KokkosComm_include_mpi.hpp" #include "KokkosComm_collective.hpp" #include "KokkosComm_version.hpp" #include "KokkosComm_isend.hpp" @@ -28,8 +29,56 @@ #include +#include +#include +#include + namespace KokkosComm { +inline void initialize(int &argc, char ***argv) { + int flag; + MPI_Initialized(&flag); + // Eagerly abort if MPI has already been initialized + if (0 != flag) { + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (0 == rank) { + fprintf(stderr, "error: MPI must not be initialized prior to initializing KokkosComm\n"); + } + MPI_Abort(MPI_COMM_WORLD, -1); + } + + int provided; + MPI_Init_thread(&argc, argv, MPI_THREAD_MULTIPLE, &provided); + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + // Abort if MPI failed to provide Thread Multiple + if (MPI_THREAD_MULTIPLE != provided) { + if (0 == rank) { + fprintf(stderr, "error: failed to initialized with required thread support\n"); + } + MPI_Abort(MPI_COMM_WORLD, -1); + } + + // Strip "--help" and "--kokkos-help" from the flags passed to Kokkos if we are not on rank 0 to prevent Kokkos + // from printing the help message multiple times. + if (0 != rank) { + auto *help_it = std::find_if(*argv, *argv + argc, + [](std::string_view const &x) { return x == "--help" || x == "--kokkos-help"; }); + if (help_it != *argv + argc) { + std::swap(*help_it, *(*argv + argc - 1)); + --argc; + } + } + Kokkos::initialize(argc, *argv); +} + +inline void finalize() { + Kokkos::finalize(); + MPI_Finalize(); +} + template Req isend(const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm) { return Impl::isend(space, sv, dest, tag, comm); From e055c019d3408f512d45e8f935359779dee68c9b Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Thu, 13 Jun 2024 23:33:10 +0200 Subject: [PATCH 02/13] test(init/fini): update tests with new initialization/finalization APIs --- perf_tests/test_main.cpp | 15 ++++++--------- unit_tests/test_main.cpp | 10 ++++------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/perf_tests/test_main.cpp b/perf_tests/test_main.cpp index 4898d6e4..f024f8c5 100644 --- a/perf_tests/test_main.cpp +++ b/perf_tests/test_main.cpp @@ -15,6 +15,7 @@ //@HEADER #include "KokkosComm_include_mpi.hpp" +#include "KokkosComm.hpp" #include #include @@ -32,15 +33,12 @@ class NullReporter : public ::benchmark::BenchmarkReporter { // The main is rewritten to allow for MPI initializing and for selecting a // reporter according to the process rank int main(int argc, char **argv) { - MPI_Init(&argc, &argv); - - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - Kokkos::initialize(); + KokkosComm::initialize(argc, &argv); ::benchmark::Initialize(&argc, argv); + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (rank == 0) // root process will use a reporter from the usual set provided by // ::benchmark @@ -51,7 +49,6 @@ int main(int argc, char **argv) { ::benchmark::RunSpecifiedBenchmarks(&null); } - Kokkos::finalize(); - MPI_Finalize(); + KokkosComm::finalize(); return 0; -} \ No newline at end of file +} diff --git a/unit_tests/test_main.cpp b/unit_tests/test_main.cpp index d5dea47a..ec27c036 100644 --- a/unit_tests/test_main.cpp +++ b/unit_tests/test_main.cpp @@ -81,7 +81,8 @@ int main(int argc, char *argv[]) { // Intialize google test ::testing::InitGoogleTest(&argc, argv); - MPI_Init(&argc, &argv); + KokkosComm::initialize(argc, &argv); + int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); @@ -90,8 +91,6 @@ int main(int argc, char *argv[]) { << KOKKOSCOMM_VERSION_PATCH << ")\n"; } - Kokkos::initialize(); - // Intialize google test ::testing::InitGoogleTest(&argc, argv); @@ -105,9 +104,8 @@ int main(int argc, char *argv[]) { // run tests auto exit_code = RUN_ALL_TESTS(); - // Finalize MPI before exiting - Kokkos::finalize(); - MPI_Finalize(); + // Finalize KokkosComm before exiting + KokkosComm::finalize(); return exit_code; } From d098a725df11f03a12e6395aad8cea48314b13b8 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 14:01:25 +0200 Subject: [PATCH 03/13] docs(init/fini): add documentation + tiny fixes --- docs/api/core.rst | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/api/core.rst b/docs/api/core.rst index d229dc5d..fb80f6d5 100644 --- a/docs/api/core.rst +++ b/docs/api/core.rst @@ -1,12 +1,15 @@ Core -==== +**** + +MPI API Support +=============== .. list-table:: MPI API Support :widths: 40 30 15 :header-rows: 1 * - MPI - - ``KokkosComm::`` + - KokkosComm - ``Kokkos::View`` * - ``MPI_Send`` - ``send`` or ``send`` @@ -33,6 +36,41 @@ Core - ``reduce`` - ✓ + +Initialization and finalization +------------------------------- + +KokkosComm provides a unified interface for initializing and finalizing both Kokkos and MPI. + +.. Attention:: It is mandatory to use KokkosComm's initialization and finalization functions instead of their respective Kokkos and MPI counterparts. + +.. cpp:function:: void KokkosComm::initialize(int &argc, char ***argv) + + Initializes the MPI execution environment with ``THREAD_MULTIPLE`` support, and then initializes the Kokkos execution environment. This function also strips ``--kokkos-help`` flags to prevent Kokkos from printing the help on all MPI ranks. + + :param argc: Non-negative value representing the number of command-line arguments passed to the program. + :param argv: Pointer to a pointer to the first element of an array of ``argc + 1`` pointers, of which the last one is null and the previous, if any, point to null-terminated multi-byte strings that represent the arguments passed to the program. + + **Requirements:** + + * ``KokkosComm::initialize`` has the same combined requirements as ``MPI_Init`` and ``Kokkos::initialize``. + * ``KokkosComm::initialize`` must be called in place of ``MPI_Init`` and ``Kokkos::initialize``. + * User-initiated MPI objects cannot be constructed, and MPI functions cannot be called until after ``KokkosComm::initialize`` is called. + * User-initiated Kokkos objects cannot be constructed until after ``KokkosComm::initialize`` is called. + +.. cpp:function:: void KokkosComm::finalize() + + Terminates the Kokkos and MPI execution environments. + + Programs are ill-formed if they do not call this function *after* calling ``KokkosComm::initialize``. + + **Requirements:** + + * ``KokkosComm::finalize`` has the same combined requirements as ``MPI_Finalize`` and ``Kokkos::finalize``. + * ``KokkosComm::finalize`` must be called in place of ``MPI_Finalize`` and ``Kokkos::finalize``. + * ``KokkosComm::finalize`` must be called after user-initialized Kokkos objects are out of scope. + + Point-to-point -------------- From 0baa4ca66650ef7a3d2e8a57472d34664eb4c985 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 14:19:31 +0200 Subject: [PATCH 04/13] fix(init): remove checks for printing stuff --- src/KokkosComm.hpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index 3998da3a..d51a0ff5 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -40,27 +40,18 @@ inline void initialize(int &argc, char ***argv) { MPI_Initialized(&flag); // Eagerly abort if MPI has already been initialized if (0 != flag) { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (0 == rank) { - fprintf(stderr, "error: MPI must not be initialized prior to initializing KokkosComm\n"); - } MPI_Abort(MPI_COMM_WORLD, -1); } int provided; MPI_Init_thread(&argc, argv, MPI_THREAD_MULTIPLE, &provided); - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - // Abort if MPI failed to provide Thread Multiple if (MPI_THREAD_MULTIPLE != provided) { - if (0 == rank) { - fprintf(stderr, "error: failed to initialized with required thread support\n"); - } MPI_Abort(MPI_COMM_WORLD, -1); } + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Strip "--help" and "--kokkos-help" from the flags passed to Kokkos if we are not on rank 0 to prevent Kokkos // from printing the help message multiple times. if (0 != rank) { From 5c5ed7e39705ff9b776fb6e9baa4a8b0c4b08983 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 14:21:01 +0200 Subject: [PATCH 05/13] feat(init): let users provide their thread level + simplify signature Also keep a version without required MPI thread level support for simpler usage --- src/KokkosComm.hpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index d51a0ff5..88bcc0bb 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -35,7 +35,7 @@ namespace KokkosComm { -inline void initialize(int &argc, char ***argv) { +inline void initialize(int &argc, char *argv[], int mpi_required_thread_lvl) { int flag; MPI_Initialized(&flag); // Eagerly abort if MPI has already been initialized @@ -44,9 +44,9 @@ inline void initialize(int &argc, char ***argv) { } int provided; - MPI_Init_thread(&argc, argv, MPI_THREAD_MULTIPLE, &provided); - // Abort if MPI failed to provide Thread Multiple - if (MPI_THREAD_MULTIPLE != provided) { + MPI_Init_thread(&argc, &argv, mpi_required_thread_lvl, &provided); + // Abort if MPI failed to provide the required thread level + if (mpi_required_thread_lvl != provided) { MPI_Abort(MPI_COMM_WORLD, -1); } @@ -55,14 +55,18 @@ inline void initialize(int &argc, char ***argv) { // Strip "--help" and "--kokkos-help" from the flags passed to Kokkos if we are not on rank 0 to prevent Kokkos // from printing the help message multiple times. if (0 != rank) { - auto *help_it = std::find_if(*argv, *argv + argc, + auto *help_it = std::find_if(argv, argv + argc, [](std::string_view const &x) { return x == "--help" || x == "--kokkos-help"; }); - if (help_it != *argv + argc) { - std::swap(*help_it, *(*argv + argc - 1)); + if (help_it != argv + argc) { + std::swap(*help_it, *(argv + argc - 1)); --argc; } } - Kokkos::initialize(argc, *argv); + Kokkos::initialize(argc, argv); +} + +inline void initialize(int &argc, char *argv[]) { + initialize(argc, argv, MPI_THREAD_MULTIPLE); } inline void finalize() { From 8987ce50711365fc5b616f794123b9e0c7ee6b75 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 14:21:26 +0200 Subject: [PATCH 06/13] fix(init): fix tests to match new function prototype --- perf_tests/test_main.cpp | 2 +- unit_tests/test_main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/perf_tests/test_main.cpp b/perf_tests/test_main.cpp index f024f8c5..7a22f1bc 100644 --- a/perf_tests/test_main.cpp +++ b/perf_tests/test_main.cpp @@ -33,7 +33,7 @@ class NullReporter : public ::benchmark::BenchmarkReporter { // The main is rewritten to allow for MPI initializing and for selecting a // reporter according to the process rank int main(int argc, char **argv) { - KokkosComm::initialize(argc, &argv); + KokkosComm::initialize(argc, argv); ::benchmark::Initialize(&argc, argv); diff --git a/unit_tests/test_main.cpp b/unit_tests/test_main.cpp index ec27c036..853f90a8 100644 --- a/unit_tests/test_main.cpp +++ b/unit_tests/test_main.cpp @@ -81,7 +81,7 @@ int main(int argc, char *argv[]) { // Intialize google test ::testing::InitGoogleTest(&argc, argv); - KokkosComm::initialize(argc, &argv); + KokkosComm::initialize(argc, argv); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); From a68902eb8a4d152d17eeb9f13892d4a1c75cc3df Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 15:04:13 +0200 Subject: [PATCH 07/13] fix: only filter all `--kokkos-` flags + format --- src/KokkosComm.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index 88bcc0bb..bab34940 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -55,8 +55,8 @@ inline void initialize(int &argc, char *argv[], int mpi_required_thread_lvl) { // Strip "--help" and "--kokkos-help" from the flags passed to Kokkos if we are not on rank 0 to prevent Kokkos // from printing the help message multiple times. if (0 != rank) { - auto *help_it = std::find_if(argv, argv + argc, - [](std::string_view const &x) { return x == "--help" || x == "--kokkos-help"; }); + auto *help_it = std::find_if( + argv, argv + argc, [](std::string_view const &x) { return x.find("--kokkos-") != std::string_view::npos; }); if (help_it != argv + argc) { std::swap(*help_it, *(argv + argc - 1)); --argc; @@ -65,9 +65,7 @@ inline void initialize(int &argc, char *argv[], int mpi_required_thread_lvl) { Kokkos::initialize(argc, argv); } -inline void initialize(int &argc, char *argv[]) { - initialize(argc, argv, MPI_THREAD_MULTIPLE); -} +inline void initialize(int &argc, char *argv[]) { initialize(argc, argv, MPI_THREAD_MULTIPLE); } inline void finalize() { Kokkos::finalize(); From 4868c877e3be80a0833fb9bb235ee064d75b6910 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 15:20:05 +0200 Subject: [PATCH 08/13] docs(init): update with new prototypes --- docs/api/core.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/api/core.rst b/docs/api/core.rst index fb80f6d5..67a0f9b2 100644 --- a/docs/api/core.rst +++ b/docs/api/core.rst @@ -44,12 +44,14 @@ KokkosComm provides a unified interface for initializing and finalizing both Kok .. Attention:: It is mandatory to use KokkosComm's initialization and finalization functions instead of their respective Kokkos and MPI counterparts. -.. cpp:function:: void KokkosComm::initialize(int &argc, char ***argv) +.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[], int mpi_required_thread_lvl) +.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[]) - Initializes the MPI execution environment with ``THREAD_MULTIPLE`` support, and then initializes the Kokkos execution environment. This function also strips ``--kokkos-help`` flags to prevent Kokkos from printing the help on all MPI ranks. + Initializes the MPI execution environment with the required MPI thread level support (``MPI_THREAD_MULTIPLE`` if unspecified), and then initializes the Kokkos execution environment. This function also strips ``--kokkos-*`` flags to prevent Kokkos from printing them on all MPI ranks. :param argc: Non-negative value representing the number of command-line arguments passed to the program. - :param argv: Pointer to a pointer to the first element of an array of ``argc + 1`` pointers, of which the last one is null and the previous, if any, point to null-terminated multi-byte strings that represent the arguments passed to the program. + :param argv: Pointer to the first element of an array of ``argc + 1`` pointers, of which the last one is null and the previous, if any, point to null-terminated multi-byte strings that represent the arguments passed to the program. + :param mpi_required_thread_lvl: Level of desired MPI thread support. **Requirements:** From f3c838cca4c9637bcfe24b7bee1b4b697b4d3c03 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 17:02:24 +0200 Subject: [PATCH 09/13] feat: add wrapper for `MPI_THREAD_*` support levels --- src/KokkosComm.hpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index bab34940..befceed1 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -35,7 +35,14 @@ namespace KokkosComm { -inline void initialize(int &argc, char *argv[], int mpi_required_thread_lvl) { +enum class ThreadSupportLevel { + Single = MPI_THREAD_SINGLE, + Funneled = MPI_THREAD_FUNNELED, + Serialized = MPI_THREAD_SERIALIZED, + Multiple = MPI_THREAD_MULTIPLE, +}; + +inline void initialize(int &argc, char *argv[], ThreadSupportLevel required) { int flag; MPI_Initialized(&flag); // Eagerly abort if MPI has already been initialized @@ -44,9 +51,9 @@ inline void initialize(int &argc, char *argv[], int mpi_required_thread_lvl) { } int provided; - MPI_Init_thread(&argc, &argv, mpi_required_thread_lvl, &provided); - // Abort if MPI failed to provide the required thread level - if (mpi_required_thread_lvl != provided) { + MPI_Init_thread(&argc, &argv, static_cast(required), &provided); + // Abort if MPI failed to provide the required thread support level + if (static_cast(required) < provided) { MPI_Abort(MPI_COMM_WORLD, -1); } @@ -65,7 +72,7 @@ inline void initialize(int &argc, char *argv[], int mpi_required_thread_lvl) { Kokkos::initialize(argc, argv); } -inline void initialize(int &argc, char *argv[]) { initialize(argc, argv, MPI_THREAD_MULTIPLE); } +inline void initialize(int &argc, char *argv[]) { initialize(argc, argv, ThreadSupportLevel::Multiple); } inline void finalize() { Kokkos::finalize(); From cfdd01e0be43519345e6e5b8c317d69e39d04254 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 17:04:08 +0200 Subject: [PATCH 10/13] fix: revert filter on `--kokkos-*` to only filter `--kokkos-help` flags --- src/KokkosComm.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index befceed1..76bb6a6e 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -62,9 +62,8 @@ inline void initialize(int &argc, char *argv[], ThreadSupportLevel required) { // Strip "--help" and "--kokkos-help" from the flags passed to Kokkos if we are not on rank 0 to prevent Kokkos // from printing the help message multiple times. if (0 != rank) { - auto *help_it = std::find_if( - argv, argv + argc, [](std::string_view const &x) { return x.find("--kokkos-") != std::string_view::npos; }); - if (help_it != argv + argc) { + if (auto *help_it = std::find_if(argv, argv + argc, [](std::string_view const &x) { return x == "--kokkos-help"; }); + help_it != argv + argc) { std::swap(*help_it, *(argv + argc - 1)); --argc; } From ee700c6e2f53e13880dd319988c1cbae69ff714f Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 17:05:39 +0200 Subject: [PATCH 11/13] fix: prevent overlap w/ `MPI_{Init,Finalize}` & `Kokkos::{initialize,finalize}` --- src/KokkosComm.hpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index 76bb6a6e..9aa25256 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -45,11 +45,16 @@ enum class ThreadSupportLevel { inline void initialize(int &argc, char *argv[], ThreadSupportLevel required) { int flag; MPI_Initialized(&flag); - // Eagerly abort if MPI has already been initialized + // Forbid calling this function if MPI has already been initialized if (0 != flag) { MPI_Abort(MPI_COMM_WORLD, -1); } + // Forbid calling this function if Kokkos has already been initialized + if (Kokkos::is_initialized()) { + std::abort(); + } + int provided; MPI_Init_thread(&argc, &argv, static_cast(required), &provided); // Abort if MPI failed to provide the required thread support level @@ -74,7 +79,18 @@ inline void initialize(int &argc, char *argv[], ThreadSupportLevel required) { inline void initialize(int &argc, char *argv[]) { initialize(argc, argv, ThreadSupportLevel::Multiple); } inline void finalize() { + // Forbid calling this function if Kokkos has already been finalized or isn't yet initialized + if (Kokkos::is_finalized() || !Kokkos::is_initialized()) { + MPI_Abort(MPI_COMM_WORLD, -1); + } Kokkos::finalize(); + + int flag; + MPI_Finalized(&flag); + // Forbid calling this function if MPI has already been finalized + if (0 != flag) { + std::abort(); + } MPI_Finalize(); } From cfefa4e83a9de785d0cd1d4dff93875fb4ed841c Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 17:06:13 +0200 Subject: [PATCH 12/13] docs: clarify valid usage and document `ThreadSupportLevel` wrapper --- docs/api/core.rst | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/api/core.rst b/docs/api/core.rst index 67a0f9b2..21850e6d 100644 --- a/docs/api/core.rst +++ b/docs/api/core.rst @@ -42,20 +42,44 @@ Initialization and finalization KokkosComm provides a unified interface for initializing and finalizing both Kokkos and MPI. -.. Attention:: It is mandatory to use KokkosComm's initialization and finalization functions instead of their respective Kokkos and MPI counterparts. +.. Attention:: It is strongly recommended to use KokkosComm's initialization and finalization functions instead of their respective Kokkos and MPI counterparts. However, users have two options for using KokkosComm: -.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[], int mpi_required_thread_lvl) + 1. Initialize/finalize KokkosComm using ``KokkosComm::{initialize,finalize}``. In this case, it is **forbidden** to call ``MPI_{Init,Init_thread,Finalize}`` and ``Kokkos::{initialize,finalize}`` (otherwise, the application will abort). + 2. Initialize/finalize MPI and Kokkos manually through their respective interfaces. In this case, it is **forbidden** to call ``KokkosComm::{initialize,finalize}`` (otherwise, the application will abort). + +.. cpp:enum-class:: KokkosComm::ThreadSupportLevel + + A scoped enum to wrap the MPI thread support levels. + + .. cpp:enumerator:: KokkosComm::ThreadSupportLevel::Single + + Only one thread will execute. + + .. cpp:enumerator:: KokkosComm::ThreadSupportLevel::Funneled + + The process may be multi-threaded, but only the main thread will make MPI calls (all MPI calls are funneled to the main thread). + + .. cpp:enumerator:: KokkosComm::ThreadSupportLevel::Serialized + + The process may be multi-threaded, and multiple threads may make MPI calls, but only one at a time: MPI calls are not made concurrently from two distinct threads (all MPI calls are serialized). + + .. cpp:enumerator:: KokkosComm::ThreadSupportLevel::Multiple + + Multiple threads may call MPI, with no restrictions. + + +.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[], KokkosComm::ThreadSupportLevel required) .. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[]) - Initializes the MPI execution environment with the required MPI thread level support (``MPI_THREAD_MULTIPLE`` if unspecified), and then initializes the Kokkos execution environment. This function also strips ``--kokkos-*`` flags to prevent Kokkos from printing them on all MPI ranks. + Initializes the MPI execution environment with the required MPI thread level support (``MPI_THREAD_MULTIPLE`` if unspecified), and then initializes the Kokkos execution environment. This function also strips ``--kokkos-help`` flags to prevent Kokkos from printing them on all MPI ranks. :param argc: Non-negative value representing the number of command-line arguments passed to the program. :param argv: Pointer to the first element of an array of ``argc + 1`` pointers, of which the last one is null and the previous, if any, point to null-terminated multi-byte strings that represent the arguments passed to the program. - :param mpi_required_thread_lvl: Level of desired MPI thread support. + :param required: Level of desired MPI thread support. **Requirements:** - * ``KokkosComm::initialize`` has the same combined requirements as ``MPI_Init`` and ``Kokkos::initialize``. + * ``KokkosComm::initialize`` has the same combined requirements as ``MPI_{Init,Init_thread}`` and ``Kokkos::initialize``. * ``KokkosComm::initialize`` must be called in place of ``MPI_Init`` and ``Kokkos::initialize``. * User-initiated MPI objects cannot be constructed, and MPI functions cannot be called until after ``KokkosComm::initialize`` is called. * User-initiated Kokkos objects cannot be constructed until after ``KokkosComm::initialize`` is called. From d3b00de53d17edc4c0d4ef4e1526ebe887b5fd98 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos Date: Fri, 14 Jun 2024 17:11:54 +0200 Subject: [PATCH 13/13] fix: suggestion from Andrey Prokopenko (@aprokop) --- docs/api/core.rst | 5 ++--- src/KokkosComm.hpp | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/api/core.rst b/docs/api/core.rst index 21850e6d..d657c013 100644 --- a/docs/api/core.rst +++ b/docs/api/core.rst @@ -68,10 +68,9 @@ KokkosComm provides a unified interface for initializing and finalizing both Kok Multiple threads may call MPI, with no restrictions. -.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[], KokkosComm::ThreadSupportLevel required) -.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[]) +.. cpp:function:: void KokkosComm::initialize(int &argc, char *argv[], KokkosComm::ThreadSupportLevel required = KokkosComm::ThreadSupportLevel::Multiple) - Initializes the MPI execution environment with the required MPI thread level support (``MPI_THREAD_MULTIPLE`` if unspecified), and then initializes the Kokkos execution environment. This function also strips ``--kokkos-help`` flags to prevent Kokkos from printing them on all MPI ranks. + Initializes the MPI execution environment with the required MPI thread level support (``Multiple`` if left unspecified), and then initializes the Kokkos execution environment. This function also strips ``--kokkos-help`` flags to prevent Kokkos from printing them on all MPI ranks. :param argc: Non-negative value representing the number of command-line arguments passed to the program. :param argv: Pointer to the first element of an array of ``argc + 1`` pointers, of which the last one is null and the previous, if any, point to null-terminated multi-byte strings that represent the arguments passed to the program. diff --git a/src/KokkosComm.hpp b/src/KokkosComm.hpp index 9aa25256..e9783b24 100644 --- a/src/KokkosComm.hpp +++ b/src/KokkosComm.hpp @@ -42,7 +42,7 @@ enum class ThreadSupportLevel { Multiple = MPI_THREAD_MULTIPLE, }; -inline void initialize(int &argc, char *argv[], ThreadSupportLevel required) { +inline void initialize(int &argc, char *argv[], ThreadSupportLevel required = ThreadSupportLevel::Multiple) { int flag; MPI_Initialized(&flag); // Forbid calling this function if MPI has already been initialized @@ -76,8 +76,6 @@ inline void initialize(int &argc, char *argv[], ThreadSupportLevel required) { Kokkos::initialize(argc, argv); } -inline void initialize(int &argc, char *argv[]) { initialize(argc, argv, ThreadSupportLevel::Multiple); } - inline void finalize() { // Forbid calling this function if Kokkos has already been finalized or isn't yet initialized if (Kokkos::is_finalized() || !Kokkos::is_initialized()) {