Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove mdspan support #26

Merged
merged 1 commit into from
Apr 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 0 additions & 42 deletions .github/workflows/linux-mdspan.yaml

This file was deleted.

42 changes: 0 additions & 42 deletions .github/workflows/osx-mdspan.yaml

This file was deleted.

29 changes: 1 addition & 28 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,11 @@ project(KokkosComm VERSION 0.0.2)

option(KokkosComm_ENABLE_PERFTESTS "Build KokkosComm perf tests" ON)
option(KokkosComm_ENABLE_TESTS "Build KokkosComm perf tests" ON)
option(KokkosComm_ENABLE_MDSPAN "Build mdspan overloads" OFF)
option(KokkosComm_MDSPAN_IMPL "mdspan implementation to use [std, kokkos]" "kokkos")
option(KokkosComm_USE_STD_MDSPAN "Use std::mdspan (requires c++23)" OFF)
option(KokkosComm_USE_KOKKOS_MDSPAN "Use Kokkos mdspan implementation" OFF)


## resolve options
set(KOKKOSCOMM_ENABLE_PERFTESTS ${KokkosComm_ENABLE_PERFTESTS} CACHE BOOL "" FORCE)
set(KOKKOSCOMM_ENABLE_TESTS ${KokkosComm_ENABLE_TESTS} CACHE BOOL "" FORCE)
set(KOKKOSCOMM_ENABLE_MDSPAN ${KokkosComm_ENABLE_MDSPAN} CACHE BOOL "" FORCE)
set(KOKKOSCOMM_USE_KOKKOS_MDSPAN ${KokkosComm_USE_KOKKOS_MDSPAN} CACHE BOOL "" FORCE)
set(KOKKOSCOMM_USE_STD_MDSPAN ${KokkosComm_USE_STD_MDSPAN} CACHE BOOL "" FORCE)

find_package(Kokkos REQUIRED)
find_package(MPI REQUIRED)
Expand All @@ -29,26 +22,6 @@ message(STATUS "Kokkos MPI: MPI_CXX_COMPILE_DEFINITIONS = ${MPI_CXX_COMPILE_DEFI
message(STATUS "Kokkos MPI: MPI_CXX_INCLUDE_DIRS = ${MPI_CXX_INCLUDE_DIRS}")
message(STATUS "Kokkos MPI: MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}")
message(STATUS "Kokkos MPI: MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}")
if (KOKKOSCOMM_ENABLE_MDSPAN)
if (KOKKOSCOMM_USE_STD_MDSPAN)
message(STATUS "Kokkos MPI: mdspan support enabled (std::mdspan)")
elseif(KOKKOSCOMM_USE_KOKKOS_MDSPAN)
message(STATUS "Kokkos MPI: mdspan support enabled (kokkos/mdspan)")
else()
message(FATAL_ERROR "Kokkos MPI: mdspan support enabled but no MDSPAN implementation enabled!")
endif()
endif()

include(FetchContent)
if(KOKKOSCOMM_ENABLE_MDSPAN AND KOKKOSCOMM_USE_KOKKOS_MDSPAN)
message(STATUS "retrieving kokkos/mdspan...")
FetchContent_Declare(
kokkos_mdspan
GIT_REPOSITORY https://github.com/kokkos/mdspan.git
GIT_TAG 9ceface91483775a6c74d06ebf717bbb2768452f # mdspan-0.6.0
)
FetchContent_MakeAvailable(kokkos_mdspan)
endif()


include(cmake/flags.cmake)
Expand Down Expand Up @@ -126,4 +99,4 @@ endif()
if(KOKKOSCOMM_ENABLE_PERFTESTS)
enable_testing()
add_subdirectory(perf_tests)
endif()
endif()
20 changes: 2 additions & 18 deletions cmake/flags.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -59,24 +59,8 @@ function(kokkoscomm_add_cxx_flags)
target_compile_options(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} $<BUILD_INTERFACE:-Wno-gnu-zero-variadic-macro-arguments>)
endif()

# mdspan-related definitions
if (KOKKOSCOMM_ENABLE_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_ENABLE_MDSPAN)
if(KOKKOSCOMM_USE_STD_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_USE_STD_MDSPAN)
elseif(KOKKOSCOMM_USE_KOKKOS_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_USE_KOKKOS_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_MDSPAN_IN_EXPERIMENTAL)
endif()
endif()

# choose cxx standard
set_target_properties(${ADD_CXX_FLAGS_TARGET} PROPERTIES CXX_EXTENSIONS OFF)
if (KOKKOSCOMM_ENABLE_MDSPAN AND KOKKOSCOMM_USE_STD_MDSPAN)
target_compile_features(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_FEATURES_KEYWORD} cxx_std_23)
else()
target_compile_features(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_FEATURES_KEYWORD} cxx_std_20)
endif()

target_compile_features(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_FEATURES_KEYWORD} cxx_std_20)

endfunction()
endfunction()
11 changes: 5 additions & 6 deletions docs/api/core.rst
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ Core
* - MPI
- ``KokkosComm::``
- ``Kokkos::View``
- mdspan
* - MPI_Send
- send
- ✓
Expand All @@ -29,7 +28,7 @@ Core
Point-to-point
--------------

.. cpp:function:: template <KokkosExecutionSpace ExecSpace, ViewOrMdspan SendView> \
.. cpp:function:: template <KokkosExecutionSpace ExecSpace, KokkosView SendView> \
Req KokkosComm::isend(const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm)

MPI_Isend wrapper
Expand All @@ -43,7 +42,7 @@ Point-to-point
:tparam ExecSpace: A Kokkos execution space to operate in
:returns: A KokkosComm::Req representing the asynchronous communication and any lifetime-extended views.

.. cpp:function:: template <KokkosExecutionSpace ExecSpace, ViewOrMdspan SendView> \
.. cpp:function:: template <KokkosExecutionSpace ExecSpace, KokkosView SendView> \
void KokkosComm::send(const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm)

MPI_Send wrapper
Expand All @@ -56,7 +55,7 @@ Point-to-point
:tparam SendView: A Kokkos::View to send
:tparam ExecSpace: A Kokkos execution space to operate in

.. cpp:function:: template <KokkosExecutionSpace ExecSpace, ViewOrMdspan RecvView> \
.. cpp:function:: template <KokkosExecutionSpace ExecSpace, KokkosView RecvView> \
void KokkosComm::recv(const ExecSpace &space, RecvView &rv, int src, int tag, MPI_Comm comm)

MPI_Recv wrapper
Expand All @@ -73,7 +72,7 @@ Point-to-point
Collective
----------

.. cpp:function:: template <KokkosExecutionSpace ExecSpace, ViewOrMdspan SendView, ViewOrMdspan RecvView> \
.. cpp:function:: template <KokkosExecutionSpace ExecSpace, KokkosView SendView, KokkosView RecvView> \
void KokkosComm::reduce(const ExecSpace &space, const SendView &sv, const RecvView &rv, MPI_Op op, int root, MPI_Comm comm)

MPI_Reduce wrapper
Expand Down Expand Up @@ -107,4 +106,4 @@ Related Types
void KokkosComm::Req::keep_until_wait(const View &v)

Extend the lifetime of v at least until Req::wait() is called.
This is useful to prevent a View from being destroyed during an asynchronous MPI operation.
This is useful to prevent a View from being destroyed during an asynchronous MPI operation.
6 changes: 3 additions & 3 deletions docs/api/traits.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ Basic Traits
.. cpp:type:: template<typename View> \
KokkosComm::Traits

A common interface to access Kokkos::View- and std::mdspan-like types.
A common interface to access Kokkos::View-like types.

.. cpp:function:: static bool is_contiguous(const View &v)

Expand Down Expand Up @@ -41,7 +41,7 @@ Strategies for handling non-contiguous views
.. cpp:type:: template<typename View> \
KokkosComm::PackTraits

A common packing-related interface for Kokkos::View- and std::mdspan-like types.
A common packing-related interface for Kokkos::View-like types.

.. cpp:type:: packer_type

Expand All @@ -53,4 +53,4 @@ Strategies for handling non-contiguous views

.. cpp:function:: static bool needs_pack(const View &v)

:returns: true iff ``View`` ``v`` needs to be unpacked after being passed from MPI
:returns: true iff ``View`` ``v`` needs to be unpacked after being passed from MPI
4 changes: 0 additions & 4 deletions docs/design.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ Asynchronous MPI operations and view lifetimes
Non-contiguous Data
-------------------

- Packer::MpiDatatype which just constructs an MPI Datatype matching the mdspan and hands it off to MPI to deal with the non-contiguous data
- Packer::DeepCopy uses `Kokkos::deep_copy` to handle packing and unpacking of non-contiguous `Kokkos::View`. This requires an intermediate allocation, which only works for Kokkos Views, see `Device Data`_.

Device Data
Expand All @@ -18,6 +17,3 @@ Device Data
Contiguous device data is handed to MPI as-is.

For non-contiguous Kokkos::Views in a non-``Kokkos::HostSpace``, any temporary buffers are allocated in the same memory space as the view being operated on.

For non-contiguous mdspan, there is no standards-compliant way to get an allocator that can produce the same kind of allocation as the mdspan.
In that case, the Packer::MpiDatatype packer needs to be used, where a datatype is created to describe the mdspan (without accessing any of the mdspan's data!) and then that is handed off to the MPI implementation.
20 changes: 0 additions & 20 deletions docs/usage/getting_started.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,9 @@ A basic build:

ctest -V --test-dir /path/to/build/directory

A build with ``mdspan`` support through kokkos/mdspan.
You need to turn on ONE of ``KokkosComm_USE_STD_MDSPAN`` or ``KokkosComm_USE_KOKKOS_MDSPAN``.
As of March 2024, only Clang 18 has full ``std::mdspan`` support, so you will probably want ``KokkosComm_USE_KOKKOS_MDSPAN``.

.. code-block:: bash

cmake -S /path/to/kokkos-comm \
-B /path/to/build/directory \
-DKokkos_ROOT=/path/to/kokkos-install/ \
-DKokkosComm_ENABLE_MDSPAN=ON \
-DKokkosComm_USE_KOKKOS_MDSPAN=ON


Configuration Options
---------------------

* ``KokkosComm_ENABLE_MDSPAN``: (default=OFF) build with mdspan support
* Causes ``KOKKOSCOMM_ENABLE_MDSPAN`` to be defined in source files
* ``KokkosComm_USE_STD_MDSPAN``: (default=OFF) use std::mdspan as the mdspan implementation (if KokkosComm_ENABLE_MDSPAN)
* Causes ``KOKKOSCOMM_USE_STD_MDSPAN`` to be defined in source files
* ``KokkosComm_USE_KOKKOS_MDSPAN``: (default=OFF) retrieve and use kokkos/mdspan as the mdspan implementation (if KokkosComm_ENABLE_MDSPAN)
* Causes ``KOKKOSCOMM_USE_KOKKOS_MDSPAN`` to be defined in source files
* Causes ``KOKKOSCOMM_MDSPAN_IN_EXPERIMENTAL`` to be defined in source files
* ``KokkosComm_ENABLE_PERFTESTS``: (default=ON) build performance tests
* ``KokkosComm_ENABLE_TESTS``: (default=ON) build unit tests

Expand Down
5 changes: 1 addition & 4 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ target_include_directories(KokkosComm INTERFACE
$<INSTALL_INTERFACE:include>
)
target_link_libraries(KokkosComm INTERFACE MPI::MPI_CXX Kokkos::kokkos)
if(KOKKOSCOMM_USE_KOKKOS_MDSPAN)
target_link_libraries(KokkosComm INTERFACE mdspan)
endif()

file(GLOB_RECURSE KOKKOSCOMM_PUBLIC_HEADERS ${CMAKE_CURRENT_LIST_DIR}/*.hpp)
set(KOKKOSCOMM_PUBLIC_HEADERS ${KOKKOSCOMM_PUBLIC_HEADERS} PARENT_SCOPE)
set(KOKKOSCOMM_PUBLIC_HEADERS ${KOKKOSCOMM_PUBLIC_HEADERS} PARENT_SCOPE)
6 changes: 3 additions & 3 deletions src/KokkosComm.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,19 @@

namespace KokkosComm {

template <KokkosExecutionSpace ExecSpace, ViewOrMdspan SendView>
template <KokkosExecutionSpace ExecSpace, KokkosView SendView>
Req isend(const ExecSpace &space, const SendView &sv, int dest, int tag,
MPI_Comm comm) {
return Impl::isend(space, sv, dest, tag, comm);
}

template <KokkosExecutionSpace ExecSpace, ViewOrMdspan SendView>
template <KokkosExecutionSpace ExecSpace, KokkosView SendView>
void send(const ExecSpace &space, const SendView &sv, int dest, int tag,
MPI_Comm comm) {
return Impl::send(space, sv, dest, tag, comm);
}

template <KokkosExecutionSpace ExecSpace, ViewOrMdspan RecvView>
template <KokkosExecutionSpace ExecSpace, KokkosView RecvView>
void recv(const ExecSpace &space, RecvView &sv, int src, int tag,
MPI_Comm comm) {
return Impl::recv(space, sv, src, tag, comm);
Expand Down
2 changes: 1 addition & 1 deletion src/KokkosComm_collective.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

namespace KokkosComm {

template <ViewOrMdspan SendView, ViewOrMdspan RecvView,
template <KokkosView SendView, KokkosView RecvView,
KokkosExecutionSpace ExecSpace>
void reduce(const ExecSpace &space, const SendView &sv, const RecvView &rv,
MPI_Op op, int root, MPI_Comm comm) {
Expand Down
16 changes: 0 additions & 16 deletions src/KokkosComm_pack_traits.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
#include "KokkosComm_traits.hpp"

#include "KokkosComm_concepts.hpp"
#include "KokkosComm_mdspan.hpp"
#include "KokkosComm_packer.hpp"

/*! \brief Defines a common interface for packing and unpacking
Expand Down Expand Up @@ -47,19 +46,4 @@ struct PackTraits<View> {
}
};

#if KOKKOSCOMM_ENABLE_MDSPAN

template <Mdspan Span>
struct PackTraits<Span> {
using packer_type = Impl::Packer::MpiDatatype<Span>;

static bool needs_unpack(const Span &v) {
return !Traits<Span>::is_contiguous(v);
}
static bool needs_pack(const Span &v) {
return !Traits<Span>::is_contiguous(v);
}
};

#endif // KOKKOSCOMM_ENABLE_MDSPAN
} // namespace KokkosComm
Loading