From 81dff9b2e61609290c7e267fc8ba7c01109ddf0e Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 3 Jun 2023 06:21:25 -0500 Subject: [PATCH 01/45] move as buffers when data type is trivially copyable Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 856ec14f4538..c86b213542fc 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -7,6 +7,7 @@ /// \file parallel/algorithms/uninitialized_copy.hpp #pragma once +#include #if defined(DOXYGEN) namespace hpx { @@ -252,15 +253,23 @@ namespace hpx::parallel { InIter1 first, std::size_t count, InIter2 dest, util::cancellation_token& tok) { - return {std::next(first, count), - util::loop_with_cleanup_n_with_token( - first, count, dest, tok, - [](InIter1 it, InIter2 dest) -> void { - hpx::construct_at(std::addressof(*dest), *it); - }, - [](InIter2 dest) -> void { - std::destroy_at(std::addressof(*dest)); - })}; + using T = ::hpx::traits::iter_value_t; + if constexpr (std::is_trivially_copyable_v) + { + InIter1 dest = std::advance(first, count); + std::memcpy(std::addressof(*first), std::addressof(*dest), + count * sizeof(T)); + } + else + return {std::next(first, count), + util::loop_with_cleanup_n_with_token( + first, count, dest, tok, + [](InIter1 it, InIter2 dest) -> void { + hpx::construct_at(std::addressof(*dest), *it); + }, + [](InIter2 dest) -> void { + std::destroy_at(std::addressof(*dest)); + })}; } /////////////////////////////////////////////////////////////////////// @@ -334,10 +343,8 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_copy( - first, dest, [last](InIter1 first, FwdIter2) -> bool { - return first != last; - }); + return sequential_uninitialized_copy_n( + first, std::distance(first, last), dest); } template ; + if constexpr (std::is_trivially_copyable_v) + { + std::memcpy(std::addressof(*first), + std::addressof(*dest), count * sizeof(T)); + } for (/* */; count > 0; ++first, (void) ++current, --count) { hpx::construct_at(std::addressof(*current), *first); From 8bd974069dffb06a9db55be83fa348d8f954c744 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 11:49:29 -0500 Subject: [PATCH 02/45] copy as buffers for triviably copyable types Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index c86b213542fc..f09da5046155 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -235,7 +235,7 @@ namespace hpx::parallel { { hpx::construct_at(std::addressof(*current), *first); } - return util::in_out_result{first, current}; + return {first, current}; } catch (...) { @@ -256,11 +256,15 @@ namespace hpx::parallel { using T = ::hpx::traits::iter_value_t; if constexpr (std::is_trivially_copyable_v) { - InIter1 dest = std::advance(first, count); - std::memcpy(std::addressof(*first), std::addressof(*dest), + std::memcpy(std::addressof(*dest), std::addressof(*first), count * sizeof(T)); + std::advance(first, count); + std::advance(dest, count); + + return util::in_out_result{first, dest}; } else + { return {std::next(first, count), util::loop_with_cleanup_n_with_token( first, count, dest, tok, @@ -270,6 +274,7 @@ namespace hpx::parallel { [](InIter2 dest) -> void { std::destroy_at(std::addressof(*dest)); })}; + } } /////////////////////////////////////////////////////////////////////// @@ -424,29 +429,39 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { - FwdIter2 current = dest; - try + using T = ::hpx::traits::iter_value_t; + + if constexpr (std::is_trivially_copyable_v) { - using T = ::hpx::traits::iter_value_t; - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*first), - std::addressof(*dest), count * sizeof(T)); - } - for (/* */; count > 0; ++first, (void) ++current, --count) - { - hpx::construct_at(std::addressof(*current), *first); - } - return util::in_out_result{ - first, current}; + std::memcpy(std::addressof(*dest), std::addressof(*first), + count * sizeof(T)); + std::advance(first, count); + std::advance(dest, count); + + return util::in_out_result{first, dest}; } - catch (...) + else { - for (/* */; dest != current; ++dest) + FwdIter2 current = dest; + + try + { + for (/* */; count > 0; + ++first, (void) ++current, --count) + { + hpx::construct_at(std::addressof(*current), *first); + } + return util::in_out_result{ + first, current}; + } + catch (...) { - std::destroy_at(std::addressof(*dest)); + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; } - throw; } } From 78069731263880e4ca13cdea8c92d1337050fa5d Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 13:53:22 -0500 Subject: [PATCH 03/45] added cancellation token Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index f09da5046155..add66abe7f66 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -7,7 +7,6 @@ /// \file parallel/algorithms/uninitialized_copy.hpp #pragma once -#include #if defined(DOXYGEN) namespace hpx { @@ -348,8 +347,10 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy, InIter1 first, Sent last, FwdIter2 dest) { + util::cancellation_token tok; + return sequential_uninitialized_copy_n( - first, std::distance(first, last), dest); + first, std::distance(first, last), dest, tok); } template Date: Sun, 4 Jun 2023 14:50:03 -0500 Subject: [PATCH 04/45] added unseq support Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 118 ++++++++++++++++-- .../include/hpx/parallel/unseq/loop.hpp | 113 +++++++++++++++++ 2 files changed, 224 insertions(+), 7 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index add66abe7f66..414f5a657b34 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -198,6 +198,7 @@ namespace hpx { #include #include #include +#include #include #include #include @@ -210,6 +211,7 @@ namespace hpx { #include #include +#include #include #include #include @@ -246,10 +248,63 @@ namespace hpx::parallel { } } + template + util::in_out_result sequential_uninitialized_copy_n( + InIter1 first, FwdIter2 dest, std::size_t count, + std::true_type /*is_unseq*/) + { + FwdIter2 current = dest; + try + { + // clang-format on + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/* */; count != 0; + (void) ++first, + ++current, count--) + { + hpx::construct_at(std::addressof(*current), *first); + } + // clang-format off + + return {first, current}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + + template + util::in_out_result sequential_uninitialized_copy_n( + InIter1 first, FwdIter2 dest, std::size_t count, + std::false_type /*is_unseq*/) + { + FwdIter2 current = dest; + try + { + for (/* */; count != 0; (void) ++first, ++current, count--) + { + hpx::construct_at(std::addressof(*current), *first); + } + return {first, current}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + /////////////////////////////////////////////////////////////////////// template util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, std::size_t count, InIter2 dest, + InIter1 first, InIter2 dest, std::size_t count, util::cancellation_token& tok) { using T = ::hpx::traits::iter_value_t; @@ -308,7 +363,7 @@ namespace hpx::parallel { return std::make_pair(dest, util::get_second_element( sequential_uninitialized_copy_n( - get<0>(iters), part_size, dest, tok))); + get<0>(iters), dest, part_size, tok))); }, // finalize, called once if no error occurred [dest, first, count](auto&& data) mutable @@ -345,12 +400,10 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { - util::cancellation_token tok; - return sequential_uninitialized_copy_n( - first, std::distance(first, last), dest, tok); + ExPolicy, first, dest, std::distance(first, last)); } template + + // vectorized overload + template >> + static util::in_out_result sequential( + ExPolicy, InIter first, std::size_t count, FwdIter2 dest) + { + using T = ::hpx::traits::iter_value_t; + + if constexpr (std::is_trivially_copyable_v) + { + std::memcpy(std::addressof(*dest), std::addressof(*first), + count * sizeof(T)); + std::advance(first, count); + std::advance(dest, count); + + return util::in_out_result{first, dest}; + } + else + { + FwdIter2 current = dest; + + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/* */; count > 0; + ++first, (void) ++current, --count) + { + hpx::construct_at(std::addressof(*current), *first); + } + // clang-format on + + return util::in_out_result{ + first, current}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + } + + // non vectorized overload + template >> static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { diff --git a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp index 32fa4ec41275..131f62492014 100644 --- a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp @@ -200,6 +200,119 @@ namespace hpx::parallel::util { return call(base_idx, it, count, HPX_FORWARD(F, f)); } }; + + template + struct unseq_loop_with_cleanup_n + { + /////////////////////////////////////////////////////////////////// + template + static FwdIter call( + FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + { + FwdIter base = it; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < count; + (void) ++it, i += 4) // -V112 + { + HPX_INVOKE(f, it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it) + { + HPX_INVOKE(f, it); + } + // clang-format on + + return it; + } + catch (...) + { + for (/**/; base != it; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + + template + static FwdIter call(Iter it, std::size_t num, FwdIter dest, F&& f, + Cleanup&& cleanup) + { + FwdIter base = dest; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + + //clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; + i < count; + (void) ++it, ++dest, + i += 4) // -V112 + { + HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); + } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; count < num; + (void) ++count, + ++it, ++dest) + { + HPX_INVOKE(f, it, dest); + } + //clang-format on + + return dest; + } + catch (...) + { + for (/**/; base != dest; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token( + FwdIter it, std::size_t num, CancelToken& tok, F&& f, + Cleanup&& cleanup) + { + // check at the start of a partition only + if (tok.was_cancelled()) + return it; + + return call( + it, num, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + static FwdIter call_with_token(Iter it, std::size_t num, + FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) + { + // check at the start of a partition only + if (tok.was_cancelled()) + return dest; + + return call(it, num, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + }; } // namespace detail /////////////////////////////////////////////////////////////////////////// From 20d9a17ec6de8391f15220cce4fda7152af41931 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 15:24:54 -0500 Subject: [PATCH 05/45] fixed unused error Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 414f5a657b34..c96fdacb5b32 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -403,7 +403,7 @@ namespace hpx::parallel { ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { return sequential_uninitialized_copy_n( - ExPolicy, first, dest, std::distance(first, last)); + ExPolicy policy, first, dest, std::distance(first, last)); } template >> + template static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { From 79cbd6b79bfbcacfbe007a37324481d660261da9 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 15:32:28 -0500 Subject: [PATCH 06/45] unedd parameter Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index c96fdacb5b32..fbe33787c913 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -403,7 +403,7 @@ namespace hpx::parallel { ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { return sequential_uninitialized_copy_n( - ExPolicy policy, first, dest, std::distance(first, last)); + policy, first, dest, std::distance(first, last)); } template Date: Sun, 4 Jun 2023 19:04:57 -0500 Subject: [PATCH 07/45] enable if else error to be fixed Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_copy.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index fbe33787c913..2e8e649cf5bf 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -256,14 +256,14 @@ namespace hpx::parallel { FwdIter2 current = dest; try { - // clang-format on + // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/* */; count != 0; (void) ++first, ++current, count--) { hpx::construct_at(std::addressof(*current), *first); } - // clang-format off + // clang-format on return {first, current}; } @@ -402,8 +402,9 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_copy_n( - policy, first, dest, std::distance(first, last)); + return sequential_uninitialized_copy_n(first, dest, + std::distance(first, last), + hpx::is_unsequenced_execution_policy_v()); } template Date: Sun, 4 Jun 2023 19:08:37 -0500 Subject: [PATCH 08/45] cf Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 2e8e649cf5bf..5d3100a1ac85 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -480,7 +480,6 @@ namespace hpx::parallel { { } - // vectorized overload template Date: Sun, 4 Jun 2023 20:09:18 -0500 Subject: [PATCH 09/45] unused param Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 5d3100a1ac85..d0108f4dc383 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -400,7 +400,7 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy, InIter1 first, Sent last, FwdIter2 dest) { return sequential_uninitialized_copy_n(first, dest, std::distance(first, last), From 4008ad96e1a7842923e6ce4719df9ab95f483d8f Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 04:59:55 -0500 Subject: [PATCH 10/45] customisation point for loop_with_cleanup Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 5 +- .../include/hpx/parallel/util/loop.hpp | 185 ++++++++++++------ 2 files changed, 130 insertions(+), 60 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index d0108f4dc383..ec05c700edaf 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -257,9 +257,8 @@ namespace hpx::parallel { try { // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/* */; count != 0; - (void) ++first, - ++current, count--) + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/* */; count != 0; (void) ++first, ++current, count--) { hpx::construct_at(std::addressof(*current), *first); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 9bf131fd7c01..8503babc3a46 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -38,23 +38,65 @@ namespace hpx::parallel::util { HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( Begin it, End end, F&& f) { + return call(hpx::execution::seq, it, end, HPX_FORWARD(F, f)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( + Begin it, End end, CancelToken& tok, F&& f) + { + return call( + hpx::execution::seq, it, end, tok, HPX_FORWARD(F, f)); + } + + template && + !hpx::is_parallel_execution_policy_v)> + HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( + ExPolicy, Begin it, End end, F&& f) + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != end; ++it) { HPX_INVOKE(f, it); } + // clang-format on + return it; } - template + template && + !hpx::is_parallel_execution_policy_v)> + HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( + ExPolicy, Begin it, End end, F&& f) + { + for (/**/; it != end; ++it) + { + HPX_INVOKE(f, it); + } + + return it; + } + + template HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( - Begin it, End end, CancelToken& tok, F&& f) + typename ExPolicy policy, Begin it, End end, CancelToken& tok, + F&& f) { // check at the start of a partition only if (tok.was_cancelled()) return it; - return call(it, end, HPX_FORWARD(F, f)); + return call( + HPX_FORWARD(ExPolicy, policy), it, end, HPX_FORWARD(F, f)); } }; } // namespace detail @@ -64,10 +106,11 @@ namespace hpx::parallel::util { private: template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&&, + tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&& policy, Begin begin, End end, F&& f) { - return detail::loop::call(begin, end, HPX_FORWARD(F, f)); + return detail::loop::call( + HPX_FORWARD(ExPolicy, policy), begin, end, HPX_FORWARD(F, f)); } template ::call( + return detail::loop::call(HPX_FORWARD(ExPolicy, policy), begin, end, tok, HPX_FORWARD(F, f)); } }; @@ -575,31 +618,50 @@ namespace hpx::parallel::util { struct loop_with_cleanup { /////////////////////////////////////////////////////////////////// - template - static FwdIter call( - FwdIter it, FwdIter last, F&& f, Cleanup&& cleanup) + template + static FwdIter call(ExPolicy&& policy, FwdIter first, FwdIter last, + F&& f, Cleanup&& cleanup) { - FwdIter base = it; + return call(HPX_FORWARD(ExPolicy, policy), first, last, first, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template && + !hpx::is_parallel_execution_policy_v)> + static FwdIter call(ExPolicy&& policy, Iter it, Iter last, + FwdIter dest, F&& f, Cleanup&& cleanup) + { + FwdIter base = dest; try { - for (/**/; it != last; ++it) - { - HPX_INVOKE(f, it); - } - return it; + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != last; (void) ++it, ++dest) + f(it, dest); + // clang-format on + + return dest; } catch (...) { - for (/**/; base != it; ++base) - cleanup(base); + for (/**/; base != dest; ++base) + { + HPX_INVOKE(cleanup, base); + } throw; } } - template - static FwdIter call( - Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) + template )> + static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, + F&& f, Cleanup&& cleanup) { FwdIter base = dest; try @@ -620,24 +682,52 @@ namespace hpx::parallel::util { }; } // namespace detail - /////////////////////////////////////////////////////////////////////////// - template - HPX_FORCEINLINE constexpr Iter loop_with_cleanup( - Iter it, Iter last, F&& f, Cleanup&& cleanup) + inline constexpr struct loop_with_cleanup_t final + : hpx::functional::detail::tag_fallback { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup::call( - it, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } + private: + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin + tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, + ExPolicy&& policy, Iter it, Iter last, FwdIter dest, F&& f, + Cleanup&& cleanup) + { + return detail::loop_with_cleanup::call( + HPX_FORWARD(ExPolicy, policy), it, last, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } - template - HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup( - Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) - { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup::call( - it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin + tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, ExPolicy&&, + ExPolicy&& policy, FwdIter first, FwdIter last, F&& f, + Cleanup&& cleanup) + { + return detail::loop_with_cleanup::call( + HPX_FORWARD(ExPolicy, policy), it, last, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + } loop_with_cleanup{}; + + // /////////////////////////////////////////////////////////////////////////// + // template + // HPX_FORCEINLINE constexpr Iter loop_with_cleanup( + // Iter it, Iter last, F&& f, Cleanup&& cleanup) + // { + // using cat = typename std::iterator_traits::iterator_category; + // return detail::loop_with_cleanup::call( + // it, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + // } + + // template + // HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup( + // Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) + // { + // using cat = typename std::iterator_traits::iterator_category; + // return detail::loop_with_cleanup::call( + // it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + // } /////////////////////////////////////////////////////////////////////////// namespace detail { @@ -742,25 +832,6 @@ namespace hpx::parallel::util { }; } // namespace detail - /////////////////////////////////////////////////////////////////////////// - template - HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n( - Iter it, std::size_t count, F&& f, Cleanup&& cleanup) - { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call( - it, count, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } - - template - HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup_n( - Iter it, std::size_t count, FwdIter dest, F&& f, Cleanup&& cleanup) - { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call( - it, count, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } - template HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) From bfb59625b71ae28d96a7cc8931676efa3ca89a6b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:04:15 -0500 Subject: [PATCH 11/45] fixed ambiguos funtion error Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_copy.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index ec05c700edaf..99f5c9fd2e43 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -481,8 +481,9 @@ namespace hpx::parallel { // vectorized overload template >> + HPX_CONCEPT_REQUIRES_( // forces hpx::execution::unseq + hpx::is_unsequenced_execution_policy_v && + !hpx::is_parallel_execution_policy_v)> static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { @@ -527,7 +528,9 @@ namespace hpx::parallel { } // non vectorized overload - template + template )> static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { From 85159db0c69bb2d4d5eb3d7c54319eb4caefb74e Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:17:19 -0500 Subject: [PATCH 12/45] fixed template and typename mismatch Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 8503babc3a46..e961bffd614a 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -56,7 +56,7 @@ namespace hpx::parallel::util { hpx::is_unsequenced_execution_policy_v && !hpx::is_parallel_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy, Begin it, End end, F&& f) + ExPolicy&&, Begin it, End end, F&& f) { // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE @@ -70,12 +70,12 @@ namespace hpx::parallel::util { } template && !hpx::is_parallel_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy, Begin it, End end, F&& f) + ExPolicy&&, Begin it, End end, F&& f) { for (/**/; it != end; ++it) { @@ -88,8 +88,7 @@ namespace hpx::parallel::util { template HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( - typename ExPolicy policy, Begin it, End end, CancelToken& tok, - F&& f) + ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) { // check at the start of a partition only if (tok.was_cancelled()) @@ -116,7 +115,7 @@ namespace hpx::parallel::util { template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&&, + tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&& policy, Begin begin, End end, CancelToken& tok, F&& f) { return detail::loop::call(HPX_FORWARD(ExPolicy, policy), @@ -632,8 +631,8 @@ namespace hpx::parallel::util { HPX_CONCEPT_REQUIRES_( // forces hpx::execution::unseq hpx::is_unsequenced_execution_policy_v && !hpx::is_parallel_execution_policy_v)> - static FwdIter call(ExPolicy&& policy, Iter it, Iter last, - FwdIter dest, F&& f, Cleanup&& cleanup) + static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, + F&& f, Cleanup&& cleanup) { FwdIter base = dest; try @@ -686,23 +685,22 @@ namespace hpx::parallel::util { : hpx::functional::detail::tag_fallback { private: - template + template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, - ExPolicy&& policy, Iter it, Iter last, FwdIter dest, F&& f, - Cleanup&& cleanup) + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + Begin it, End last, FwdIter dest, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( HPX_FORWARD(ExPolicy, policy), it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - template + template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, ExPolicy&&, - ExPolicy&& policy, FwdIter first, FwdIter last, F&& f, - Cleanup&& cleanup) + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + Begin first, End last, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( HPX_FORWARD(ExPolicy, policy), it, last, HPX_FORWARD(F, f), From 9dc7a5f441a73cf7d17688e12bf99f6b7e09dd71 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:50:18 -0500 Subject: [PATCH 13/45] template arguments were not provided in certain cases Signed-off-by: Hari Hara Naveen S --- .../core/algorithms/include/hpx/parallel/util/loop.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index e961bffd614a..7cf1da9402f8 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -688,22 +688,22 @@ namespace hpx::parallel::util { template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, - Begin it, End last, FwdIter dest, F&& f, Cleanup&& cleanup) + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), it, last, dest, + HPX_FORWARD(ExPolicy, policy), first, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, Begin first, End last, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), it, last, HPX_FORWARD(F, f), + HPX_FORWARD(ExPolicy, policy), first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } } loop_with_cleanup{}; From f108785c3c2d68abd12061c5286680b4df2f889f Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 06:00:47 -0500 Subject: [PATCH 14/45] added concept to resolve ambiguity Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 7cf1da9402f8..686e5a1aaadf 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -86,7 +86,8 @@ namespace hpx::parallel::util { } template + typename CancelToken, typename F, + HPX_CONCEPT_REQUIRES_(hpx::is_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) { From e76f7ef42704e69d5c63e874c4c237aba881efa0 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 09:22:02 -0500 Subject: [PATCH 15/45] fixed ambiguity using concepts Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 686e5a1aaadf..ceebd231934f 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -42,7 +42,9 @@ namespace hpx::parallel::util { } template + typename F, + HPX_CONCEPT_REQUIRES_(hpx::traits::is_iterator_v&& + hpx::traits::is_forward_iterator_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( Begin it, End end, CancelToken& tok, F&& f) { @@ -72,8 +74,7 @@ namespace hpx::parallel::util { template && - !hpx::is_parallel_execution_policy_v)> + hpx::is_sequenced_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) { From 502adfa0614ca8843c51fe94f400fc86f8a092ab Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 10:54:22 -0500 Subject: [PATCH 16/45] fixed inspect and include errors Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 99f5c9fd2e43..a59b8118dee6 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -403,7 +403,7 @@ namespace hpx::parallel { { return sequential_uninitialized_copy_n(first, dest, std::distance(first, last), - hpx::is_unsequenced_execution_policy_v()); + hpx::is_unsequenced_execution_policy_v); } template #include #include +#include #include #include #include @@ -61,7 +62,7 @@ namespace hpx::parallel::util { ExPolicy&&, Begin it, End end, F&& f) { // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != end; ++it) { HPX_INVOKE(f, it); From f265e7f4514d2038c34388bb7b7d2d7412f02134 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 01:22:39 -0500 Subject: [PATCH 17/45] used hpx::parallel::util::transfer for uninit_copy Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 229 ++---------------- .../include/hpx/parallel/util/transfer.hpp | 114 +++++++++ 2 files changed, 139 insertions(+), 204 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index a59b8118dee6..2b5c0fc4fd3a 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -206,6 +206,7 @@ namespace hpx { #include #include #include +#include #include #include @@ -225,116 +226,11 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// - template - util::in_out_result sequential_uninitialized_copy( - InIter1 first, FwdIter2 dest, Cond cond) - { - FwdIter2 current = dest; - try - { - for (/* */; cond(first, current); (void) ++first, ++current) - { - hpx::construct_at(std::addressof(*current), *first); - } - return {first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template - util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, FwdIter2 dest, std::size_t count, - std::true_type /*is_unseq*/) - { - FwdIter2 current = dest; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/* */; count != 0; (void) ++first, ++current, count--) - { - hpx::construct_at(std::addressof(*current), *first); - } - // clang-format on - - return {first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template - util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, FwdIter2 dest, std::size_t count, - std::false_type /*is_unseq*/) - { - FwdIter2 current = dest; - try - { - for (/* */; count != 0; (void) ++first, ++current, count--) - { - hpx::construct_at(std::addressof(*current), *first); - } - return {first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - /////////////////////////////////////////////////////////////////////// - template - util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, InIter2 dest, std::size_t count, - util::cancellation_token& tok) - { - using T = ::hpx::traits::iter_value_t; - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*dest), std::addressof(*first), - count * sizeof(T)); - std::advance(first, count); - std::advance(dest, count); - - return util::in_out_result{first, dest}; - } - else - { - return {std::next(first, count), - util::loop_with_cleanup_n_with_token( - first, count, dest, tok, - [](InIter1 it, InIter2 dest) -> void { - hpx::construct_at(std::addressof(*dest), *it); - }, - [](InIter2 dest) -> void { - std::destroy_at(std::addressof(*dest)); - })}; - } - } - /////////////////////////////////////////////////////////////////////// template typename util::detail::algorithm_result>::type - parallel_sequential_uninitialized_copy_n( + parallel_uninitialized_copy_n( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { if (count == 0) @@ -354,14 +250,15 @@ namespace hpx::parallel { call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [tok](zip_iterator t, std::size_t part_size) mutable + [tok, policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); FwdIter2 dest = get<1>(iters); return std::make_pair(dest, util::get_second_element( - sequential_uninitialized_copy_n( + hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), get<0>(iters), dest, part_size, tok))); }, // finalize, called once if no error occurred @@ -399,11 +296,11 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy&& policy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_copy_n(first, dest, - std::distance(first, last), - hpx::is_unsequenced_execution_policy_v); + std::size_t n = std::distance(first, last); + return hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), first, n, dest); } template >::type parallel(ExPolicy&& policy, Iter first, Sent last, FwdIter2 dest) { - return parallel_sequential_uninitialized_copy_n( + return parallel_uninitialized_copy_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last), dest); } @@ -437,13 +334,16 @@ namespace hpx::parallel { template - static util::in_out_result sequential(ExPolicy, - InIter1 first, Sent1 last, FwdIter2 dest, Sent2 last_d) + static util::in_out_result sequential( + ExPolicy&& policy, InIter1 first, Sent1 last, FwdIter2 dest, + Sent2 last_d) { - return sequential_uninitialized_copy(first, dest, - [last, last_d](InIter1 first, FwdIter2 current) -> bool { - return !(first == last || current == last_d); - }); + std::size_t const dist1 = detail::distance(first, last); + std::size_t const dist2 = detail::distance(dest, last_d); + std::size_t dist = dist1 <= dist2 ? dist1 : dist2; + + return hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), first, dist, dest); } template && - !hpx::is_parallel_execution_policy_v)> - static util::in_out_result sequential( - ExPolicy, InIter first, std::size_t count, FwdIter2 dest) - { - using T = ::hpx::traits::iter_value_t; - - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*dest), std::addressof(*first), - count * sizeof(T)); - std::advance(first, count); - std::advance(dest, count); - - return util::in_out_result{first, dest}; - } - else - { - FwdIter2 current = dest; - - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/* */; count > 0; - ++first, (void) ++current, --count) - { - hpx::construct_at(std::addressof(*current), *first); - } - // clang-format on - - return util::in_out_result{ - first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - } - // non vectorized overload template )> static util::in_out_result sequential( - ExPolicy, InIter first, std::size_t count, FwdIter2 dest) + ExPolicy&& policy, InIter first, std::size_t count, + FwdIter2 dest) { - using T = ::hpx::traits::iter_value_t; - - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*dest), std::addressof(*first), - count * sizeof(T)); - std::advance(first, count); - std::advance(dest, count); - - return util::in_out_result{first, dest}; - } - else - { - FwdIter2 current = dest; - - try - { - for (/* */; count > 0; - ++first, (void) ++current, --count) - { - hpx::construct_at(std::addressof(*current), *first); - } - return util::in_out_result{ - first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } + return hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), first, count, dest); } template @@ -576,7 +397,7 @@ namespace hpx::parallel { parallel( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { - return parallel_sequential_uninitialized_copy_n( + return parallel_uninitialized_copy_n( HPX_FORWARD(ExPolicy, policy), first, count, dest); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index dd93874fff7e..c67139787914 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -38,6 +38,9 @@ namespace hpx::parallel::util { template struct move_n_helper; + template + struct uninit_copy_n_helper; + /////////////////////////////////////////////////////////////////////// template HPX_FORCEINLINE std::enable_if_t, char*> to_ptr( @@ -348,4 +351,115 @@ namespace hpx::parallel::util { std::decay_t>; return detail::move_n_helper::call(first, count, dest); } + + // helpers for uninit_copy_n + namespace detail { + // Customization point for optimizing copy_n operations + template + struct uninit_copy_n_helper + { + template )> + HPX_FORCEINLINE static constexpr in_out_result + call(ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + + try + { + std::size_t count( + num & static_cast(-4)); // -V112 + for (std::size_t i = 0; i < count; i += 4) //-V112 + { + hpx::construct_at(std::addressof(*current++), *first++); + // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) + hpx::construct_at(std::addressof(*current++), *first++); + // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) + hpx::construct_at(std::addressof(*current++), *first++); + // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) + hpx::construct_at(std::addressof(*current++), *first++); + } + for (/**/; count < num; count++) + { + hpx::construct_at(std::addressof(*current++), *first++); + } + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + + template )> + HPX_FORCEINLINE static constexpr in_out_result + call(ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < num; i++) //-V112 + { + hpx::construct_at(std::addressof(*current++), *first++); + } + // clang-format on + + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + }; + + template + struct uninit_copy_n_helper + { + template + HPX_FORCEINLINE static in_out_result call(ExPolicy, + InIter first, std::size_t count, OutIter dest) noexcept + { + return copy_memmove(first, count, dest); + } + }; + } // namespace detail + + template + HPX_FORCEINLINE constexpr in_out_result uninit_copy_n( + InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_copy_n_helper::call( + hpx::execution::seq, first, count, dest); + } + + template + HPX_FORCEINLINE constexpr in_out_result uninit_copy_n( + ExPolicy&& policy, InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_copy_n_helper::call( + HPX_FORWARD(ExPolicy, policy), first, count, dest); + } } // namespace hpx::parallel::util From ad499de06921dc55e04af6e37b099408122398d1 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 02:04:49 -0500 Subject: [PATCH 18/45] try in constexpr function fixed Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index c67139787914..8226d757aa7b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -17,6 +17,7 @@ #include #include // for std::memmove #include +#include #include #include @@ -361,7 +362,7 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static constexpr in_out_result + HPX_FORCEINLINE static in_out_result call(ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; @@ -400,7 +401,7 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static constexpr in_out_result + HPX_FORCEINLINE static in_out_result call(ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; From 7030661055ab23781d4b972b724c8bc7fbff79ca Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 04:22:09 -0500 Subject: [PATCH 19/45] fixed missing includes Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_copy.hpp | 6 ++---- .../algorithms/include/hpx/parallel/util/transfer.hpp | 10 ++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 2b5c0fc4fd3a..cacc9c3b09a3 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -243,8 +243,6 @@ namespace hpx::parallel { using zip_iterator = hpx::util::zip_iterator; using partition_result_type = std::pair; - util::cancellation_token tok; - return util::partitioner_with_cleanup, partition_result_type>:: call( @@ -259,7 +257,7 @@ namespace hpx::parallel { util::get_second_element( hpx::parallel::util::uninit_copy_n( HPX_FORWARD(ExPolicy, policy), - get<0>(iters), dest, part_size, tok))); + get<0>(iters), part_size, dest))); }, // finalize, called once if no error occurred [dest, first, count](auto&& data) mutable @@ -381,7 +379,7 @@ namespace hpx::parallel { // non vectorized overload template )> static util::in_out_result sequential( ExPolicy&& policy, InIter first, std::size_t count, diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 8226d757aa7b..bee76809bc8b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include #include @@ -362,8 +364,8 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static in_out_result - call(ExPolicy, InIter first, std::size_t num, OutIter dest) + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; @@ -401,8 +403,8 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static in_out_result - call(ExPolicy, InIter first, std::size_t num, OutIter dest) + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; try From 5a4facf604ecfe8c7ce4facc76e040abf7da8f86 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 04:28:13 -0500 Subject: [PATCH 20/45] variable name Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index cacc9c3b09a3..7fd8495aa91f 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -248,7 +248,7 @@ namespace hpx::parallel { call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [tok, policy](zip_iterator t, std::size_t part_size) mutable + [policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); From 17a6a40a812e0e11f22ed2f9cecdcbb99c5ea7a3 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 05:36:35 -0500 Subject: [PATCH 21/45] changed concept Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index bee76809bc8b..82739b98f695 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -402,7 +402,7 @@ namespace hpx::parallel::util { template )> + !hpx::is_unsequenced_execution_policy_v)> HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { From 463e1f2988da2954000de0a0d9aa8bc9fe6d6b6e Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 06:40:48 -0500 Subject: [PATCH 22/45] fixed concept ambiguity Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 82739b98f695..668453c21c13 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -363,7 +363,7 @@ namespace hpx::parallel::util { { template )> + !hpx::is_unsequenced_execution_policy_v)> HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { @@ -402,7 +402,7 @@ namespace hpx::parallel::util { template )> + hpx::is_unsequenced_execution_policy_v)> HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { From 4a0f920cbefee0a06098da90dcab65764cf05392 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 10 Jun 2023 04:01:55 -0500 Subject: [PATCH 23/45] fixes extra free which happens during tests Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/transfer.hpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 668453c21c13..2e63c86dcb13 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -372,20 +372,21 @@ namespace hpx::parallel::util { try { std::size_t count( - num & static_cast(-4)); // -V112 - for (std::size_t i = 0; i < count; i += 4) //-V112 + num & static_cast(-4)); // -V112 + for (std::size_t i = 0; i < count; + i += 4, ++current, ++first) //-V112 { - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*current), *first); // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); } for (/**/; count < num; count++) { - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); } return in_out_result{ HPX_MOVE(first), HPX_MOVE(current)}; @@ -413,7 +414,7 @@ namespace hpx::parallel::util { HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < num; i++) //-V112 { - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); } // clang-format on From 45ce76b305455dc10e715ff20f2ce4a0d70af74c Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 10 Jun 2023 08:06:31 -0500 Subject: [PATCH 24/45] fixed bug Signed-off-by: Hari Hara Naveen S --- .../algorithms/include/hpx/parallel/util/transfer.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 2e63c86dcb13..9994aab89c8e 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -384,9 +384,9 @@ namespace hpx::parallel::util { // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) hpx::construct_at(std::addressof(*++current), *++first); } - for (/**/; count < num; count++) + for (/**/; count < num; ++count, ++current, ++first) { - hpx::construct_at(std::addressof(*++current), *++first); + hpx::construct_at(std::addressof(*current), *first); } return in_out_result{ HPX_MOVE(first), HPX_MOVE(current)}; @@ -412,9 +412,9 @@ namespace hpx::parallel::util { { // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (std::size_t i = 0; i < num; i++) //-V112 + for (std::size_t i = 0; i < num; ++i, ++current, ++first) //-V112 { - hpx::construct_at(std::addressof(*++current), *++first); + hpx::construct_at(std::addressof(*current), *first); } // clang-format on From eedc23910709c47de16b3cdf8d86863ccd999e35 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 10 Jun 2023 10:45:54 -0500 Subject: [PATCH 25/45] manual unrolling causing errors Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/transfer.hpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 9994aab89c8e..c3ea0ea173bb 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -371,20 +371,8 @@ namespace hpx::parallel::util { try { - std::size_t count( - num & static_cast(-4)); // -V112 - for (std::size_t i = 0; i < count; - i += 4, ++current, ++first) //-V112 - { - hpx::construct_at(std::addressof(*current), *first); - // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*++current), *++first); - // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*++current), *++first); - // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*++current), *++first); - } - for (/**/; count < num; ++count, ++current, ++first) + for (std::size_t i = 0; i < num; + ++i, ++current, ++first) //-V112 { hpx::construct_at(std::addressof(*current), *first); } From 5fe20d004b3620802046339590b4ecc116e85978 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 12:23:10 -0500 Subject: [PATCH 26/45] uninit default construct Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 83 +++--------- .../include/hpx/parallel/util/loop.hpp | 118 ++++++++++++------ 2 files changed, 96 insertions(+), 105 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 5a485c16718a..c3989037a1cd 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -193,44 +193,18 @@ namespace hpx::parallel { namespace detail { /// \cond NOINTERNAL - // provide our own implementation of - // std::uninitialized_default_construct as some versions of MSVC - // horribly fail at compiling it for some types T - template - Iter std_uninitialized_default_construct(Iter first, Sent last) - { - using value_type = typename std::iterator_traits::value_type; - - Iter s_first = first; - try - { - for (/* */; first != last; ++first) - { - ::new (std::addressof(*first)) value_type; - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - /////////////////////////////////////////////////////////////////////// - template - InIter sequential_uninitialized_default_construct_n(InIter first, - std::size_t count, - util::cancellation_token& tok) + template + InIter sequential_uninitialized_default_construct_n( + ExPolicy&& policy, InIter first, std::size_t count) { using value_type = typename std::iterator_traits::value_type; + util::cancellation_token tok; + return util::loop_with_cleanup_n_with_token( - first, count, tok, + HPX_FORWARD(ExPolicy, policy), first, count, tok, [](InIter it) -> void { ::new (std::addressof(*it)) value_type; }, @@ -242,7 +216,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::detail::algorithm_result_t - parallel_sequential_uninitialized_default_construct_n( + parallel_uninitialized_default_construct_n( ExPolicy&& policy, FwdIter first, std::size_t count) { if (count == 0) @@ -252,17 +226,17 @@ namespace hpx::parallel { } using partition_result_type = std::pair; - util::cancellation_token tok; + return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [tok](FwdIter it, std::size_t part_size) mutable + [tok, policy](FwdIter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_default_construct_n( - it, part_size, tok)); + HPX_FORWARD(ExPolicy, policy), it, part_size)); }, // finalize, called once if no error occurred [first, count](auto&& data) mutable -> FwdIter { @@ -295,16 +269,17 @@ namespace hpx::parallel { } template - static FwdIter sequential(ExPolicy, FwdIter first, Sent last) + static FwdIter sequential(ExPolicy policy, FwdIter first, Sent last) { - return std_uninitialized_default_construct(first, last); + return sequential_uninitialized_default_construct_n( + HPX_FORWARD(ExPolicy, policy), first, std::distance(first, last)); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_sequential_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } @@ -317,34 +292,6 @@ namespace hpx::parallel { namespace detail { /// \cond NOINTERNAL - // provide our own implementation of std::uninitialized_default_construct as some - // versions of MSVC horribly fail at compiling it for some types T - template - InIter std_uninitialized_default_construct_n( - InIter first, std::size_t count) - { - using value_type = - typename std::iterator_traits::value_type; - - InIter s_first = first; - try - { - for (/* */; count != 0; (void) ++first, --count) - { - ::new (std::addressof(*first)) value_type; - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - template struct uninitialized_default_construct_n : public algorithm, @@ -367,7 +314,7 @@ namespace hpx::parallel { static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, std::size_t count) { - return parallel_sequential_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, count); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index ecfcbaf6cfe6..7e4e2feb38d3 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -739,31 +739,34 @@ namespace hpx::parallel::util { struct loop_with_cleanup_n { /////////////////////////////////////////////////////////////////// - template - static FwdIter call( - FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + template )> + static FwdIter call(ExPolicy, Iter it, std::size_t num, + FwdIter dest, F&& f, Cleanup&& cleanup) { - FwdIter base = it; + FwdIter base = dest; try { std::size_t count(num & std::size_t(-4)); // -V112 for (std::size_t i = 0; i < count; - (void) ++it, i += 4) // -V112 + (void) ++it, ++dest, i += 4) // -V112 { - HPX_INVOKE(f, it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); + HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); } - for (/**/; count < num; (void) ++count, ++it) + for (/**/; count < num; (void) ++count, ++it, ++dest) { - HPX_INVOKE(f, it); + HPX_INVOKE(f, it, dest); } - return it; + return dest; } catch (...) { - for (/**/; base != it; ++base) + for (/**/; base != dest; ++base) { HPX_INVOKE(cleanup, base); } @@ -771,27 +774,33 @@ namespace hpx::parallel::util { } } - template - static FwdIter call(Iter it, std::size_t num, FwdIter dest, F&& f, - Cleanup&& cleanup) + template )> + static FwdIter call(ExPolicy, Iter it, std::size_t num, + FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; try { std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; (void) ++it, ++dest, i += 4) // -V112 { HPX_INVOKE(f, it, dest); - HPX_INVOKE(f, ++it, ++dest); - HPX_INVOKE(f, ++it, ++dest); - HPX_INVOKE(f, ++it, ++dest); } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; count < num; (void) ++count, ++it, ++dest) { HPX_INVOKE(f, it, dest); } + // clang-format on + return dest; } catch (...) @@ -804,53 +813,88 @@ namespace hpx::parallel::util { } } - template + template HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token( - FwdIter it, std::size_t num, CancelToken& tok, F&& f, - Cleanup&& cleanup) + ExPolicy&& policy, FwdIter it, std::size_t num, + CancelToken& tok, F&& f, Cleanup&& cleanup) { // check at the start of a partition only if (tok.was_cancelled()) return it; - return call( - it, num, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + return call(HPX_FORWARD(ExPolicy, policy), it, num, it, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - template - static FwdIter call_with_token(Iter it, std::size_t num, - FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) + template + static FwdIter call_with_token(ExPolicy&& policy, Iter it, + std::size_t num, FwdIter dest, CancelToken& tok, F&& f, + Cleanup&& cleanup) { // check at the start of a partition only if (tok.was_cancelled()) return dest; - return call(it, num, dest, HPX_FORWARD(F, f), - HPX_FORWARD(Cleanup, cleanup)); + return call(HPX_FORWARD(ExPolicy, policy), it, num, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } }; } // namespace detail - template + template )> HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( - it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); } template + typename Cleanup, + HPX_CONCEPT_REQUIRES_(hpx::traits::is_iterator_v&& + hpx::traits::is_iterator_v)> HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup_n_with_token(Iter it, std::size_t count, FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token(it, count, - dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + return detail::loop_with_cleanup_n::call_with_token( + ::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + + template && + hpx::traits::is_iterator_v)> + HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( + ExPolicy&& policy, Iter it, std::size_t count, CancelToken& tok, F&& f, + Cleanup&& cleanup) + { + using cat = typename std::iterator_traits::iterator_category; + return detail::loop_with_cleanup_n::call_with_token( + HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + + template && hpx::traits::is_iterator_v< + Iter>&& hpx::traits::is_iterator_v)> + HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup_n_with_token( + ExPolicy&& policy, Iter it, std::size_t count, FwdIter dest, + CancelToken& tok, F&& f, Cleanup&& cleanup) + { + using cat = typename std::iterator_traits::iterator_category; + return detail::loop_with_cleanup_n::call_with_token( + HPX_FORWARD(ExPolicy, policy), it, count, dest, tok, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } /////////////////////////////////////////////////////////////////////////// From 8bf4da797c1461b5c7773f8cadbcc6ce50834487 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 15:57:17 -0500 Subject: [PATCH 27/45] added another call overload to loop_with_cleanup_n Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 7e4e2feb38d3..916ae71d6d9e 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -789,7 +789,7 @@ namespace hpx::parallel::util { // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; - (void) ++it, ++dest, i += 4) // -V112 + (void) ++it, ++dest, ++i) // -V112 { HPX_INVOKE(f, it, dest); } @@ -813,6 +813,79 @@ namespace hpx::parallel::util { } } + template )> + static FwdIter call( + ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + { + FwdIter base = it; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + for (std::size_t i = 0; i < count; + (void) ++it, i += 4) // -V112 + { + HPX_INVOKE(f, it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + } + for (/**/; count < num; (void) ++count, ++it) + { + HPX_INVOKE(f, it); + } + return it; + } + catch (...) + { + for (/**/; base != it; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + template )> + static FwdIter call( + ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + { + FwdIter base = it; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < count; + (void) ++it, ++i) // -V112 + { + HPX_INVOKE(f, it); + } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it) + { + HPX_INVOKE(f, it); + } + // clang-format on + + return it; + } + catch (...) + { + for (/**/; base != it; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + template HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token( @@ -823,7 +896,7 @@ namespace hpx::parallel::util { if (tok.was_cancelled()) return it; - return call(HPX_FORWARD(ExPolicy, policy), it, num, it, + return call(HPX_FORWARD(ExPolicy, policy), it, num, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } From e246fe20ab1b0286a8acb31ff6f92b0bd262a9bb Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 16:11:51 -0500 Subject: [PATCH 28/45] clang-format Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_default_construct.hpp | 3 ++- .../algorithms/include/hpx/parallel/util/loop.hpp | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index c3989037a1cd..15565220891b 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -272,7 +272,8 @@ namespace hpx::parallel { static FwdIter sequential(ExPolicy policy, FwdIter first, Sent last) { return sequential_uninitialized_default_construct_n( - HPX_FORWARD(ExPolicy, policy), first, std::distance(first, last)); + HPX_FORWARD(ExPolicy, policy), first, + std::distance(first, last)); } template diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 916ae71d6d9e..659ed86ac170 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -813,12 +813,12 @@ namespace hpx::parallel::util { } } - template )> static FwdIter call( - ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { FwdIter base = it; try @@ -847,12 +847,12 @@ namespace hpx::parallel::util { throw; } } - template )> static FwdIter call( - ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { FwdIter base = it; try @@ -873,7 +873,7 @@ namespace hpx::parallel::util { HPX_INVOKE(f, it); } // clang-format on - + return it; } catch (...) From c04c157237fff7ac8e664e9d37a7de7046726b58 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 17:24:55 -0500 Subject: [PATCH 29/45] fixed concept Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 26 +++++++++++-------- .../include/hpx/parallel/util/loop.hpp | 12 ++++----- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 15565220891b..fe2f44b7f1ee 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -269,9 +269,10 @@ namespace hpx::parallel { } template - static FwdIter sequential(ExPolicy policy, FwdIter first, Sent last) + static FwdIter sequential( + ExPolicy&& policy, FwdIter first, Sent last) { - return sequential_uninitialized_default_construct_n( + sequential_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, std::distance(first, last)); } @@ -280,7 +281,7 @@ namespace hpx::parallel { static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_uninitialized_default_construct_n( + parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } @@ -306,9 +307,10 @@ namespace hpx::parallel { template static FwdIter sequential( - ExPolicy, FwdIter first, std::size_t count) + ExPolicy&& policy, FwdIter first, std::size_t count) { - return std_uninitialized_default_construct_n(first, count); + return sequential_uninitialized_default_construct_n( + HPX_FORWARD(ExPolicy, policy), first, count); } template @@ -331,13 +333,14 @@ namespace hpx { : hpx::detail::tag_parallel_algorithm { // clang-format off - template + hpx::traits::is_forward_iterator_v && + hpx::traits::is_iterator_v )> // clang-format on friend void tag_fallback_invoke( - hpx::uninitialized_default_construct_t, FwdIter first, FwdIter last) + hpx::uninitialized_default_construct_t, FwdIter first, Sent last) { static_assert(hpx::traits::is_forward_iterator_v, "Requires at least forward iterator."); @@ -347,15 +350,16 @@ namespace hpx { } // clang-format off - template && - hpx::traits::is_forward_iterator_v + hpx::traits::is_forward_iterator_v && + hpx::traits::is_iterator_v )> // clang-format on friend hpx::parallel::util::detail::algorithm_result_t tag_fallback_invoke(hpx::uninitialized_default_construct_t, - ExPolicy&& policy, FwdIter first, FwdIter last) + ExPolicy&& policy, FwdIter first, Sent last) { static_assert(hpx::traits::is_forward_iterator_v, "Requires at least forward iterator."); diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 659ed86ac170..c51218c29c2a 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -74,8 +74,8 @@ namespace hpx::parallel::util { template )> + HPX_CONCEPT_REQUIRES_( + !hpx::is_unsequenced_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) { @@ -660,8 +660,8 @@ namespace hpx::parallel::util { template )> + HPX_CONCEPT_REQUIRES_( + !hpx::is_unsequenced_execution_policy_v)> static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -742,7 +742,7 @@ namespace hpx::parallel::util { template )> + !hpx::is_unsequenced_execution_policy_v)> static FwdIter call(ExPolicy, Iter it, std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -816,7 +816,7 @@ namespace hpx::parallel::util { template )> + !hpx::is_unsequenced_execution_policy_v)> static FwdIter call( ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { From 8d0152485ee8ab7a1a021fbdcaa19ac45972e665 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Tue, 13 Jun 2023 16:21:26 -0500 Subject: [PATCH 30/45] fixed error from test which hass seninal not convertible to iterable type Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 66 +++++++++++++++++-- .../include/hpx/parallel/util/loop.hpp | 4 +- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index fe2f44b7f1ee..6529ba3efc08 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -166,6 +166,7 @@ namespace hpx { #include #include +#include #include #include #include @@ -194,6 +195,64 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// + template )> + InIter sequential_uninitialized_default_construct( + ExPolicy&& policy, InIter first, Sent last) + { + using value_type = typename std::iterator_traits::value_type; + + Iter s_first = first; + try + { + for (/* */; first != last; ++first) + { + ::new (std::addressof(*first)) value_type; + } + return first; + } + catch (...) + { + for (/* */; s_first != first; ++s_first) + { + std::destroy_at(std::addressof(*s_first)); + } + throw; + } + } + + template )> + InIter sequential_uninitialized_default_construct( + ExPolicy&& policy, InIter first, Sent last) + { + using value_type = typename std::iterator_traits::value_type; + + Iter s_first = first; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/* */; first != last; ++first) + { + ::new (std::addressof(*first)) value_type; + } + // clang-format on + + return first; + } + catch (...) + { + for (/* */; s_first != first; ++s_first) + { + std::destroy_at(std::addressof(*s_first)); + } + throw; + } + } + template InIter sequential_uninitialized_default_construct_n( ExPolicy&& policy, InIter first, std::size_t count) @@ -272,16 +331,15 @@ namespace hpx::parallel { static FwdIter sequential( ExPolicy&& policy, FwdIter first, Sent last) { - sequential_uninitialized_default_construct_n( - HPX_FORWARD(ExPolicy, policy), first, - std::distance(first, last)); + return sequential_uninitialized_default_construct( + HPX_FORWARD(ExPolicy, policy), first, last); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - parallel_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index c51218c29c2a..3449ac1554eb 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -74,7 +74,7 @@ namespace hpx::parallel::util { template )> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) @@ -660,7 +660,7 @@ namespace hpx::parallel::util { template )> static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) From 16d5f148728196ca5ac699cadcde3f8f9dc6c11b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Tue, 13 Jun 2023 16:27:26 -0500 Subject: [PATCH 31/45] unused variable fixed Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_default_construct.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 6529ba3efc08..69d07effc387 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -195,11 +195,11 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// - template )> - InIter sequential_uninitialized_default_construct( - ExPolicy&& policy, InIter first, Sent last) + Iter sequential_uninitialized_default_construct( + ExPolicy&&, Iter first, Sent last) { using value_type = typename std::iterator_traits::value_type; @@ -222,11 +222,11 @@ namespace hpx::parallel { } } - template )> - InIter sequential_uninitialized_default_construct( - ExPolicy&& policy, InIter first, Sent last) + Iter sequential_uninitialized_default_construct( + ExPolicy&&, Iter first, Sent last) { using value_type = typename std::iterator_traits::value_type; From d9772c632f0ff6e0f03412b73a3b73372dfa6ff6 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 14 Jun 2023 03:56:22 -0500 Subject: [PATCH 32/45] uninit move Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_move.hpp | 102 +++++------------ .../include/hpx/parallel/util/transfer.hpp | 106 ++++++++++++++++++ 2 files changed, 132 insertions(+), 76 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp index 5fc3c6407123..19d49746e327 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp @@ -208,6 +208,7 @@ namespace hpx { #include #include #include +#include #include #include @@ -224,33 +225,6 @@ namespace hpx::parallel { // uninitialized_move namespace detail { /// \cond NOINTERNAL - - /////////////////////////////////////////////////////////////////////// - template - util::in_out_result sequential_uninitialized_move( - InIter1 first, FwdIter2 dest, Cond cond) - { - FwdIter2 current = dest; - try - { - for (/* */; HPX_INVOKE(cond, first, current); - (void) ++first, ++current) - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - return util::in_out_result{first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - /////////////////////////////////////////////////////////////////////// template util::in_out_result sequential_uninitialized_move_n( @@ -273,7 +247,7 @@ namespace hpx::parallel { template typename util::detail::algorithm_result>::type - parallel_sequential_uninitialized_move_n( + parallel_uninitialized_move_n( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { if (count == 0) @@ -286,21 +260,21 @@ namespace hpx::parallel { using zip_iterator = hpx::util::zip_iterator; using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup, partition_result_type>:: call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [tok](zip_iterator t, std::size_t part_size) mutable + [&policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); FwdIter2 dst = get<1>(iters); return std::make_pair(dst, util::get_second_element( - sequential_uninitialized_move_n( - get<0>(iters), part_size, dst, tok))); + hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), + get<0>(iters), part_size, dst))); }, // finalize, called once if no error occurred [first, dest, count](auto&& data) mutable @@ -337,12 +311,11 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy&& policy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_move( - first, dest, [last](InIter1 first, FwdIter2) -> bool { - return first != last; - }); + return hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), first, + std::distance(first, last), dest); } template >::type parallel(ExPolicy&& policy, Iter first, Sent last, FwdIter2 dest) { - return parallel_sequential_uninitialized_move_n( + return parallel_uninitialized_move_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last), dest); } @@ -376,13 +349,16 @@ namespace hpx::parallel { template - static util::in_out_result sequential(ExPolicy, - InIter1 first, Sent1 last, FwdIter2 dest, Sent2 last_d) + static util::in_out_result sequential( + ExPolicy&& policy, InIter1 first, Sent1 last, FwdIter2 dest, + Sent2 last_d) { - return sequential_uninitialized_move(first, dest, - [last, last_d](InIter1 first, FwdIter2 current) -> bool { - return !(first == last || current == last_d); - }); + std::size_t const dist1 = detail::distance(first, last); + std::size_t const dist2 = detail::distance(dest, last_d); + std::size_t dist = dist1 <= dist2 ? dist1 : dist2; + + return hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), first, dist, dest); } template - util::in_out_result std_uninitialized_move_n( - InIter1 first, std::size_t count, InIter2 d_first) - { - InIter2 current = d_first; - try - { - for (/* */; count != 0; ++first, (void) ++current, --count) - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - return util::in_out_result{first, current}; - } - catch (...) - { - for (/* */; d_first != current; ++d_first) - { - std::destroy_at(std::addressof(*d_first)); - } - throw; - } - } - template struct uninitialized_move_n : public algorithm, IterPair> @@ -445,10 +394,11 @@ namespace hpx::parallel { } template - static IterPair sequential( - ExPolicy, InIter1 first, std::size_t count, InIter2 dest) + static IterPair sequential(ExPolicy&& policy, InIter1 first, + std::size_t count, InIter2 dest) { - return std_uninitialized_move_n(first, count, dest); + return hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), first, count, dest); } template @@ -457,7 +407,7 @@ namespace hpx::parallel { parallel( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { - return parallel_sequential_uninitialized_move_n( + return parallel_uninitialized_move_n( HPX_FORWARD(ExPolicy, policy), first, count, dest); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index c3ea0ea173bb..a6b0b978da83 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -44,6 +44,9 @@ namespace hpx::parallel::util { template struct uninit_copy_n_helper; + template + struct uninit_move_n_helper; + /////////////////////////////////////////////////////////////////////// template HPX_FORCEINLINE std::enable_if_t, char*> to_ptr( @@ -454,4 +457,107 @@ namespace hpx::parallel::util { return detail::uninit_copy_n_helper::call( HPX_FORWARD(ExPolicy, policy), first, count, dest); } + + // helpers for uninit_move_n + namespace detail { + // Customization point for optimizing move_n operations + template + struct uninit_move_n_helper + { + template )> + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + + try + { + for (std::size_t i = 0; i < num; + ++i, ++current, ++first) //-V112 + { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*first)); + } + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + + template )> + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < num; + ++i, ++current, ++first) //-V112 + { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*first)); + } + // clang-format on + + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + }; + + template + struct uninit_move_n_helper + { + template + HPX_FORCEINLINE static in_out_result call(ExPolicy, + InIter first, std::size_t count, OutIter dest) noexcept + { + return copy_memmove(first, count, dest); + } + }; + } // namespace detail + + template + HPX_FORCEINLINE constexpr in_out_result uninit_move_n( + InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_move_n_helper::call( + hpx::execution::seq, first, count, dest); + } + + template + HPX_FORCEINLINE constexpr in_out_result uninit_move_n( + ExPolicy&& policy, InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_move_n_helper::call( + HPX_FORWARD(ExPolicy, policy), first, count, dest); + } } // namespace hpx::parallel::util From a2c9ce156a2b90c3f3cf6c509cea4d5950ca6b46 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Thu, 15 Jun 2023 12:03:19 -0500 Subject: [PATCH 33/45] uninit value construct Signed-off-by: Hari Hara Naveen S --- .../uninitialized_value_construct.hpp | 90 +++++----------- .../include/hpx/parallel/util/loop.hpp | 101 ++++++++++++------ 2 files changed, 97 insertions(+), 94 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp index 59ad6801cdb8..d49f4579fcf1 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp @@ -193,36 +193,29 @@ namespace hpx::parallel { // provide our own implementation of std::uninitialized_value_construct // as some versions of MSVC horribly fail at compiling it for some types // T - template - InIter std_uninitialized_value_construct(InIter first, Sent last) + template + InIter sequential_uninitialized_value_construct( + ExPolicy&& policy, InIter first, Sent last) { - InIter s_first = first; - try - { - for (/* */; first != last; ++first) - { - hpx::construct_at(std::addressof(*first)); - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } + return util::loop_with_cleanup( + HPX_FORWARD(ExPolicy, policy), first, last, + [](InIter it) -> void { + hpx::construct_at(std::addressof(*it)); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + }); } /////////////////////////////////////////////////////////////////////// - template - InIter sequential_uninitialized_value_construct_n(InIter first, - std::size_t count, - util::cancellation_token& tok) + template + InIter sequential_uninitialized_value_construct_n( + ExPolicy&& policy, InIter first, std::size_t count) { + util::cancellation_token tok; + return util::loop_with_cleanup_n_with_token( - first, count, tok, + HPX_FORWARD(ExPolicy, policy), first, count, tok, [](InIter it) -> void { hpx::construct_at(std::addressof(*it)); }, @@ -234,7 +227,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::detail::algorithm_result_t - parallel_sequential_uninitialized_value_construct_n( + parallel_uninitialized_value_construct_n( ExPolicy&& policy, FwdIter first, std::size_t count) { if (count == 0) @@ -245,16 +238,15 @@ namespace hpx::parallel { using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [tok](FwdIter it, std::size_t part_size) mutable + [policy](FwdIter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_value_construct_n( - it, part_size, tok)); + HPX_FORWARD(ExPolicy, policy), it, part_size)); }, // finalize, called once if no error occurred [first, count](auto&& data) mutable -> FwdIter { @@ -288,16 +280,18 @@ namespace hpx::parallel { } template - static FwdIter sequential(ExPolicy, FwdIter first, Sent last) + static FwdIter sequential( + ExPolicy&& policy, FwdIter first, Sent last) { - return std_uninitialized_value_construct(first, last); + return sequential_uninitialized_value_construct( + HPX_FORWARD(ExPolicy, policy), first, last); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_sequential_uninitialized_value_construct_n( + return parallel_uninitialized_value_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } @@ -308,33 +302,6 @@ namespace hpx::parallel { // uninitialized_value_construct_n namespace detail { /// \cond NOINTERNAL - - // provide our own implementation of std::uninitialized_value_construct - // as some versions of MSVC horribly fail at compiling it for some types - // T - template - InIter std_uninitialized_value_construct_n( - InIter first, std::size_t count) - { - InIter s_first = first; - try - { - for (/* */; count != 0; (void) ++first, --count) - { - hpx::construct_at(std::addressof(*first)); - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - template struct uninitialized_value_construct_n : public algorithm, FwdIter> @@ -347,16 +314,17 @@ namespace hpx::parallel { template static FwdIter sequential( - ExPolicy, FwdIter first, std::size_t count) + ExPolicy&& policy, FwdIter first, std::size_t count) { - return std_uninitialized_value_construct_n(first, count); + return sequential_uninitialized_value_construct_n( + HPX_FORWARD(ExPolicy, policy), first, count); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, std::size_t count) { - return parallel_sequential_uninitialized_value_construct_n( + return parallel_uninitialized_value_construct_n( HPX_FORWARD(ExPolicy, policy), first, count); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 3449ac1554eb..d2908ffe79e7 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -616,24 +616,65 @@ namespace hpx::parallel::util { // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. If an exception is thrown, // the given cleanup function will be called. - template struct loop_with_cleanup { /////////////////////////////////////////////////////////////////// template - static FwdIter call(ExPolicy&& policy, FwdIter first, FwdIter last, - F&& f, Cleanup&& cleanup) + typename Sent, typename Cleanup, + HPX_CONCEPT_REQUIRES_( + hpx::is_unsequenced_execution_policy_v)> + static FwdIter call( + ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { - return call(HPX_FORWARD(ExPolicy, policy), first, last, first, - HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + FwdIter it = first; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != last; (void) ++it) + HPX_INVOKE(f, it); + // clang-format on + + return it; + } + catch (...) + { + for (/**/; first != it; ++first) + { + HPX_INVOKE(cleanup, first); + } + throw; + } + } + + template )> + static FwdIter call( + ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + { + FwdIter it = first; + try + { + for (/**/; it != last; (void) ++it) + HPX_INVOKE(f, it); + return it; + } + catch (...) + { + for (/**/; first != it; ++first) + { + HPX_INVOKE(cleanup, first); + } + throw; + } } template && - !hpx::is_parallel_execution_policy_v)> + HPX_CONCEPT_REQUIRES_( + hpx::is_unsequenced_execution_policy_v)> static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -684,32 +725,26 @@ namespace hpx::parallel::util { }; } // namespace detail - inline constexpr struct loop_with_cleanup_t final - : hpx::functional::detail::tag_fallback + template )> + HPX_FORCEINLINE constexpr Begin loop_with_cleanup(ExPolicy&& policy, + Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) { - private: - template - friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, - Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) - { - return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), first, last, dest, - HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } + return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), + first, last, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } - template - friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, - Begin first, End last, F&& f, Cleanup&& cleanup) - { - return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), first, last, HPX_FORWARD(F, f), - HPX_FORWARD(Cleanup, cleanup)); - } - } loop_with_cleanup{}; + template )> + HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + ExPolicy&& policy, Begin first, End last, F&& f, Cleanup&& cleanup) + { + return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), + first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } // /////////////////////////////////////////////////////////////////////////// // template From 4765db916e9fc4ea5b06bba981db24bad235d0b3 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Thu, 15 Jun 2023 12:20:53 -0500 Subject: [PATCH 34/45] uninit fill Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_fill.hpp | 90 +++++++------------ 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp index 4f1b0b8dd033..2ffbb717cfaf 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp @@ -196,35 +196,28 @@ namespace hpx::parallel { // provide our own implementation of std::uninitialized_fill as some // versions of MSVC horribly fail at compiling it for some types T - template - InIter std_uninitialized_fill(InIter first, Sent last, T const& value) + template + InIter sequential_uninitialized_fill( + ExPolicy&& policy, InIter first, Sent last, T const& value) { - InIter current = first; - try - { - for (/* */; current != last; ++current) - { - hpx::construct_at(std::addressof(*current), value); - } - return current; - } - catch (...) - { - for (/* */; first != current; ++first) - { - std::destroy_at(std::addressof(*first)); - } - throw; - } + return util::loop_with_cleanup( + HPX_FORWARD(ExPolicy, policy), first, last, + [&value](InIter it) -> void { + hpx::construct_at(std::addressof(*it), value); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + }); } - template - InIter sequential_uninitialized_fill_n(InIter first, std::size_t count, - T const& value, - util::cancellation_token& tok) + template + InIter sequential_uninitialized_fill_n( + ExPolicy&& policy, InIter first, std::size_t count, T const& value) { + util::cancellation_token tok; + return util::loop_with_cleanup_n_with_token( - first, count, tok, + HPX_FORWARD(ExPolicy, policy), first, count, tok, [&value](InIter it) -> void { hpx::construct_at(std::addressof(*it), value); }, @@ -236,7 +229,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::detail::algorithm_result_t - parallel_sequential_uninitialized_fill_n( + parallel_uninitialized_fill_n( ExPolicy&& policy, Iter first, std::size_t count, T const& value) { if (count == 0) @@ -252,11 +245,12 @@ namespace hpx::parallel { partition_result_type>:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [value, tok](Iter it, std::size_t part_size) mutable + [value, tok, policy](Iter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_fill_n( - it, part_size, value, tok)); + HPX_FORWARD(ExPolicy, policy), it, part_size, + value)); }, // finalize, called once if no error occurred [first, count](auto&& data) mutable -> Iter { @@ -290,9 +284,10 @@ namespace hpx::parallel { template static Iter sequential( - ExPolicy, Iter first, Sent last, T const& value) + ExPolicy&& policy, Iter first, Sent last, T const& value) { - return std_uninitialized_fill(first, last, value); + return sequential_uninitialized_fill( + HPX_FORWARD(ExPolicy, policy), first, last, value); } template @@ -303,7 +298,7 @@ namespace hpx::parallel { return util::detail::algorithm_result::get( HPX_MOVE(first)); - return parallel_sequential_uninitialized_fill_n( + return parallel_uninitialized_fill_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last), value); } @@ -315,32 +310,6 @@ namespace hpx::parallel { // uninitialized_fill_n namespace detail { /// \cond NOINTERNAL - - // provide our own implementation of std::uninitialized_fill_n as some - // versions of MSVC horribly fail at compiling it for some types T - template - InIter std_uninitialized_fill_n( - InIter first, Size count, T const& value) - { - InIter current = first; - try - { - for (/* */; count > 0; ++current, (void) --count) - { - hpx::construct_at(std::addressof(*current), value); - } - return current; - } - catch (...) - { - for (/* */; first != current; ++first) - { - std::destroy_at(std::addressof(*first)); - } - throw; - } - } - template struct uninitialized_fill_n : public algorithm, Iter> @@ -351,10 +320,11 @@ namespace hpx::parallel { } template - static Iter sequential( - ExPolicy, Iter first, std::size_t count, T const& value) + static Iter sequential(ExPolicy&& policy, Iter first, + std::size_t count, T const& value) { - return std_uninitialized_fill_n(first, count, value); + return sequential_uninitialized_fill_n( + HPX_FORWARD(ExPolicy, policy), first, count, value); } template @@ -362,7 +332,7 @@ namespace hpx::parallel { ExPolicy&& policy, Iter first, std::size_t count, T const& value) { - return parallel_sequential_uninitialized_fill_n( + return parallel_uninitialized_fill_n( HPX_FORWARD(ExPolicy, policy), first, count, value); } }; From 29e90facaf19dc8d262e4846afb619370ba3cc2e Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Thu, 15 Jun 2023 17:43:08 -0500 Subject: [PATCH 35/45] removed commented out code Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index d2908ffe79e7..338bf8bc9572 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -746,26 +746,6 @@ namespace hpx::parallel::util { first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - // /////////////////////////////////////////////////////////////////////////// - // template - // HPX_FORCEINLINE constexpr Iter loop_with_cleanup( - // Iter it, Iter last, F&& f, Cleanup&& cleanup) - // { - // using cat = typename std::iterator_traits::iterator_category; - // return detail::loop_with_cleanup::call( - // it, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - // } - - // template - // HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup( - // Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) - // { - // using cat = typename std::iterator_traits::iterator_category; - // return detail::loop_with_cleanup::call( - // it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - // } - - /////////////////////////////////////////////////////////////////////////// namespace detail { // Helper class to repeatedly call a function a given number of times From e87991ee9e233af41a522e9f47abd9e83d4a91f9 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 10:59:38 -0500 Subject: [PATCH 36/45] reeplaced try catch loops with loop_with_cleanup Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 63 ++------ .../include/hpx/parallel/unseq/loop.hpp | 14 +- .../include/hpx/parallel/util/transfer.hpp | 136 ++++-------------- 3 files changed, 45 insertions(+), 168 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 69d07effc387..757b5ccda4ec 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -195,62 +195,21 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// - template )> - Iter sequential_uninitialized_default_construct( - ExPolicy&&, Iter first, Sent last) - { - using value_type = typename std::iterator_traits::value_type; - - Iter s_first = first; - try - { - for (/* */; first != last; ++first) - { - ::new (std::addressof(*first)) value_type; - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - - template )> + template Iter sequential_uninitialized_default_construct( ExPolicy&&, Iter first, Sent last) { - using value_type = typename std::iterator_traits::value_type; - - Iter s_first = first; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/* */; first != last; ++first) - { - ::new (std::addressof(*first)) value_type; - } - // clang-format on + using value_type = + typename std::iterator_traits::value_type; - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } + return util::loop_with_cleanup( + HPX_FORWARD(ExPolicy, policy), first, last, + [](InIter it) -> void { + ::new (std::addressof(*it)) value_type; + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + }); } template diff --git a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp index 131f62492014..3b7e5d3c7526 100644 --- a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp @@ -254,11 +254,10 @@ namespace hpx::parallel::util { { std::size_t count(num & std::size_t(-4)); // -V112 - //clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; - i < count; - (void) ++it, ++dest, - i += 4) // -V112 + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < count; (void) ++it, ++dest, + i += 4) // -V112 { HPX_INVOKE(f, it, dest); HPX_INVOKE(f, ++it, ++dest); @@ -266,9 +265,8 @@ namespace hpx::parallel::util { HPX_INVOKE(f, ++it, ++dest); } - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; count < num; - (void) ++count, - ++it, ++dest) + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it, ++dest) { HPX_INVOKE(f, it, dest); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index a6b0b978da83..97c217295d08 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -364,62 +364,23 @@ namespace hpx::parallel::util { template struct uninit_copy_n_helper { - template )> - HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) - { - OutIter current = dest; - - try - { - for (std::size_t i = 0; i < num; - ++i, ++current, ++first) //-V112 - { - hpx::construct_at(std::addressof(*current), *first); - } - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template )> + template HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) + ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (std::size_t i = 0; i < num; ++i, ++current, ++first) //-V112 - { - hpx::construct_at(std::addressof(*current), *first); - } - // clang-format on - - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } + using value_type = + typename std::iterator_traits::value_type; + + return in_out_result{std::next(first, num), + util::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it) -> void { + hpx::construct_at(std::addressof(*current), *first); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; @@ -464,65 +425,24 @@ namespace hpx::parallel::util { template struct uninit_move_n_helper { - template )> - HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) - { - OutIter current = dest; - - try - { - for (std::size_t i = 0; i < num; - ++i, ++current, ++first) //-V112 - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template )> + template HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (std::size_t i = 0; i < num; - ++i, ++current, ++first) //-V112 - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - // clang-format on - - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } + using value_type = + typename std::iterator_traits::value_type; + + return in_out_result{std::next(first, num), + util::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it) -> void { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*first)); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; From c1dc43facc70776c8709e1cf5555b67f2005c1d7 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 11:16:22 -0500 Subject: [PATCH 37/45] changed variable name Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 97c217295d08..af82bf1b30e6 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -373,7 +373,7 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - util::loop_with_cleanup_n::call( + loop_with_cleanup_n::call( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at(std::addressof(*current), *first); @@ -427,7 +427,7 @@ namespace hpx::parallel::util { { template HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) + ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; using value_type = @@ -438,7 +438,7 @@ namespace hpx::parallel::util { HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); + std::addressof(*current), HPX_MOVE(*it)); }, [](InIter it) -> void { std::destroy_at(std::addressof(*it)); From aed982517c529537b4ceaedc6e92eee536046d88 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 12:07:58 -0500 Subject: [PATCH 38/45] namespace errors Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_default_construct.hpp | 11 +++-------- .../algorithms/include/hpx/parallel/util/transfer.hpp | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 757b5ccda4ec..f31df1a42e81 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -199,17 +199,12 @@ namespace hpx::parallel { Iter sequential_uninitialized_default_construct( ExPolicy&&, Iter first, Sent last) { - using value_type = - typename std::iterator_traits::value_type; + using value_type = typename std::iterator_traits::value_type; return util::loop_with_cleanup( HPX_FORWARD(ExPolicy, policy), first, last, - [](InIter it) -> void { - ::new (std::addressof(*it)) value_type; - }, - [](InIter it) -> void { - std::destroy_at(std::addressof(*it)); - }); + [](Iter it) -> void { ::new (std::addressof(*it)) value_type; }, + [](Iter it) -> void { std::destroy_at(std::addressof(*it)); }); } template diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index af82bf1b30e6..eef8bf198b6d 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -373,7 +373,7 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - loop_with_cleanup_n::call( + hpx::parallel::util::loop_with_cleanup_n::call( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at(std::addressof(*current), *first); @@ -434,7 +434,7 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - util::loop_with_cleanup_n::call( + hpx::parallel::util::loop_with_cleanup_n::call( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at( From bfc74d01a3f5527e0439deca209b9b5cba560b8c Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 02:01:10 -0500 Subject: [PATCH 39/45] fixed template errors Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 2 +- .../include/hpx/parallel/util/loop.hpp | 1 - .../include/hpx/parallel/util/transfer.hpp | 38 ++++++++++--------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index f31df1a42e81..1a852004bd03 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -197,7 +197,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template Iter sequential_uninitialized_default_construct( - ExPolicy&&, Iter first, Sent last) + ExPolicy&& policy, Iter first, Sent last) { using value_type = typename std::iterator_traits::value_type; diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 338bf8bc9572..f4167c3f28c7 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -747,7 +747,6 @@ namespace hpx::parallel::util { } namespace detail { - // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. template diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index eef8bf198b6d..9c02be34ae36 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -373,14 +374,16 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - hpx::parallel::util::loop_with_cleanup_n::call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it) -> void { - hpx::construct_at(std::addressof(*current), *first); - }, - [](InIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n:: + call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at( + std::addressof(*current), *it); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; @@ -434,15 +437,16 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - hpx::parallel::util::loop_with_cleanup_n::call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it) -> void { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*it)); - }, - [](InIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n:: + call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*it)); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; From ef0476dd7704138071ebc8e62bcf76734547d496 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 02:13:57 -0500 Subject: [PATCH 40/45] cf Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 9c02be34ae36..200ef5b2e3f2 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -362,7 +362,7 @@ namespace hpx::parallel::util { // helpers for uninit_copy_n namespace detail { // Customization point for optimizing copy_n operations - template + template struct uninit_copy_n_helper { template From 7202338f44b97d5c70380a618536f7a65cd78dc4 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 07:04:16 -0500 Subject: [PATCH 41/45] removed unused vars Signed-off-by: Hari Hara Naveen S --- .../algorithms/include/hpx/parallel/util/transfer.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 200ef5b2e3f2..de65cef2247a 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -369,10 +369,6 @@ namespace hpx::parallel::util { HPX_FORCEINLINE static in_out_result call( ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { - OutIter current = dest; - using value_type = - typename std::iterator_traits::value_type; - return in_out_result{std::next(first, num), ::hpx::parallel::util::detail::loop_with_cleanup_n:: call( @@ -432,10 +428,6 @@ namespace hpx::parallel::util { HPX_FORCEINLINE static in_out_result call( ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { - OutIter current = dest; - using value_type = - typename std::iterator_traits::value_type; - return in_out_result{std::next(first, num), ::hpx::parallel::util::detail::loop_with_cleanup_n:: call( From 15e89648b5f41002ea4ff41a9b4b42e9c2b700e5 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 24 Jul 2023 04:56:13 -0500 Subject: [PATCH 42/45] requested changes Signed-off-by: Hari Hara Naveen S --- .../parallel/algorithms/uninitialized_default_construct.hpp | 3 +-- .../include/hpx/parallel/algorithms/uninitialized_move.hpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 1a852004bd03..ceab0f25b5b2 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -239,13 +239,12 @@ namespace hpx::parallel { } using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [tok, policy](FwdIter it, std::size_t part_size) mutable + [policy](FwdIter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_default_construct_n( diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp index 19d49746e327..1422be66b4b8 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp @@ -265,7 +265,7 @@ namespace hpx::parallel { call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [&policy](zip_iterator t, std::size_t part_size) mutable + [policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); From bb560af53eca65305627372f0b87e6b471a07c1a Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 28 Jul 2023 15:28:03 -0500 Subject: [PATCH 43/45] unseq dispatch psuhed to tag_invoke stage Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 157 ++++++++++++------ .../include/hpx/parallel/util/transfer.hpp | 37 ++--- 2 files changed, 125 insertions(+), 69 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 04430e07f587..8467855f761f 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -747,16 +747,15 @@ namespace hpx::parallel::util { } namespace detail { + // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. - template - struct loop_with_cleanup_n + struct loop_with_cleanup_n final + : hpx::functional::detail::tag_fallback { /////////////////////////////////////////////////////////////////// template )> + typename F, typename Cleanup> static FwdIter call(ExPolicy, Iter it, std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -788,38 +787,32 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call(ExPolicy, Iter it, std::size_t num, - FwdIter dest, F&& f, Cleanup&& cleanup) + template + static FwdIter call( + ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { - FwdIter base = dest; + FwdIter base = it; try { std::size_t count(num & std::size_t(-4)); // -V112 - - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; - (void) ++it, ++dest, ++i) // -V112 + (void) ++it, i += 4) // -V112 { - HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); } - - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/**/; count < num; (void) ++count, ++it, ++dest) + for (/**/; count < num; (void) ++count, ++it) { - HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, it); } - // clang-format on - - return dest; + return it; } catch (...) { - for (/**/; base != dest; ++base) + for (/**/; base != it; ++base) { HPX_INVOKE(cleanup, base); } @@ -827,45 +820,46 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call( - ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + template + FwdIter unseq_call(Iter it, std::size_t num, FwdIter dest, F&& f, + Cleanup&& cleanup) { - FwdIter base = it; + FwdIter base = dest; try { std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; - (void) ++it, i += 4) // -V112 + (void) ++it, ++dest, ++i) // -V112 { - HPX_INVOKE(f, it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); + HPX_INVOKE(f, it, dest); } - for (/**/; count < num; (void) ++count, ++it) + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it, ++dest) { - HPX_INVOKE(f, it); + HPX_INVOKE(f, it, dest); } - return it; + // clang-format on + + return dest; } catch (...) { - for (/**/; base != it; ++base) + for (/**/; base != dest; ++base) { HPX_INVOKE(cleanup, base); } throw; } } + template )> - static FwdIter call( + typename Cleanup> + static FwdIter unseq_call( ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { FwdIter base = it; @@ -930,13 +924,78 @@ namespace hpx::parallel::util { }; } // namespace detail +#if !defined(HPX_COMPUTE_DEVICE_CODE) + inline constexpr detail::loop_with_cleanup_n loop_with_cleanup_n = + detail::loop_with_cleanup_n{}; +#else + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n( + ExPolicy&& policy, Begin begin, std::size_t n, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n{}( + HPX_FORWARD(ExPolicy, policy), begin, n, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n( + ExPolicy&& policy, Begin begin, std::size_t n, Begin2 dest, F&& f, + Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n{}( + HPX_FORWARD(ExPolicy, policy), begin, n, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } +#endif + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, + std::size_t n, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + std::size_t n, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, + std::size_t n, Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + std::size_t n, Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + template )> HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } @@ -950,7 +1009,7 @@ namespace hpx::parallel::util { Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } @@ -964,7 +1023,7 @@ namespace hpx::parallel::util { Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } @@ -979,7 +1038,7 @@ namespace hpx::parallel::util { CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index de65cef2247a..a8e74c536527 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -370,16 +370,14 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n:: - call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it, OutIter current) -> void { - hpx::construct_at( - std::addressof(*current), *it); - }, - [](OutIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at(std::addressof(*current), *it); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; @@ -429,16 +427,15 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n:: - call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it, OutIter current) -> void { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*it)); - }, - [](OutIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*it)); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; From f573137ab4096ca2d7b8876cb8a7fe94815739da Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 2 Aug 2023 16:13:56 -0500 Subject: [PATCH 44/45] loop with cleanup moved to tag fallback invoke Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 126 ++++++++++++------ 1 file changed, 82 insertions(+), 44 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 8467855f761f..23fbbbef4f3c 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -616,14 +616,13 @@ namespace hpx::parallel::util { // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. If an exception is thrown, // the given cleanup function will be called. - struct loop_with_cleanup + struct loop_with_cleanup final + : hpx::functional::detail::tag_fallback { /////////////////////////////////////////////////////////////////// template )> - static FwdIter call( + typename Sent, typename Cleanup> + static FwdIter unseq_call( ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { FwdIter it = first; @@ -648,11 +647,10 @@ namespace hpx::parallel::util { } template )> - static FwdIter call( - ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + typename Sent, typename Cleanup> + friend FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, + FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { FwdIter it = first; try @@ -671,22 +669,17 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, - F&& f, Cleanup&& cleanup) + template + friend FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, + Iter it, Sent last, FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; try { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != last; (void) ++it, ++dest) f(it, dest); - // clang-format on - return dest; } catch (...) @@ -699,18 +692,20 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, - F&& f, Cleanup&& cleanup) + template + static FwdIter unseq_call(ExPolicy&&, Iter it, Sent last, + FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; try { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != last; (void) ++it, ++dest) f(it, dest); + // clang-format on + return dest; } catch (...) @@ -725,25 +720,72 @@ namespace hpx::parallel::util { }; } // namespace detail - template )> - HPX_FORCEINLINE constexpr Begin loop_with_cleanup(ExPolicy&& policy, - Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) +#if !defined(HPX_COMPUTE_DEVICE_CODE) + inline constexpr detail::loop_with_cleanup loop_with_cleanup = + detail::loop_with_cleanup{}; +#else + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + ExPolicy&& policy, Begin begin, Sent end, F&& f, Cleanup&& cleanup) { - return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), - first, last, dest, HPX_FORWARD(F, f), + return hpx::parallel::util::detail::loop_with_cleanup{}( + HPX_FORWARD(ExPolicy, policy), begin, end, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - template )> - HPX_FORCEINLINE constexpr Begin loop_with_cleanup( - ExPolicy&& policy, Begin first, End last, F&& f, Cleanup&& cleanup) + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + ExPolicy&& policy, Begin begin, Sent end, Begin2 dest, F&& f, + Cleanup&& cleanup) { - return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), - first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + return hpx::parallel::util::detail::loop_with_cleanup{}( + HPX_FORWARD(ExPolicy, policy), begin, end, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } +#endif + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, Sent end, + F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + Sent end, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, Sent end, + Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + Sent end, Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } namespace detail { @@ -994,7 +1036,6 @@ namespace hpx::parallel::util { HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); @@ -1008,7 +1049,6 @@ namespace hpx::parallel::util { std::size_t count, FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); @@ -1022,7 +1062,6 @@ namespace hpx::parallel::util { ExPolicy&& policy, Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); @@ -1037,7 +1076,6 @@ namespace hpx::parallel::util { ExPolicy&& policy, Iter it, std::size_t count, FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); From e2277ce8eef99711d353c5c0d0a8db441488a36f Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 5 Aug 2023 05:35:21 -0500 Subject: [PATCH 45/45] removed tokens Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 1 - .../uninitialized_default_construct.hpp | 7 +- .../algorithms/uninitialized_fill.hpp | 10 +- .../algorithms/uninitialized_move.hpp | 8 +- .../uninitialized_value_construct.hpp | 7 +- .../include/hpx/parallel/util/loop.hpp | 101 ++++++++++++------ .../include/hpx/parallel/util/transfer.hpp | 4 +- 7 files changed, 82 insertions(+), 56 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 7fd8495aa91f..8fb58d086586 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -199,7 +199,6 @@ namespace hpx { #include #include #include -#include #include #include #include diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index ceab0f25b5b2..47d3967b9dde 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -171,7 +171,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -214,10 +213,8 @@ namespace hpx::parallel { using value_type = typename std::iterator_traits::value_type; - util::cancellation_token tok; - - return util::loop_with_cleanup_n_with_token( - HPX_FORWARD(ExPolicy, policy), first, count, tok, + return util::loop_with_cleanup_n( + HPX_FORWARD(ExPolicy, policy), first, count, [](InIter it) -> void { ::new (std::addressof(*it)) value_type; }, diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp index 2ffbb717cfaf..0587f4962566 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp @@ -170,7 +170,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -214,10 +213,8 @@ namespace hpx::parallel { InIter sequential_uninitialized_fill_n( ExPolicy&& policy, InIter first, std::size_t count, T const& value) { - util::cancellation_token tok; - - return util::loop_with_cleanup_n_with_token( - HPX_FORWARD(ExPolicy, policy), first, count, tok, + return util::loop_with_cleanup_n( + HPX_FORWARD(ExPolicy, policy), first, count, [&value](InIter it) -> void { hpx::construct_at(std::addressof(*it), value); }, @@ -240,12 +237,11 @@ namespace hpx::parallel { using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [value, tok, policy](Iter it, std::size_t part_size) mutable + [value, policy](Iter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_fill_n( diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp index 1422be66b4b8..2749c5773852 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp @@ -201,7 +201,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -228,13 +227,12 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::in_out_result sequential_uninitialized_move_n( - InIter1 first, std::size_t count, InIter2 dest, - util::cancellation_token& tok) + InIter1 first, std::size_t count, InIter2 dest) { return util::in_out_result{ std::next(first, count), - util::loop_with_cleanup_n_with_token( - first, count, dest, tok, + util::loop_with_cleanup_n( + first, count, dest, [](InIter1 it, InIter2 dest) -> void { hpx::construct_at(std::addressof(*dest), HPX_MOVE(*it)); }, diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp index d49f4579fcf1..790709d40baa 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp @@ -167,7 +167,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -212,10 +211,8 @@ namespace hpx::parallel { InIter sequential_uninitialized_value_construct_n( ExPolicy&& policy, InIter first, std::size_t count) { - util::cancellation_token tok; - - return util::loop_with_cleanup_n_with_token( - HPX_FORWARD(ExPolicy, policy), first, count, tok, + return util::loop_with_cleanup_n( + HPX_FORWARD(ExPolicy, policy), first, count, [](InIter it) -> void { hpx::construct_at(std::addressof(*it)); }, diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 23fbbbef4f3c..dfa67230e5d6 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -622,18 +622,14 @@ namespace hpx::parallel::util { /////////////////////////////////////////////////////////////////// template - static FwdIter unseq_call( + static FwdIter call( ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { FwdIter it = first; try { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != last; (void) ++it) HPX_INVOKE(f, it); - // clang-format on - return it; } catch (...) @@ -645,48 +641,71 @@ namespace hpx::parallel::util { throw; } } - - template - friend FwdIter tag_fallback_invoke( - hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, - FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + template + static FwdIter call(ExPolicy&&, Iter it, Sent last, FwdIter dest, + F&& f, Cleanup&& cleanup) { - FwdIter it = first; + FwdIter base = dest; try { - for (/**/; it != last; (void) ++it) - HPX_INVOKE(f, it); - return it; + for (/**/; it != last; (void) ++it, ++dest) + f(it, dest); + return dest; } catch (...) { - for (/**/; first != it; ++first) + for (/**/; base != dest; ++base) { - HPX_INVOKE(cleanup, first); + HPX_INVOKE(cleanup, base); } throw; } } + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + ExPolicy&& policy, FwdIter first, Sent last, F&& f, + Cleanup&& cleanup) + { + return call(HPX_FORWARD(ExPolicy, policy), first, last, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + template - friend FwdIter tag_fallback_invoke( - hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, - Iter it, Sent last, FwdIter dest, F&& f, Cleanup&& cleanup) + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + ExPolicy&& policy, Iter it, Sent last, FwdIter dest, F&& f, + Cleanup&& cleanup) { - FwdIter base = dest; + return call(HPX_FORWARD(ExPolicy, policy), it, last, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + static FwdIter unseq_call( + ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + { + FwdIter it = first; try { - for (/**/; it != last; (void) ++it, ++dest) - f(it, dest); - return dest; + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != last; (void) ++it) + HPX_INVOKE(f, it); + // clang-format on + + return it; } catch (...) { - for (/**/; base != dest; ++base) + for (/**/; first != it; ++first) { - HPX_INVOKE(cleanup, base); + HPX_INVOKE(cleanup, first); } throw; } @@ -798,7 +817,7 @@ namespace hpx::parallel::util { /////////////////////////////////////////////////////////////////// template - static FwdIter call(ExPolicy, Iter it, std::size_t num, + static FwdIter call(ExPolicy&&, Iter it, std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; @@ -831,8 +850,8 @@ namespace hpx::parallel::util { template - static FwdIter call( - ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + static FwdIter call(ExPolicy&&, FwdIter it, std::size_t num, F&& f, + Cleanup&& cleanup) { FwdIter base = it; try @@ -862,6 +881,26 @@ namespace hpx::parallel::util { } } + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + loop_with_cleanup_n, ExPolicy&& policy, Iter it, + std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) + { + return call(HPX_FORWARD(ExPolicy, policy), it, num, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + loop_with_cleanup_n, ExPolicy&& policy, FwdIter it, + std::size_t num, F&& f, Cleanup&& cleanup) + { + return call(HPX_FORWARD(ExPolicy, policy), it, num, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + template FwdIter unseq_call(Iter it, std::size_t num, FwdIter dest, F&& f, @@ -1092,7 +1131,7 @@ namespace hpx::parallel::util { /////////////////////////////////////////////////////////////////// // handle sequences of non-futures template - HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Iter call( + HPX_HOST_DEVICE HPX_FORCEINLINE static Iter call( std::size_t base_idx, Iter it, std::size_t num, F&& f) { std::size_t count(num & std::size_t(-4)); // -V112 @@ -1113,7 +1152,7 @@ namespace hpx::parallel::util { } template - HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Iter call( + HPX_HOST_DEVICE HPX_FORCEINLINE static Iter call( std::size_t base_idx, Iter it, std::size_t count, CancelToken& tok, F&& f) { diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index a8e74c536527..69f2556ebb26 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -370,7 +370,7 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + ::hpx::parallel::util::loop_with_cleanup_n( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it, OutIter current) -> void { hpx::construct_at(std::addressof(*current), *it); @@ -427,7 +427,7 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + ::hpx::parallel::util::loop_with_cleanup_n( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it, OutIter current) -> void { hpx::construct_at(