Adding local work stealing scheduler that is based on message passing…

… internally

- Using uniform_int_distribution with proper bounds
- Removing queue index from thread_queues as it was unused
  - flyby: remove commented out options from .clang-format
- Renaming workstealing --> workrequesting
- Adding adaptive work stealing (steal half/steal one)
  - this makes this scheduler consistently (albeit only slightly) faster than
    the (default) local-priority scheduler
- Adding LIFO and FIFO variations of local work-stealing scheduler
  - flyby: fixing HPX_WITH_SWAP_CONTEXT_EMULATION
  - flyby: minor changes to fibonacci_local example
- Adding high- and low- priority queues
  - flyby: cache_line_data now does not generate warnings errors if padding is not needed

.circleci/config.yml

@@ -233,7 +233,8 @@ jobs:
                 -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On \
                 -DCMAKE_EXPORT_COMPILE_COMMANDS=On \
                 -DHPX_WITH_DOCUMENTATION=On \
-                -DHPX_WITH_DOCUMENTATION_OUTPUT_FORMATS="${DOCUMENTATION_OUTPUT_FORMATS}"
+                -DHPX_WITH_DOCUMENTATION_OUTPUT_FORMATS="${DOCUMENTATION_OUTPUT_FORMATS}" \
+                -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo
       - persist_to_workspace:
           root: /hpx
           paths:

.clang-format

@@ -1,5 +1,5 @@
 # Copyright (c) 2016 Thomas Heller
-# Copyright (c) 2016-2018 Hartmut Kaiser
+# Copyright (c) 2016-2019 Hartmut Kaiser
 #
 # SPDX-License-Identifier: BSL-1.0
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -105,7 +105,7 @@ PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
 ReflowComments: false
-SortIncludes:    true
+SortIncludes: true
 SpaceAfterCStyleCast: true
 SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true

.jenkins/cscs-perftests/launch_perftests.sh

@@ -15,13 +15,13 @@ hpx_targets=(
 hpx_test_options=(
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --hpx:threads=4 --vector_size=10000 --work_delay=1 \
-    --chunk_size=0 --test_count=5000"
+    --chunk_size=0 --test_count=5000 --hpx:queuing=local-workrequesting-fifo"
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --hpx:queuing=local-priority --hpx:threads=4 --test-all \
-    --repetitions=100 --futures=500000"
+    --repetitions=100 --futures=500000 --hpx:queuing=local-workrequesting-fifo"
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --vector_size=1048576 --hpx:threads=4 --iterations=5000 \
-    --warmup_iterations=500")
+    --warmup_iterations=500 --hpx:queuing=local-workrequesting-fifo")
 
 # Build binaries for performance tests
 ${perftests_dir}/driver.py -v -l $logfile build -b release -o build \

.jenkins/lsu/env-clang-13.sh

@@ -23,8 +23,9 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
-configure_extra_options+=" -DLCI_SERVER=ibv"
 configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"
+configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo"
+configure_extra_options+=" -DLCI_SERVER=ibv"
 
 # Make sure HWLOC does not report 'cores'. This is purely an option to enable
 # testing the topology code under conditions close to those on FreeBSD.

.jenkins/lsu/env-gcc-11.sh

@@ -14,6 +14,7 @@ module load openmpi
 export HPXRUN_RUNWRAPPER=srun
 export CXX_STD="20"
 
+configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}"
 configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}"
 configure_extra_options+=" -DHPX_WITH_MALLOC=system"
 configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON"
@@ -22,4 +23,5 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON"
 configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON"
 configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
+configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo"
 configure_extra_options+=" -DLCI_SERVER=ibv"

CMakeLists.txt

@@ -1388,6 +1388,13 @@ hpx_option(
   ADVANCED
 )
 
+hpx_option(
+  HPX_WITH_TESTS_COMMAND_LINE STRING
+  "Add given command line options to all tests run" ""
+  CATEGORY "Debugging"
+  ADVANCED
+)
+
 hpx_option(
   HPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY
   STRING

cmake/HPX_AddTest.cmake

@@ -66,6 +66,11 @@ function(add_hpx_test category name)
     )
   endif()
 
+  # add additional command line arguments to all test executions
+  if(NOT x${HPX_WITH_TESTS_COMMAND_LINE} STREQUAL x"")
+    set(args ${args} "${HPX_WITH_TESTS_COMMAND_LINE}")
+  endif()
+
   if(${HPX_WITH_PARALLEL_TESTS_BIND_NONE}
      AND NOT run_serial
      AND NOT "${name}_RUNWRAPPER"

examples/quickstart/fibonacci_local.cpp

@@ -18,6 +18,9 @@
 
 #include <cstdint>
 #include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
 
 ///////////////////////////////////////////////////////////////////////////////
 //[fibonacci
@@ -26,22 +29,20 @@ std::uint64_t fibonacci(std::uint64_t n)
     if (n < 2)
         return n;
 
-    // Invoking the Fibonacci algorithm twice is inefficient.
-    // However, we intentionally demonstrate it this way to create some
-    // heavy workload.
-
     hpx::future<std::uint64_t> n1 = hpx::async(fibonacci, n - 1);
-    hpx::future<std::uint64_t> n2 = hpx::async(fibonacci, n - 2);
+    std::uint64_t n2 = fibonacci(n - 2);
 
-    return n1.get() +
-        n2.get();    // wait for the Futures to return their values
+    return n1.get() + n2;    // wait for the Future to return their values
 }
 //fibonacci]
 
 ///////////////////////////////////////////////////////////////////////////////
 //[hpx_main
 int hpx_main(hpx::program_options::variables_map& vm)
 {
+    hpx::threads::add_scheduler_mode(
+        hpx::threads::policies::scheduler_mode::fast_idle_mode);
+
     // extract command line argument, i.e. fib(N)
     std::uint64_t n = vm["n-value"].as<std::uint64_t>();
 
@@ -67,9 +68,13 @@ int main(int argc, char* argv[])
     hpx::program_options::options_description desc_commandline(
         "Usage: " HPX_APPLICATION_STRING " [options]");
 
-    desc_commandline.add_options()("n-value",
-        hpx::program_options::value<std::uint64_t>()->default_value(10),
-        "n value for the Fibonacci function");
+    // clang-format off
+    desc_commandline.add_options()
+        ("n-value",
+            hpx::program_options::value<std::uint64_t>()->default_value(10),
+            "n value for the Fibonacci function")
+        ;
+    // clang-format on
 
     // Initialize and run HPX
     hpx::local::init_params init_args;

libs/core/command_line_handling_local/src/parse_command_line_local.cpp

@@ -363,6 +363,7 @@ namespace hpx { namespace local { namespace detail {
                 ("hpx:queuing", value<std::string>(),
                   "the queue scheduling policy to use, options are "
                   "'local', 'local-priority-fifo','local-priority-lifo', "
+                  "'local-workrequesting-fifo', 'local-workrequesting-lifo' "
                   "'abp-priority-fifo', 'abp-priority-lifo', 'static', and "
                   "'static-priority' (default: 'local-priority'; "
                   "all option values can be abbreviated)")

libs/core/concurrency/include/hpx/concurrency/detail/contiguous_index_queue.hpp

@@ -139,7 +139,7 @@ namespace hpx { namespace concurrency { namespace detail {
             {
                 if (expected_range.empty())
                 {
-                    return hpx::nullopt;
+                    return hpx::optional<T>(hpx::nullopt);
                 }
 
                 index = expected_range.first;
@@ -166,7 +166,7 @@ namespace hpx { namespace concurrency { namespace detail {
             {
                 if (expected_range.empty())
                 {
-                    return hpx::nullopt;
+                    return hpx::optional<T>(hpx::nullopt);
                 }
 
                 desired_range = expected_range.decrement_last();

libs/core/concurrency/include/hpx/concurrency/spinlock.hpp

@@ -13,9 +13,11 @@
 #include <hpx/modules/itt_notify.hpp>
 #include <hpx/thread_support/spinlock.hpp>
 
+#include <utility>
+
 namespace hpx { namespace util {
 
-    /// Lockable spinlock class
+    // Lockable spinlock class
     struct spinlock
     {
     public:
@@ -25,25 +27,32 @@ namespace hpx { namespace util {
         hpx::util::detail::spinlock m;
 
     public:
-        spinlock(char const* /*desc*/ = nullptr)
+        spinlock() noexcept
         {
-            HPX_ITT_SYNC_CREATE(this, "util::spinlock", "");
+            HPX_ITT_SYNC_CREATE(this, "util::spinlock", nullptr);
+        }
+
+        explicit spinlock(char const* desc) noexcept
+        {
+            HPX_ITT_SYNC_CREATE(this, "util::spinlock", desc);
         }
 
         ~spinlock()
         {
             HPX_ITT_SYNC_DESTROY(this);
         }
 
-        void lock() noexcept
+        void lock() noexcept(
+            noexcept(util::register_lock(std::declval<spinlock*>())))
         {
             HPX_ITT_SYNC_PREPARE(this);
             m.lock();
             HPX_ITT_SYNC_ACQUIRED(this);
             util::register_lock(this);
         }
 
-        bool try_lock() noexcept
+        bool try_lock() noexcept(
+            noexcept(util::register_lock(std::declval<spinlock*>())))
         {
             HPX_ITT_SYNC_PREPARE(this);
             if (m.try_lock())
@@ -56,13 +65,13 @@ namespace hpx { namespace util {
             return false;
         }
 
-        void unlock() noexcept
+        void unlock() noexcept(
+            noexcept(util::unregister_lock(std::declval<spinlock*>())))
         {
             HPX_ITT_SYNC_RELEASING(this);
             m.unlock();
             HPX_ITT_SYNC_RELEASED(this);
             util::unregister_lock(this);
         }
     };
-
 }}    // namespace hpx::util

libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner_fwd.hpp

@@ -71,5 +71,7 @@ namespace hpx { namespace resource {
         abp_priority_fifo = 5,
         abp_priority_lifo = 6,
         shared_priority = 7,
+        local_workrequesting_fifo = 8,
+        local_workrequesting_lifo = 9,
     };
 }}    // namespace hpx::resource

libs/core/resource_partitioner/src/detail_partitioner.cpp

@@ -134,6 +134,12 @@ namespace hpx { namespace resource { namespace detail {
         case resource::local_priority_lifo:
             sched = "local_priority_lifo";
             break;
+        case resource::local_workrequesting_fifo:
+            sched = "local_workrequesting_fifo";
+            break;
+        case resource::local_workrequesting_lifo:
+            sched = "local_workrequesting_lifo";
+            break;
         case resource::static_:
             sched = "static";
             break;
@@ -451,6 +457,18 @@ namespace hpx { namespace resource { namespace detail {
         {
             default_scheduler = scheduling_policy::local_priority_lifo;
         }
+        else if (0 ==
+            std::string("local-workrequesting-fifo")
+                .find(default_scheduler_str))
+        {
+            default_scheduler = scheduling_policy::local_workrequesting_fifo;
+        }
+        else if (0 ==
+            std::string("local-workrequesting-lifo")
+                .find(default_scheduler_str))
+        {
+            default_scheduler = scheduling_policy::local_workrequesting_lifo;
+        }
         else if (0 == std::string("static").find(default_scheduler_str))
         {
             default_scheduler = scheduling_policy::static_;

libs/core/runtime_configuration/src/runtime_configuration.cpp

@@ -156,7 +156,7 @@ namespace hpx { namespace util {
             "localities = 1",
             "first_pu = 0",
             "runtime_mode = console",
-            "scheduler = local-priority-fifo",
+            "scheduler = local-workrequesting-fifo",
             "affinity = core",
             "pu_step = 1",
             "pu_offset = 0",

libs/core/schedulers/CMakeLists.txt

@@ -10,6 +10,7 @@ set(schedulers_headers
     hpx/schedulers/deadlock_detection.hpp
     hpx/schedulers/local_priority_queue_scheduler.hpp
     hpx/schedulers/local_queue_scheduler.hpp
+    hpx/schedulers/local_workrequesting_scheduler.hpp
     hpx/schedulers/lockfree_queue_backends.hpp
     hpx/schedulers/maintain_queue_wait_times.hpp
     hpx/schedulers/queue_helpers.hpp
@@ -51,12 +52,15 @@ add_hpx_module(
   COMPAT_HEADERS ${schedulers_compat_headers}
   SOURCES ${schedulers_sources}
   MODULE_DEPENDENCIES
+    hpx_affinity
+    hpx_concurrency
     hpx_config
     hpx_assertion
     hpx_errors
     hpx_format
     hpx_functional
     hpx_logging
+    hpx_synchronization
     hpx_threading_base
   CMAKE_SUBDIRS examples tests
 )

libs/core/schedulers/include/hpx/modules/schedulers.hpp

@@ -1,15 +1,14 @@
-//  Copyright (c) 2007-2013 Hartmut Kaiser
+//  Copyright (c) 2007-2022 Hartmut Kaiser
 //
 //  SPDX-License-Identifier: BSL-1.0
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
 #pragma once
 
-#include <hpx/config.hpp>
-
 #include <hpx/schedulers/local_priority_queue_scheduler.hpp>
 #include <hpx/schedulers/local_queue_scheduler.hpp>
+#include <hpx/schedulers/local_workrequesting_scheduler.hpp>
 #include <hpx/schedulers/shared_priority_queue_scheduler.hpp>
 #include <hpx/schedulers/static_priority_queue_scheduler.hpp>
 #include <hpx/schedulers/static_queue_scheduler.hpp>