Implement fast user-space mutexes on Windows using Wait/WakeOnAddress

bloomberg · Jan 7, 2025 · 14f2dca · 14f2dca
1 parent 7ff6042
commit 14f2dca
Show file tree

Hide file tree

Showing 7 changed files with 407 additions and 16 deletions.
diff --git a/configure.cmd b/configure.cmd
@@ -154,7 +154,7 @@ IF NOT DEFINED NTF_CONFIGURE_WITH_SPIN_LOCKS (
 )
 
 IF NOT DEFINED NTF_CONFIGURE_WITH_USERSPACE_MUTEXES (
-    set NTF_CONFIGURE_WITH_USERSPACE_MUTEXES=0
+    set NTF_CONFIGURE_WITH_USERSPACE_MUTEXES=1
 )
 
 IF NOT DEFINED NTF_CONFIGURE_WITH_SYSTEM_MUTEXES (

diff --git a/groups/ntc/ntccfg/ntccfg_inline.h b/groups/ntc/ntccfg/ntccfg_inline.h
@@ -65,4 +65,24 @@ BSLS_IDENT("$Id: $")
 #error Not implemented
 #endif
 
+#if defined(BSLS_PLATFORM_CMP_GNU) || defined(BSLS_PLATFORM_CMP_CLANG)
+
+/// Hint that the following function should never be inlined at its call site.
+/// @ingroup module_ntccfg
+#define NTCCFG_INLINE_NEVER __attribute__((noinline))
+
+#elif defined(BSLS_PLATFORM_CMP_MSVC)
+
+/// Hint that the following function should never be inlined at its call site.
+/// @ingroup module_ntccfg
+#define NTCCFG_INLINE_NEVER __declspec(noinline)
+
+#else
+
+/// Hint that the following function should never be inlined at its call site.
+/// @ingroup module_ntccfg
+#define NTCCFG_INLINE_NEVER
+
+#endif
+
 #endif
diff --git a/groups/ntc/ntccfg/ntccfg_mutex.cpp b/groups/ntc/ntccfg/ntccfg_mutex.cpp
@@ -18,9 +18,7 @@
 #include <bsls_ident.h>
 BSLS_IDENT_RCSID(ntccfg_mutex_cpp, "$Id$ $CSID$")
 
-#include <bslma_allocator.h>
-#include <bslma_default.h>
-#include <bsls_assert.h>
+#if NTCCFG_FUTEX_ENABLED
 
 #if defined(BSLS_PLATFORM_OS_LINUX)
 #include <errno.h>
@@ -29,29 +27,61 @@ BSLS_IDENT_RCSID(ntccfg_mutex_cpp, "$Id$ $CSID$")
 #include <unistd.h>
 #endif
 
+#if defined(BSLS_PLATFORM_OS_WINDOWS)
+#ifdef NTDDI_VERSION
+#undef NTDDI_VERSION
+#endif
+#ifdef WINVER
+#undef WINVER
+#endif
+#ifdef _WIN32_WINNT
+#undef _WIN32_WINNT
+#endif
+#define NTDDI_VERSION 0x06000100
+#define WINVER 0x0600
+#define _WIN32_WINNT 0x0600
+#ifndef _WINSOCK_DEPRECATED_NO_WARNINGS
+#define _WINSOCK_DEPRECATED_NO_WARNINGS
+#endif
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+// clang-format off
+#include <windows.h>
+// clang-format on
+#pragma comment(lib, "synchronization")
+#endif
+
+#endif // NTCCFG_FUTEX_ENABLED
+
 namespace BloombergLP {
 namespace ntccfg {
 
 #if NTCCFG_FUTEX_ENABLED
 
+#if defined(BSLS_PLATFORM_OS_LINUX)
+
 // Some versions of GCC issue a spurious warning that the 'current' parameter
 // is set but not used when 'Futex::compareAndSwap' is called.
 #if defined(BSLS_PLATFORM_CMP_GNU)
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-but-set-parameter"
 #endif
 
-__attribute__((noinline)) void Futex::wait()
+NTCCFG_INLINE_NEVER
+void Futex::wait()
 {
     syscall(SYS_futex, (int*)(&d_value), FUTEX_WAIT, 2, 0, 0, 0);
 }
 
-__attribute__((noinline)) void Futex::wake()
+NTCCFG_INLINE_NEVER
+void Futex::wake()
 {
     syscall(SYS_futex, (int*)(&d_value), FUTEX_WAKE, 1, 0, 0, 0);
 }
 
-__attribute__((noinline)) void Futex::lockContention(int c)
+NTCCFG_INLINE_NEVER
+void Futex::lockContention(int c)
 {
     do {
         if (c == 2 || compareAndSwap(&d_value, 1, 2) != 0) {
@@ -60,7 +90,8 @@ __attribute__((noinline)) void Futex::lockContention(int c)
     } while ((c = compareAndSwap(&d_value, 0, 2)) != 0);
 }
 
-__attribute__((noinline)) void Futex::unlockContention()
+NTCCFG_INLINE_NEVER
+void Futex::unlockContention()
 {
     __atomic_store_n(&d_value, 0, __ATOMIC_SEQ_CST);
     this->wake();
@@ -70,6 +101,53 @@ __attribute__((noinline)) void Futex::unlockContention()
 #pragma GCC diagnostic pop
 #endif
 
+#elif defined(BSLS_PLATFORM_OS_WINDOWS)
+
+NTCCFG_INLINE_NEVER
+void Futex::wait()
+{
+    Value compare = 2;
+    WaitOnAddress(&d_value, &compare, sizeof compare, INFINITE);
+}
+
+NTCCFG_INLINE_NEVER
+void Futex::wake()
+{
+    WakeByAddressSingle((Value*)(&d_value));
+}
+
+NTCCFG_INLINE_NEVER
+void Futex::lockContention(Value c)
+{
+    while (true) {
+        if (c == 2) {
+            this->wait();
+        }
+        else {
+            const Value previous = compareAndSwap(&d_value, 1, 2);
+            if (previous != 0) {
+                this->wait();
+            }
+        }
+
+        c = compareAndSwap(&d_value, 0, 2);
+        if (c == 0) {
+            break;
+        }
+    }
+}
+
+NTCCFG_INLINE_NEVER
+void Futex::unlockContention()
+{
+    _InterlockedExchange(&d_value, 0);
+    this->wake();
+}
+
+#else
+#error Not implemented
+#endif
+
 #endif  // NTCCFG_FUTEX_ENABLED
 
 }  // close package namespace

diff --git a/groups/ntc/ntccfg/ntccfg_mutex.h b/groups/ntc/ntccfg/ntccfg_mutex.h
@@ -33,15 +33,23 @@ BSLS_IDENT("$Id: $")
 #if defined(BSLS_PLATFORM_OS_LINUX) &&                                        \
     (defined(BSLS_PLATFORM_CMP_GNU) || defined(BSLS_PLATFORM_CMP_CLANG))
 #define NTCCFG_FUTEX_ENABLED 1
+#elif defined(BSLS_PLATFORM_OS_WINDOWS) && defined(BSLS_PLATFORM_CMP_MSVC)
+#define NTCCFG_FUTEX_ENABLED 1
 #else
 #define NTCCFG_FUTEX_ENABLED 0
 #endif
 
+#if NTCCFG_WUTEX_ENABLED
+#include <intrin.h>
+#endif
+
 namespace BloombergLP {
 namespace ntccfg {
 
 #if NTCCFG_FUTEX_ENABLED
 
+#if defined(BSLS_PLATFORM_OS_LINUX)
+
 // Some versions of GCC issue a spurious warning that the 'current' parameter
 // is set but not used when 'Futex::compareAndSwap' is called.
 #if defined(BSLS_PLATFORM_CMP_GNU)
@@ -148,6 +156,105 @@ void Futex::unlock()
 #pragma GCC diagnostic pop
 #endif
 
+#elif defined(BSLS_PLATFORM_OS_WINDOWS)
+
+/// @internal @brief
+/// Provide a synchronization primitive for mutually-exclusive access
+/// implemented by the Win32 WaitOnAddress function.
+///
+/// @par Thread Safety
+/// This class is thread safe.
+///
+/// @ingroup module_ntccfg
+__declspec(align(4)) class Futex
+{
+    /// Define a type alias for the type of the synchronized value.
+    typedef long Value;
+
+    /// The synchronized value.
+    volatile Value d_value;
+
+  private:
+    Futex(const Futex&) BSLS_KEYWORD_DELETED;
+    Futex& operator=(const Futex&) BSLS_KEYWORD_DELETED;
+
+  private:
+    /// Wait until the lock may be acquired.
+    void wait();
+
+    /// Wake the next thread waiting to acquire the lock.
+    void wake();
+
+    /// Continue locking the mutex after discovering the mutex was probably
+    /// previously unlocked.
+    void lockContention(Value c);
+
+    /// Continue unlocking the mutex after discovering the mutex probably
+    /// has other threads trying to lock the mutex.
+    void unlockContention();
+
+    /// Compare the specified '*current' value to the specified 'expected'
+    /// value, and if equal, set '*current' to 'desired'. Return the
+    /// previous value of 'current'.
+    static Value compareAndSwap(volatile Value* current, 
+                                Value           expected, 
+                                Value           desired);
+
+  public:
+    /// Create a new mutex.
+    Futex();
+
+    /// Destroy this object.
+    ~Futex();
+
+    /// Lock the mutex.
+    void lock();
+
+    /// Unlock the mutex.
+    void unlock();
+};
+
+NTCCFG_INLINE
+Futex::Value Futex::compareAndSwap(volatile Value* current, 
+                                   Value           expected, 
+                                   Value           desired)
+{
+    return _InterlockedCompareExchange(current, desired, expected);
+}
+
+NTCCFG_INLINE
+Futex::Futex()
+{
+    _InterlockedExchange(&d_value, 0);
+}
+
+NTCCFG_INLINE
+Futex::~Futex()
+{
+}
+
+NTCCFG_INLINE
+void Futex::lock()
+{
+    const Value previous = compareAndSwap(&d_value, 0, 1);
+    if (previous != 0) {
+        this->lockContention(previous);
+    }
+}
+
+NTCCFG_INLINE
+void Futex::unlock()
+{
+    const Value previous = _InterlockedDecrement(&d_value) + 1;
+    if (previous != 1) {
+        this->unlockContention();
+    }
+}
+
+#else
+#error Not implemented
+#endif
+
 #endif  // NTCCFG_FUTEX_ENABLED
 
 /// @internal @brief