orbis-kernel: Add SharedAtomic utility

Initial shared atomic implementation for Darwin

orbis-kernel/include/orbis/utils/SharedAtomic.hpp

@@ -0,0 +1,165 @@
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <system_error>
+#include <thread>
+#include <type_traits>
+
+namespace orbis {
+inline void yield() { std::this_thread::yield(); }
+inline void relax() {
+#if defined(__GNUC__) && (defined __i386__ || defined __x86_64__)
+  __builtin_ia32_pause();
+#else
+  yield();
+#endif
+}
+
+static constexpr auto kRelaxSpinCount = 12;
+static constexpr auto kSpinCount = 16;
+
+inline namespace utils {
+bool try_spin_wait(auto &&pred) {
+  for (std::size_t i = 0; i < kSpinCount; ++i) {
+    if (pred()) {
+      return true;
+    }
+
+    if (i < kRelaxSpinCount) {
+      relax();
+    } else {
+      yield();
+    }
+  }
+
+  return false;
+}
+
+bool spin_wait(auto &&pred, auto &&spinCond) {
+  if (try_spin_wait(pred)) {
+    return true;
+  }
+
+  while (spinCond()) {
+    if (pred()) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+struct shared_atomic32 : std::atomic<std::uint32_t> {
+  using atomic::atomic;
+  using atomic::operator=;
+
+  template <typename Clock, typename Dur>
+  [[nodiscard]] std::errc wait(std::uint32_t oldValue,
+                               std::chrono::time_point<Clock, Dur> timeout) {
+    if (try_spin_wait(
+            [&] { return load(std::memory_order::acquire) != oldValue; })) {
+      return {};
+    }
+
+    auto now = Clock::now();
+
+    if (timeout < now) {
+      return std::errc::timed_out;
+    }
+
+    return wait_impl(
+        oldValue,
+        std::chrono::duration_cast<std::chrono::microseconds>(timeout - now));
+  }
+
+  [[nodiscard]] std::errc wait(std::uint32_t oldValue,
+                               std::chrono::microseconds usec_timeout) {
+    return wait_impl(oldValue, usec_timeout);
+  }
+
+  [[nodiscard]] std::errc wait(std::uint32_t oldValue) {
+    if (try_spin_wait(
+            [&] { return load(std::memory_order::acquire) != oldValue; })) {
+      return {};
+    }
+
+    return wait_impl(oldValue);
+  }
+
+  auto wait(auto &fn) -> decltype(fn(std::declval<std::uint32_t &>())) {
+    while (true) {
+      std::uint32_t lastValue;
+      if (try_spin_wait([&] {
+            lastValue = load(std::memory_order::acquire);
+            return fn(lastValue);
+          })) {
+        return;
+      }
+
+      while (wait_impl(lastValue) != std::errc{}) {
+      }
+    }
+  }
+
+  int notify_one() const { return notify_n(1); }
+  int notify_all() const { return notify_n(std::numeric_limits<int>::max()); }
+
+  int notify_n(int count) const;
+
+  // Atomic operation; returns old value, or pair of old value and return value
+  // (cancel op if evaluates to false)
+  template <typename F, typename RT = std::invoke_result_t<F, std::uint32_t &>>
+  std::conditional_t<std::is_void_v<RT>, std::uint32_t,
+                     std::pair<std::uint32_t, RT>>
+  fetch_op(F &&func) {
+    std::uint32_t _new;
+    std::uint32_t old = load(std::memory_order::relaxed);
+    while (true) {
+      _new = old;
+      if constexpr (std::is_void_v<RT>) {
+        std::invoke(std::forward<F>(func), _new);
+        if (compare_exchange_strong(old, _new)) [[likely]] {
+          return old;
+        }
+      } else {
+        RT ret = std::invoke(std::forward<F>(func), _new);
+        if (!ret || compare_exchange_strong(old, _new)) [[likely]] {
+          return {old, std::move(ret)};
+        }
+      }
+    }
+  }
+
+  // Atomic operation; returns function result value
+  template <typename F, typename RT = std::invoke_result_t<F, std::uint32_t &>>
+  RT op(F &&func) {
+    std::uint32_t _new;
+    std::uint32_t old = load(std::memory_order::relaxed);
+
+    while (true) {
+      _new = old;
+      if constexpr (std::is_void_v<RT>) {
+        std::invoke(std::forward<F>(func), _new);
+        if (compare_exchange_strong(old, _new)) [[likely]] {
+          return;
+        }
+      } else {
+        RT result = std::invoke(std::forward<F>(func), _new);
+        if (compare_exchange_strong(old, _new)) [[likely]] {
+          return result;
+        }
+      }
+    }
+  }
+
+private:
+  [[nodiscard]] std::errc wait_impl(std::uint32_t oldValue,
+                                    std::chrono::microseconds usec_timeout =
+                                        std::chrono::microseconds::max());
+};
+} // namespace utils
+} // namespace orbis

orbis-kernel/include/orbis/utils/SharedCV.hpp

@@ -1,8 +1,12 @@
 #pragma once
 
+#include "orbis/utils/SharedAtomic.hpp"
+#include <chrono>
 #include <cstdint>
+#include <mutex>
 #include <orbis/utils/AtomicOp.hpp>
 #include <orbis/utils/SharedMutex.hpp>
+#include <system_error>
 
 namespace orbis {
 inline namespace utils {
@@ -11,16 +15,18 @@ class shared_cv final {
   enum : unsigned {
     c_waiter_mask = 0xffff,
     c_signal_mask = 0x7fff0000,
+#ifdef ORBIS_HAS_FUTEX
     c_locked_mask = 0x80000000,
+#endif
     c_signal_one = c_waiter_mask + 1,
   };
 
-  std::atomic<unsigned> m_value{0};
+  shared_atomic32 m_value{0};
 
 protected:
   // Increment waiter count
   unsigned add_waiter() noexcept {
-    return atomic_op(m_value, [](unsigned &value) -> unsigned {
+    return m_value.op([](unsigned &value) -> unsigned {
       if ((value & c_signal_mask) == c_signal_mask ||
           (value & c_waiter_mask) == c_waiter_mask) {
         // Signal or waiter overflow, return immediately
@@ -34,8 +40,8 @@ protected:
   }
 
   // Internal waiting function
-  int impl_wait(shared_mutex &mutex, unsigned _val,
-                 std::uint64_t usec_timeout) noexcept;
+  std::errc impl_wait(shared_mutex &mutex, unsigned _val,
+                      std::uint64_t usec_timeout) noexcept;
 
   // Try to notify up to _count threads
   void impl_wake(shared_mutex &mutex, int _count) noexcept;
@@ -43,10 +49,25 @@ protected:
 public:
   constexpr shared_cv() = default;
 
-  int wait(shared_mutex &mutex, std::uint64_t usec_timeout = -1) noexcept {
+  std::errc
+  wait(std::unique_lock<shared_mutex> &lock,
+       std::chrono::microseconds timeout = std::chrono::microseconds::max()) {
+    return wait(*lock.mutex(), timeout.count());
+  }
+
+  template <typename Rep, typename Period>
+  std::errc wait(std::unique_lock<shared_mutex> &lock,
+                 std::chrono::duration<Rep, Period> timeout) {
+    return wait(
+        lock,
+        std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
+  }
+
+  std::errc wait(shared_mutex &mutex,
+                 std::uint64_t usec_timeout = -1) noexcept {
     const unsigned _val = add_waiter();
     if (!_val) {
-      return 0;
+      return {};
     }
 
     mutex.unlock();

orbis-kernel/include/orbis/utils/SharedMutex.hpp

@@ -1,8 +1,9 @@
 #pragma once
 
 #include <atomic>
-#include <mutex>
 #include <orbis/utils/AtomicOp.hpp>
+#include <orbis/utils/SharedAtomic.hpp>
+#include <system_error>
 
 namespace orbis {
 inline namespace utils {
@@ -16,11 +17,11 @@ class shared_mutex final {
     c_err = 1u << 31,
   };
 
-  std::atomic<unsigned> m_value{};
+  shared_atomic32 m_value{};
 
   void impl_lock_shared(unsigned val);
   void impl_unlock_shared(unsigned old);
-  int impl_wait();
+  std::errc impl_wait();
   void impl_signal();
   void impl_lock(unsigned val);
   void impl_unlock(unsigned old);
@@ -99,10 +100,10 @@ public:
   }
 
   // Check whether can immediately obtain an exclusive (writer) lock
-  bool is_free() const { return m_value.load() == 0; }
+  [[nodiscard]] bool is_free() const { return m_value.load() == 0; }
 
   // Check whether can immediately obtain a shared (reader) lock
-  bool is_lockable() const { return m_value.load() < c_one - 1; }
+  [[nodiscard]] bool is_lockable() const { return m_value.load() < c_one - 1; }
 
 private:
   // For CV