orbis-kernel: Add SharedAtomic utility

Initial shared atomic implementation for Darwin

orbis-kernel/include/orbis/error.hpp

@@ -2,3 +2,165 @@
 
 #include "error/ErrorCode.hpp" // IWYU pragma: export
 #include "error/SysResult.hpp" // IWYU pragma: export
+#include <system_error>
+
+namespace orbis {
+static orbis::ErrorCode toErrorCode(std::errc errc) {
+  if (errc == std::errc{}) {
+    return {};
+  }
+
+  switch (errc) {
+  case std::errc::address_family_not_supported:
+    return orbis::ErrorCode::AFNOSUPPORT;
+  case std::errc::address_in_use:
+    return orbis::ErrorCode::ADDRINUSE;
+  case std::errc::address_not_available:
+    return orbis::ErrorCode::ADDRNOTAVAIL;
+  case std::errc::already_connected:
+    return orbis::ErrorCode::ISCONN;
+  case std::errc::argument_out_of_domain:
+    return orbis::ErrorCode::DOM;
+  case std::errc::bad_address:
+    return orbis::ErrorCode::FAULT;
+  case std::errc::bad_file_descriptor:
+    return orbis::ErrorCode::BADF;
+  case std::errc::bad_message:
+    return orbis::ErrorCode::BADMSG;
+  case std::errc::broken_pipe:
+    return orbis::ErrorCode::PIPE;
+  case std::errc::connection_aborted:
+    return orbis::ErrorCode::CONNABORTED;
+  case std::errc::connection_already_in_progress:
+    return orbis::ErrorCode::ALREADY;
+  case std::errc::connection_refused:
+    return orbis::ErrorCode::CONNREFUSED;
+  case std::errc::connection_reset:
+    return orbis::ErrorCode::CONNRESET;
+  case std::errc::cross_device_link:
+    return orbis::ErrorCode::XDEV;
+  case std::errc::destination_address_required:
+    return orbis::ErrorCode::DESTADDRREQ;
+  case std::errc::device_or_resource_busy:
+    return orbis::ErrorCode::BUSY;
+  case std::errc::directory_not_empty:
+    return orbis::ErrorCode::NOTEMPTY;
+  case std::errc::executable_format_error:
+    return orbis::ErrorCode::NOEXEC;
+  case std::errc::file_exists:
+    return orbis::ErrorCode::EXIST;
+  case std::errc::file_too_large:
+    return orbis::ErrorCode::FBIG;
+  case std::errc::filename_too_long:
+    return orbis::ErrorCode::NAMETOOLONG;
+  case std::errc::function_not_supported:
+    return orbis::ErrorCode::NOSYS;
+  case std::errc::host_unreachable:
+    return orbis::ErrorCode::HOSTUNREACH;
+  case std::errc::identifier_removed:
+    return orbis::ErrorCode::IDRM;
+  case std::errc::illegal_byte_sequence:
+    return orbis::ErrorCode::ILSEQ;
+  case std::errc::inappropriate_io_control_operation:
+    return orbis::ErrorCode::NOTTY;
+  case std::errc::interrupted:
+    return orbis::ErrorCode::INTR;
+  case std::errc::invalid_argument:
+    return orbis::ErrorCode::INVAL;
+  case std::errc::invalid_seek:
+    return orbis::ErrorCode::SPIPE;
+  case std::errc::io_error:
+    return orbis::ErrorCode::IO;
+  case std::errc::is_a_directory:
+    return orbis::ErrorCode::ISDIR;
+  case std::errc::message_size:
+    return orbis::ErrorCode::MSGSIZE;
+  case std::errc::network_down:
+    return orbis::ErrorCode::NETDOWN;
+  case std::errc::network_reset:
+    return orbis::ErrorCode::NETRESET;
+  case std::errc::network_unreachable:
+    return orbis::ErrorCode::NETUNREACH;
+  case std::errc::no_buffer_space:
+    return orbis::ErrorCode::NOBUFS;
+  case std::errc::no_child_process:
+    return orbis::ErrorCode::CHILD;
+  case std::errc::no_link:
+    return orbis::ErrorCode::NOLINK;
+  case std::errc::no_lock_available:
+    return orbis::ErrorCode::NOLCK;
+  case std::errc::no_message:
+    return orbis::ErrorCode::NOMSG;
+  case std::errc::no_protocol_option:
+    return orbis::ErrorCode::NOPROTOOPT;
+  case std::errc::no_space_on_device:
+    return orbis::ErrorCode::NOSPC;
+  case std::errc::no_such_device_or_address:
+    return orbis::ErrorCode::NXIO;
+  case std::errc::no_such_device:
+    return orbis::ErrorCode::NODEV;
+  case std::errc::no_such_file_or_directory:
+    return orbis::ErrorCode::NOENT;
+  case std::errc::no_such_process:
+    return orbis::ErrorCode::SRCH;
+  case std::errc::not_a_directory:
+    return orbis::ErrorCode::NOTDIR;
+  case std::errc::not_a_socket:
+    return orbis::ErrorCode::NOTSOCK;
+  case std::errc::not_connected:
+    return orbis::ErrorCode::NOTCONN;
+  case std::errc::not_enough_memory:
+    return orbis::ErrorCode::NOMEM;
+  case std::errc::not_supported:
+    return orbis::ErrorCode::NOTSUP;
+  case std::errc::operation_canceled:
+    return orbis::ErrorCode::CANCELED;
+  case std::errc::operation_in_progress:
+    return orbis::ErrorCode::INPROGRESS;
+  case std::errc::operation_not_permitted:
+    return orbis::ErrorCode::PERM;
+  case std::errc::operation_would_block:
+    return orbis::ErrorCode::WOULDBLOCK;
+  case std::errc::permission_denied:
+    return orbis::ErrorCode::ACCES;
+  case std::errc::protocol_error:
+    return orbis::ErrorCode::PROTO;
+  case std::errc::protocol_not_supported:
+    return orbis::ErrorCode::PROTONOSUPPORT;
+  case std::errc::read_only_file_system:
+    return orbis::ErrorCode::ROFS;
+  case std::errc::resource_deadlock_would_occur:
+    return orbis::ErrorCode::DEADLK;
+  case std::errc::result_out_of_range:
+    return orbis::ErrorCode::RANGE;
+  case std::errc::text_file_busy:
+    return orbis::ErrorCode::TXTBSY;
+  case std::errc::timed_out:
+    return orbis::ErrorCode::TIMEDOUT;
+  case std::errc::too_many_files_open_in_system:
+    return orbis::ErrorCode::NFILE;
+  case std::errc::too_many_files_open:
+    return orbis::ErrorCode::MFILE;
+  case std::errc::too_many_links:
+    return orbis::ErrorCode::MLINK;
+  case std::errc::too_many_symbolic_link_levels:
+    return orbis::ErrorCode::LOOP;
+  case std::errc::value_too_large:
+    return orbis::ErrorCode::OVERFLOW;
+  case std::errc::wrong_protocol_type:
+    return orbis::ErrorCode::PROTOTYPE;
+  default:
+    return orbis::ErrorCode::FAULT;
+  }
+}
+
+inline constexpr orbis::ErrorCode toErrorCode(const std::error_code &code) {
+  if (!code) {
+    return {};
+  }
+  if (code.category() != std::generic_category()) {
+    return orbis::ErrorCode::DOOFUS;
+  }
+  return toErrorCode(static_cast<std::errc>(code.value()));
+}
+} // namespace orbis

orbis-kernel/include/orbis/utils/SharedAtomic.hpp

@@ -0,0 +1,165 @@
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <system_error>
+#include <thread>
+#include <type_traits>
+
+namespace orbis {
+inline void yield() { std::this_thread::yield(); }
+inline void relax() {
+#if defined(__GNUC__) && (defined __i386__ || defined __x86_64__)
+  __builtin_ia32_pause();
+#else
+  yield();
+#endif
+}
+
+static constexpr auto kRelaxSpinCount = 12;
+static constexpr auto kSpinCount = 16;
+
+inline namespace utils {
+bool try_spin_wait(auto &&pred) {
+  for (std::size_t i = 0; i < kSpinCount; ++i) {
+    if (pred()) {
+      return true;
+    }
+
+    if (i < kRelaxSpinCount) {
+      relax();
+    } else {
+      yield();
+    }
+  }
+
+  return false;
+}
+
+bool spin_wait(auto &&pred, auto &&spinCond) {
+  if (try_spin_wait(pred)) {
+    return true;
+  }
+
+  while (spinCond()) {
+    if (pred()) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+struct shared_atomic32 : std::atomic<std::uint32_t> {
+  using atomic::atomic;
+  using atomic::operator=;
+
+  template <typename Clock, typename Dur>
+  [[nodiscard]] std::errc wait(std::uint32_t oldValue,
+                               std::chrono::time_point<Clock, Dur> timeout) {
+    if (try_spin_wait(
+            [&] { return load(std::memory_order::acquire) != oldValue; })) {
+      return {};
+    }
+
+    auto now = Clock::now();
+
+    if (timeout < now) {
+      return std::errc::timed_out;
+    }
+
+    return wait_impl(
+        oldValue,
+        std::chrono::duration_cast<std::chrono::microseconds>(timeout - now));
+  }
+
+  [[nodiscard]] std::errc wait(std::uint32_t oldValue,
+                               std::chrono::microseconds usec_timeout) {
+    return wait_impl(oldValue, usec_timeout);
+  }
+
+  [[nodiscard]] std::errc wait(std::uint32_t oldValue) {
+    if (try_spin_wait(
+            [&] { return load(std::memory_order::acquire) != oldValue; })) {
+      return {};
+    }
+
+    return wait_impl(oldValue);
+  }
+
+  auto wait(auto &fn) -> decltype(fn(std::declval<std::uint32_t &>())) {
+    while (true) {
+      std::uint32_t lastValue;
+      if (try_spin_wait([&] {
+            lastValue = load(std::memory_order::acquire);
+            return fn(lastValue);
+          })) {
+        return;
+      }
+
+      while (wait_impl(lastValue) != std::errc{}) {
+      }
+    }
+  }
+
+  int notify_one() const { return notify_n(1); }
+  int notify_all() const { return notify_n(std::numeric_limits<int>::max()); }
+
+  int notify_n(int count) const;
+
+  // Atomic operation; returns old value, or pair of old value and return value
+  // (cancel op if evaluates to false)
+  template <typename F, typename RT = std::invoke_result_t<F, std::uint32_t &>>
+  std::conditional_t<std::is_void_v<RT>, std::uint32_t,
+                     std::pair<std::uint32_t, RT>>
+  fetch_op(F &&func) {
+    std::uint32_t _new;
+    std::uint32_t old = load(std::memory_order::relaxed);
+    while (true) {
+      _new = old;
+      if constexpr (std::is_void_v<RT>) {
+        std::invoke(std::forward<F>(func), _new);
+        if (compare_exchange_strong(old, _new)) [[likely]] {
+          return old;
+        }
+      } else {
+        RT ret = std::invoke(std::forward<F>(func), _new);
+        if (!ret || compare_exchange_strong(old, _new)) [[likely]] {
+          return {old, std::move(ret)};
+        }
+      }
+    }
+  }
+
+  // Atomic operation; returns function result value
+  template <typename F, typename RT = std::invoke_result_t<F, std::uint32_t &>>
+  RT op(F &&func) {
+    std::uint32_t _new;
+    std::uint32_t old = load(std::memory_order::relaxed);
+
+    while (true) {
+      _new = old;
+      if constexpr (std::is_void_v<RT>) {
+        std::invoke(std::forward<F>(func), _new);
+        if (compare_exchange_strong(old, _new)) [[likely]] {
+          return;
+        }
+      } else {
+        RT result = std::invoke(std::forward<F>(func), _new);
+        if (compare_exchange_strong(old, _new)) [[likely]] {
+          return result;
+        }
+      }
+    }
+  }
+
+private:
+  [[nodiscard]] std::errc wait_impl(std::uint32_t oldValue,
+                                    std::chrono::microseconds usec_timeout =
+                                        std::chrono::microseconds::max());
+};
+} // namespace utils
+} // namespace orbis

orbis-kernel/include/orbis/utils/SharedCV.hpp

@@ -1,8 +1,12 @@
 #pragma once
 
+#include "orbis/utils/SharedAtomic.hpp"
+#include <chrono>
 #include <cstdint>
+#include <mutex>
 #include <orbis/utils/AtomicOp.hpp>
 #include <orbis/utils/SharedMutex.hpp>
+#include <system_error>
 
 namespace orbis {
 inline namespace utils {
@@ -11,16 +15,18 @@ class shared_cv final {
   enum : unsigned {
     c_waiter_mask = 0xffff,
     c_signal_mask = 0x7fff0000,
+#ifdef ORBIS_HAS_FUTEX
     c_locked_mask = 0x80000000,
+#endif
     c_signal_one = c_waiter_mask + 1,
   };
 
-  std::atomic<unsigned> m_value{0};
+  shared_atomic32 m_value{0};
 
 protected:
   // Increment waiter count
   unsigned add_waiter() noexcept {
-    return atomic_op(m_value, [](unsigned &value) -> unsigned {
+    return m_value.op([](unsigned &value) -> unsigned {
       if ((value & c_signal_mask) == c_signal_mask ||
           (value & c_waiter_mask) == c_waiter_mask) {
         // Signal or waiter overflow, return immediately
@@ -34,8 +40,8 @@ protected:
   }
 
   // Internal waiting function
-  int impl_wait(shared_mutex &mutex, unsigned _val,
-                 std::uint64_t usec_timeout) noexcept;
+  std::errc impl_wait(shared_mutex &mutex, unsigned _val,
+                      std::uint64_t usec_timeout) noexcept;
 
   // Try to notify up to _count threads
   void impl_wake(shared_mutex &mutex, int _count) noexcept;
@@ -43,10 +49,25 @@ protected:
 public:
   constexpr shared_cv() = default;
 
-  int wait(shared_mutex &mutex, std::uint64_t usec_timeout = -1) noexcept {
+  std::errc
+  wait(std::unique_lock<shared_mutex> &lock,
+       std::chrono::microseconds timeout = std::chrono::microseconds::max()) {
+    return wait(*lock.mutex(), timeout.count());
+  }
+
+  template <typename Rep, typename Period>
+  std::errc wait(std::unique_lock<shared_mutex> &lock,
+                 std::chrono::duration<Rep, Period> timeout) {
+    return wait(
+        lock,
+        std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
+  }
+
+  std::errc wait(shared_mutex &mutex,
+                 std::uint64_t usec_timeout = -1) noexcept {
     const unsigned _val = add_waiter();
     if (!_val) {
-      return 0;
+      return {};
     }
 
     mutex.unlock();

orbis-kernel/include/orbis/utils/SharedMutex.hpp

@@ -1,8 +1,9 @@
 #pragma once
 
 #include <atomic>
-#include <mutex>
 #include <orbis/utils/AtomicOp.hpp>
+#include <orbis/utils/SharedAtomic.hpp>
+#include <system_error>
 
 namespace orbis {
 inline namespace utils {
@@ -16,11 +17,11 @@ class shared_mutex final {
     c_err = 1u << 31,
   };
 
-  std::atomic<unsigned> m_value{};
+  shared_atomic32 m_value{};
 
   void impl_lock_shared(unsigned val);
   void impl_unlock_shared(unsigned old);
-  int impl_wait();
+  std::errc impl_wait();
   void impl_signal();
   void impl_lock(unsigned val);
   void impl_unlock(unsigned old);
@@ -99,10 +100,10 @@ public:
   }
 
   // Check whether can immediately obtain an exclusive (writer) lock
-  bool is_free() const { return m_value.load() == 0; }
+  [[nodiscard]] bool is_free() const { return m_value.load() == 0; }
 
   // Check whether can immediately obtain a shared (reader) lock
-  bool is_lockable() const { return m_value.load() < c_one - 1; }
+  [[nodiscard]] bool is_lockable() const { return m_value.load() < c_one - 1; }
 
 private:
   // For CV