RPCSX/rpcsx
1
0
Fork 0
mirror of https://github.com/RPCSX/rpcsx.git synced 2026-04-05 14:37:08 +00:00
Code Issues Projects Releases Packages Wiki Activity

move IPC utilities from orbis-kernel to rx

Browse source
This commit is contained in:
DH 2025-10-05 00:09:42 +03:00
parent 30469f7fb9
commit e73a0b962d
41 changed files with 558 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

24
kernel/orbis/CMakeLists.txt
View file

@ -1,10 +1,5 @@
set(CMAKE_POSITION_INDEPENDENT_CODE on)
add_library(obj.orbis-utils-ipc OBJECT
src/utils/SharedMutex.cpp
src/utils/SharedCV.cpp
src/utils/SharedAtomic.cpp
)
add_library(obj.orbis-kernel OBJECT
src/module.cpp
src/pipe.cpp
@ -72,7 +67,7 @@ add_library(obj.orbis-kernel OBJECT
src/utils/Logs.cpp
)
target_link_libraries(obj.orbis-kernel PUBLIC orbis::kernel::config $<TARGET_OBJECTS:obj.orbis-utils-ipc>)
target_link_libraries(obj.orbis-kernel PUBLIC orbis::kernel::config rx)
target_include_directories(obj.orbis-kernel
PUBLIC
@ -82,27 +77,10 @@ target_include_directories(obj.orbis-kernel
${CMAKE_CURRENT_SOURCE_DIR}/include/orbis
)
target_include_directories(obj.orbis-utils-ipc
PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/include
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include/orbis
)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
target_compile_definitions(obj.orbis-utils-ipc PUBLIC ORBIS_HAS_FUTEX)
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
target_compile_definitions(obj.orbis-utils-ipc PUBLIC ORBIS_HAS_ULOCK)
endif()
add_library(orbis-utils-ipc STATIC)
add_library(orbis-kernel STATIC)
add_library(orbis-kernel-shared SHARED)
add_library(orbis::utils::ipc ALIAS orbis-utils-ipc)
add_library(orbis::kernel ALIAS orbis-kernel)
add_library(orbis::kernel-shared ALIAS orbis-kernel-shared)
target_link_libraries(orbis-utils-ipc PUBLIC obj.orbis-utils-ipc)
target_link_libraries(orbis-kernel PUBLIC obj.orbis-kernel)
target_link_libraries(orbis-kernel-shared PUBLIC obj.orbis-kernel)

5
kernel/orbis/include/orbis/Budget.hpp
View file

@ -1,12 +1,13 @@
#pragma once
#include "orbis-config.hpp"
#include "rx/SharedMutex.hpp"
#include "utils/BitSet.hpp"
#include "utils/Rc.hpp"
#include "utils/SharedMutex.hpp"
#include <array>
#include <cstring>
#include <mutex>
#include <span>
#include <string_view>
namespace orbis {
@ -132,7 +133,7 @@ public:
[[nodiscard]] ProcessType processType() const { return mProcessType; }
private:
mutable shared_mutex mMtx;
mutable rx::shared_mutex mMtx;
orbis::BitSet<static_cast<int>(BudgetResource::_count)> mUsed;
ProcessType mProcessType{};
BudgetList mList;

32
kernel/orbis/include/orbis/KernelContext.hpp
View file

@ -6,11 +6,11 @@
#include "ipmi.hpp"
#include "orbis/note.hpp"
#include "osem.hpp"
#include "rx/SharedCV.hpp"
#include "rx/SharedMutex.hpp"
#include "thread/types.hpp"
#include "utils/IdMap.hpp"
#include "utils/LinkedNode.hpp"
#include "utils/SharedCV.hpp"
#include "utils/SharedMutex.hpp"
#include <cstdint>
#include <mutex>
@ -32,13 +32,13 @@ struct UmtxKey {
struct UmtxCond {
Thread *thr;
utils::shared_cv cv;
rx::shared_cv cv;
UmtxCond(Thread *thr) : thr(thr) {}
};
struct UmtxChain {
utils::shared_mutex mtx;
rx::shared_mutex mtx;
using queue_type = utils::kmultimap<UmtxKey, UmtxCond>;
queue_type sleep_queue;
queue_type spare_queue;
@ -158,7 +158,7 @@ public:
}
std::tuple<utils::kmap<utils::kstring, char[128]> &,
std::unique_lock<shared_mutex>>
std::unique_lock<rx::shared_mutex>>
getKernelEnv() {
std::unique_lock lock(m_kenv_mtx);
return {m_kenv, std::move(lock)};
@ -178,7 +178,7 @@ public:
};
// Use getUmtxChain0 or getUmtxChain1
std::tuple<UmtxChain &, UmtxKey, std::unique_lock<shared_mutex>>
std::tuple<UmtxChain &, UmtxKey, std::unique_lock<rx::shared_mutex>>
getUmtxChainIndexed(int i, Thread *t, uint32_t flags, void *ptr);
// Internal Umtx: Wait/Cv/Sem
@ -197,7 +197,7 @@ public:
Ref<RcBase> blockpoolDevice;
Ref<RcBase> gpuDevice;
Ref<RcBase> dceDevice;
shared_mutex gpuDeviceMtx;
rx::shared_mutex gpuDeviceMtx;
uint sdkVersion{};
uint fwSdkVersion{};
uint safeMode{};
@ -206,7 +206,7 @@ public:
RcIdMap<Budget, sint, 4097, 1> budgets;
Ref<Budget> processTypeBudgets[4];
shared_mutex regMgrMtx;
rx::shared_mutex regMgrMtx;
kmap<std::uint32_t, std::uint32_t> regMgrInt;
std::vector<std::tuple<std::uint8_t *, size_t>> dialogs{};
@ -227,8 +227,8 @@ public:
}
private:
shared_mutex m_heap_mtx;
shared_mutex m_heap_map_mtx;
rx::shared_mutex m_heap_mtx;
rx::shared_mutex m_heap_map_mtx;
void *m_heap_next = this + 1;
utils::kmultimap<std::size_t, void *> m_free_heap;
@ -238,21 +238,21 @@ private:
std::atomic<long> m_tsc_freq{0};
shared_mutex m_thread_id_mtx;
rx::shared_mutex m_thread_id_mtx;
OwningIdMap<char, long, 256, 0> m_thread_id_map;
mutable shared_mutex m_proc_mtx;
mutable rx::shared_mutex m_proc_mtx;
utils::LinkedNode<Process> *m_processes = nullptr;
shared_mutex m_evf_mtx;
rx::shared_mutex m_evf_mtx;
utils::kmap<utils::kstring, Ref<EventFlag>> m_event_flags;
shared_mutex m_sem_mtx;
rx::shared_mutex m_sem_mtx;
utils::kmap<utils::kstring, Ref<Semaphore>> m_semaphores;
shared_mutex mIpmiServerMtx;
rx::shared_mutex mIpmiServerMtx;
utils::kmap<utils::kstring, Ref<IpmiServer>> mIpmiServers;
shared_mutex m_kenv_mtx;
rx::shared_mutex m_kenv_mtx;
utils::kmap<utils::kstring, char[128]> m_kenv; // max size: 127 + '\0'
};

4
kernel/orbis/include/orbis/event.hpp
View file

@ -1,12 +1,12 @@
#pragma once
#include "file.hpp"
#include "note.hpp"
#include "utils/SharedCV.hpp"
#include "rx/SharedCV.hpp"
#include <list>
namespace orbis {
struct KQueue : orbis::File {
shared_cv cv;
rx::shared_cv cv;
kstring name;
kvector<KEvent> triggeredEvents;
std::list<KNote, kallocator<KNote>> notes;

6
kernel/orbis/include/orbis/evf.hpp
View file

@ -1,7 +1,7 @@
#pragma once
#include "KernelAllocator.hpp"
#include "thread/Thread.hpp"
#include "utils/SharedMutex.hpp"
#include "rx/SharedMutex.hpp"
#include <atomic>
namespace orbis {
@ -56,7 +56,7 @@ struct EventFlag final {
}
};
utils::shared_mutex queueMtx;
rx::shared_mutex queueMtx;
utils::kvector<WaitingThread> waitingThreads;
enum class NotifyType { Set, Cancel, Destroy };
@ -80,7 +80,7 @@ struct EventFlag final {
std::size_t set(std::uint64_t bits) { return notify(NotifyType::Set, bits); }
void clear(std::uint64_t bits) {
writer_lock lock(queueMtx);
rx::writer_lock lock(queueMtx);
value.fetch_and(bits, std::memory_order::relaxed);
}

4
kernel/orbis/include/orbis/file.hpp
View file

@ -3,9 +3,9 @@
#include "KernelAllocator.hpp"
#include "error/ErrorCode.hpp"
#include "note.hpp"
#include "rx/SharedMutex.hpp"
#include "stat.hpp"
#include "utils/Rc.hpp"
#include "utils/SharedMutex.hpp"
#include <cstdint>
namespace orbis {
@ -74,7 +74,7 @@ struct FileOps {
};
struct File : RcBase {
shared_mutex mtx;
rx::shared_mutex mtx;
Ref<EventEmitter> event;
const FileOps *ops = nullptr;
Ref<RcBase> device;

22
kernel/orbis/include/orbis/ipmi.hpp
View file

@ -3,8 +3,8 @@
#include "KernelAllocator.hpp"
#include "evf.hpp"
#include "orbis-config.hpp"
#include "orbis/utils/SharedCV.hpp"
#include "orbis/utils/SharedMutex.hpp"
#include "rx/SharedCV.hpp"
#include "rx/SharedMutex.hpp"
#include "utils/Rc.hpp"
#include <list>
#include <optional>
@ -43,8 +43,8 @@ struct IpmiServer : RcBase {
ptr<void> serverImpl;
ptr<void> eventHandler;
ptr<void> userData;
shared_mutex mutex;
shared_cv receiveCv;
rx::shared_mutex mutex;
rx::shared_cv receiveCv;
sint pid;
kdeque<Packet> packets;
std::list<ConnectionRequest, kallocator<ConnectionRequest>>
@ -55,7 +55,7 @@ struct IpmiServer : RcBase {
struct IpmiClient : RcBase {
struct MessageQueue {
shared_cv messageCv;
rx::shared_cv messageCv;
kdeque<kvector<std::byte>> messages;
};
@ -69,10 +69,10 @@ struct IpmiClient : RcBase {
ptr<void> clientImpl;
ptr<void> userData;
Ref<IpmiSession> session;
shared_mutex mutex;
shared_cv sessionCv;
shared_cv asyncResponseCv;
shared_cv connectCv;
rx::shared_mutex mutex;
rx::shared_cv sessionCv;
rx::shared_cv asyncResponseCv;
rx::shared_cv connectCv;
std::optional<sint> connectionStatus{};
Process *process;
kdeque<MessageQueue> messageQueues;
@ -93,8 +93,8 @@ struct IpmiSession : RcBase {
ptr<void> userData;
Ref<IpmiClient> client;
Ref<IpmiServer> server;
shared_mutex mutex;
shared_cv responseCv;
rx::shared_mutex mutex;
rx::shared_cv responseCv;
kdeque<SyncResponse> syncResponses;
uint expectedOutput{0};
};

6
kernel/orbis/include/orbis/note.hpp
View file

@ -3,7 +3,7 @@
#include "KernelAllocator.hpp"
#include "orbis-config.hpp"
#include "orbis/utils/Rc.hpp"
#include "utils/SharedMutex.hpp"
#include "rx/SharedMutex.hpp"
#include <limits>
#include <set>
@ -75,7 +75,7 @@ struct KEvent {
struct EventEmitter;
struct KQueue;
struct KNote {
shared_mutex mutex;
rx::shared_mutex mutex;
KQueue *queue;
Ref<File> file;
KEvent event{};
@ -88,7 +88,7 @@ struct KNote {
};
struct EventEmitter : orbis::RcBase {
shared_mutex mutex;
rx::shared_mutex mutex;
std::set<KNote *, std::less<>, kallocator<KNote *>> notes;
void emit(sshort filter, uint fflags = 0, intptr_t data = 0,

11
kernel/orbis/include/orbis/osem.hpp
View file

@ -1,11 +1,10 @@
#pragma once
#include "KernelAllocator.hpp"
#include "thread/Thread.hpp"
#include "utils/SharedCV.hpp"
#include "utils/SharedMutex.hpp"
#include "orbis-config.hpp"
#include "rx/SharedCV.hpp"
#include "rx/SharedMutex.hpp"
#include <atomic>
#include <condition_variable>
namespace orbis {
enum {
@ -22,8 +21,8 @@ struct Semaphore final {
std::atomic<unsigned> references{0};
std::atomic<sint> value;
const sint maxValue;
utils::shared_mutex mtx;
utils::shared_cv cond;
rx::shared_mutex mtx;
rx::shared_cv cond;
Semaphore(uint attrs, sint value, sint max)
: attrs(attrs), value(value), maxValue(max) {}

6
kernel/orbis/include/orbis/pipe.hpp
View file

@ -2,14 +2,14 @@
#include "KernelAllocator.hpp"
#include "file.hpp"
#include "rx/SharedCV.hpp"
#include "rx/SharedMutex.hpp"
#include "utils/Rc.hpp"
#include "utils/SharedCV.hpp"
#include "utils/SharedMutex.hpp"
#include <utility>
namespace orbis {
struct Pipe final : File {
shared_cv cv;
rx::shared_cv cv;
kvector<std::byte> data;
Ref<Pipe> other;
};

8
kernel/orbis/include/orbis/thread/Process.hpp
View file

@ -15,7 +15,7 @@
#include "orbis/file.hpp"
#include "orbis/module/Module.hpp"
#include "orbis/utils/IdMap.hpp"
#include "orbis/utils/SharedMutex.hpp"
#include "rx/SharedMutex.hpp"
#include <optional>
namespace orbis {
@ -59,7 +59,7 @@ struct Process final {
sysentvec *sysent = nullptr;
ProcessState state = ProcessState::NEW;
Process *parentProcess = nullptr;
shared_mutex mtx;
rx::shared_mutex mtx;
int vmId = -1;
ProcessType type = ProcessType::FreeBsd;
void (*onSysEnter)(Thread *thread, int id, uint64_t *args,
@ -92,14 +92,14 @@ struct Process final {
utils::RcIdMap<orbis::File, sint> fileDescriptors;
// Named objects for debugging
utils::shared_mutex namedObjMutex;
rx::shared_mutex namedObjMutex;
utils::kmap<void *, utils::kstring> namedObjNames;
utils::OwningIdMap<NamedObjInfo, uint, 65535, 1> namedObjIds;
utils::kmap<std::int32_t, SigAction> sigActions;
// Named memory ranges for debugging
utils::shared_mutex namedMemMutex;
rx::shared_mutex namedMemMutex;
utils::kmap<NamedMemoryRange, utils::kstring> namedMem;
};
} // namespace orbis

18
kernel/orbis/include/orbis/thread/Thread.hpp
View file

@ -7,9 +7,9 @@
#include "../KernelAllocator.hpp"
#include "../ucontext.hpp"
#include "../utils/SharedAtomic.hpp"
#include "../utils/SharedCV.hpp"
#include "../utils/SharedMutex.hpp"
#include "rx/SharedAtomic.hpp"
#include "rx/SharedCV.hpp"
#include "rx/SharedMutex.hpp"
#include <thread>
namespace orbis {
@ -17,7 +17,7 @@ struct Process;
static constexpr std::uint32_t kThreadSuspendFlag = 1 << 31;
struct Thread {
utils::shared_mutex mtx;
rx::shared_mutex mtx;
Process *tproc = nullptr;
uint64_t retval[2]{};
void *context{};
@ -34,14 +34,14 @@ struct Thread {
.type = 2,
.prio = 10,
};
utils::shared_mutex suspend_mtx;
utils::shared_cv suspend_cv;
rx::shared_mutex suspend_mtx;
rx::shared_cv suspend_cv;
kvector<UContext> sigReturns;
kvector<SigInfo> blockedSignals;
kvector<SigInfo> queuedSignals;
shared_atomic32 suspendFlags{0};
rx::shared_atomic32 suspendFlags{0};
utils::shared_atomic32 interruptedMtx{0};
rx::shared_atomic32 interruptedMtx{0};
std::int64_t hostTid = -1;
lwpid_t tid = -1;
@ -51,7 +51,7 @@ struct Thread {
std::thread::native_handle_type nativeHandle;
// Used to wake up thread in sleep queue
utils::shared_cv sync_cv;
rx::shared_cv sync_cv;
uint64_t evfResultPattern;
uint64_t evfIsCancelled;

4
kernel/orbis/include/orbis/utils/IdMap.hpp
View file

@ -2,7 +2,7 @@
#include "BitSet.hpp"
#include "Rc.hpp"
#include "orbis/utils/SharedMutex.hpp"
#include "rx/SharedMutex.hpp"
#include <algorithm>
#include <bit>
@ -66,7 +66,7 @@ class RcIdMap {
public:
static constexpr auto npos = static_cast<IdT>(~static_cast<std::size_t>(0));
mutable shared_mutex mutex;
mutable rx::shared_mutex mutex;
struct end_iterator {};

167
kernel/orbis/include/orbis/utils/SharedAtomic.hpp
View file

@ -1,167 +0,0 @@
#pragma once
#include <atomic>
#include <chrono>
#include <cstdint>
#include <functional>
#include <limits>
#include <system_error>
#include <thread>
#include <type_traits>
namespace orbis {
inline void yield() { std::this_thread::yield(); }
inline void relax() {
#if defined(__GNUC__) && (defined __i386__ || defined __x86_64__)
__builtin_ia32_pause();
#else
yield();
#endif
}
static constexpr auto kRelaxSpinCount = 12;
static constexpr auto kSpinCount = 16;
inline namespace utils {
inline thread_local bool (*g_scopedUnblock)(bool) = nullptr;
bool try_spin_wait(auto &&pred) {
for (std::size_t i = 0; i < kSpinCount; ++i) {
if (pred()) {
return true;
}
if (i < kRelaxSpinCount) {
relax();
} else {
yield();
}
}
return false;
}
bool spin_wait(auto &&pred, auto &&spinCond) {
if (try_spin_wait(pred)) {
return true;
}
while (spinCond()) {
if (pred()) {
return true;
}
}
return false;
}
struct shared_atomic32 : std::atomic<std::uint32_t> {
using atomic::atomic;
using atomic::operator=;
template <typename Clock, typename Dur>
std::errc wait(std::uint32_t oldValue,
std::chrono::time_point<Clock, Dur> timeout) {
if (try_spin_wait(
[&] { return load(std::memory_order::acquire) != oldValue; })) {
return {};
}
auto now = Clock::now();
if (timeout < now) {
return std::errc::timed_out;
}
return wait_impl(
oldValue,
std::chrono::duration_cast<std::chrono::microseconds>(timeout - now));
}
std::errc wait(std::uint32_t oldValue,
std::chrono::microseconds usec_timeout) {
return wait_impl(oldValue, usec_timeout);
}
std::errc wait(std::uint32_t oldValue) {
if (try_spin_wait(
[&] { return load(std::memory_order::acquire) != oldValue; })) {
return {};
}
return wait_impl(oldValue);
}
auto wait(auto &fn) -> decltype(fn(std::declval<std::uint32_t &>())) {
while (true) {
std::uint32_t lastValue;
if (try_spin_wait([&] {
lastValue = load(std::memory_order::acquire);
return fn(lastValue);
})) {
return;
}
while (wait_impl(lastValue) != std::errc{}) {
}
}
}
int notify_one() const { return notify_n(1); }
int notify_all() const { return notify_n(std::numeric_limits<int>::max()); }
int notify_n(int count) const;
// Atomic operation; returns old value, or pair of old value and return value
// (cancel op if evaluates to false)
template <typename F, typename RT = std::invoke_result_t<F, std::uint32_t &>>
std::conditional_t<std::is_void_v<RT>, std::uint32_t,
std::pair<std::uint32_t, RT>>
fetch_op(F &&func) {
std::uint32_t _new;
std::uint32_t old = load(std::memory_order::relaxed);
while (true) {
_new = old;
if constexpr (std::is_void_v<RT>) {
std::invoke(std::forward<F>(func), _new);
if (compare_exchange_strong(old, _new)) [[likely]] {
return old;
}
} else {
RT ret = std::invoke(std::forward<F>(func), _new);
if (!ret || compare_exchange_strong(old, _new)) [[likely]] {
return {old, std::move(ret)};
}
}
}
}
// Atomic operation; returns function result value
template <typename F, typename RT = std::invoke_result_t<F, std::uint32_t &>>
RT op(F &&func) {
std::uint32_t _new;
std::uint32_t old = load(std::memory_order::relaxed);
while (true) {
_new = old;
if constexpr (std::is_void_v<RT>) {
std::invoke(std::forward<F>(func), _new);
if (compare_exchange_strong(old, _new)) [[likely]] {
return;
}
} else {
RT result = std::invoke(std::forward<F>(func), _new);
if (compare_exchange_strong(old, _new)) [[likely]] {
return result;
}
}
}
}
private:
[[nodiscard]] std::errc wait_impl(std::uint32_t oldValue,
std::chrono::microseconds usec_timeout =
std::chrono::microseconds::max());
};
} // namespace utils
} // namespace orbis

92
kernel/orbis/include/orbis/utils/SharedCV.hpp
View file

@ -1,92 +0,0 @@
#pragma once
#include "orbis/utils/SharedAtomic.hpp"
#include <chrono>
#include <cstdint>
#include <mutex>
#include <orbis/utils/AtomicOp.hpp>
#include <orbis/utils/SharedMutex.hpp>
#include <system_error>
namespace orbis {
inline namespace utils {
// IPC-ready lightweight condition variable
class shared_cv final {
enum : unsigned {
c_waiter_mask = 0xffff,
c_signal_mask = 0x7fff0000,
#ifdef ORBIS_HAS_FUTEX
c_locked_mask = 0x80000000,
#endif
c_signal_one = c_waiter_mask + 1,
};
shared_atomic32 m_value{0};
protected:
// Increment waiter count
unsigned add_waiter() noexcept {
return m_value.op([](unsigned &value) -> unsigned {
if ((value & c_signal_mask) == c_signal_mask ||
(value & c_waiter_mask) == c_waiter_mask) {
// Signal or waiter overflow, return immediately
return 0;
}
// Add waiter (c_waiter_mask)
value += 1;
return value;
});
}
// Internal waiting function
std::errc impl_wait(shared_mutex &mutex, unsigned _val,
std::uint64_t usec_timeout) noexcept;
// Try to notify up to _count threads
void impl_wake(shared_mutex &mutex, int _count) noexcept;
public:
constexpr shared_cv() = default;
std::errc
wait(std::unique_lock<shared_mutex> &lock,
std::chrono::microseconds timeout = std::chrono::microseconds::max()) {
return wait(*lock.mutex(), timeout.count());
}
template <typename Rep, typename Period>
std::errc wait(std::unique_lock<shared_mutex> &lock,
std::chrono::duration<Rep, Period> timeout) {
return wait(
lock,
std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
}
std::errc wait(shared_mutex &mutex,
std::uint64_t usec_timeout = -1) noexcept {
const unsigned _val = add_waiter();
if (!_val) {
return {};
}
mutex.unlock();
return impl_wait(mutex, _val, usec_timeout);
}
// Wake one thread
void notify_one(shared_mutex &mutex) noexcept {
if (m_value) {
impl_wake(mutex, 1);
}
}
// Wake all threads
void notify_all(shared_mutex &mutex) noexcept {
if (m_value) {
impl_wake(mutex, INT_MAX);
}
}
};
} // namespace utils
} // namespace orbis

151
kernel/orbis/include/orbis/utils/SharedMutex.hpp
View file

@ -1,151 +0,0 @@
#pragma once
#include <atomic>
#include <orbis/utils/AtomicOp.hpp>
#include <orbis/utils/SharedAtomic.hpp>
#include <system_error>
namespace orbis {
inline namespace utils {
// IPC-ready shared mutex, using only writer lock is recommended
class shared_mutex final {
friend class shared_cv;
enum : unsigned {
c_one = 1u << 14, // Fixed-point 1.0 value (one writer)
c_sig = 1u << 30,
c_err = 1u << 31,
};
shared_atomic32 m_value{};
void impl_lock_shared(unsigned val);
void impl_unlock_shared(unsigned old);
std::errc impl_wait();
void impl_signal();
void impl_lock(unsigned val);
void impl_unlock(unsigned old);
void impl_lock_upgrade();
public:
constexpr shared_mutex() = default;
bool try_lock_shared() {
// Conditional increment
unsigned value = m_value.load();
return value < c_one - 1 &&
m_value.compare_exchange_strong(value, value + 1);
}
// Lock with HLE acquire hint
void lock_shared() {
unsigned value = m_value.load();
if (value < c_one - 1) [[likely]] {
unsigned old = value;
if (compare_exchange_hle_acq(m_value, old, value + 1)) [[likely]] {
return;
}
}
impl_lock_shared(value + 1);
}
// Unlock with HLE release hint
void unlock_shared() {
const unsigned value = fetch_add_hle_rel(m_value, -1u);
if (value >= c_one) [[unlikely]] {
impl_unlock_shared(value);
}
}
bool try_lock() {
unsigned value = 0;
return m_value.compare_exchange_strong(value, c_one);
}
// Lock with HLE acquire hint
void lock() {
unsigned value = 0;
if (!compare_exchange_hle_acq(m_value, value, +c_one)) [[unlikely]] {
impl_lock(value);
}
}
// Unlock with HLE release hint
void unlock() {
const unsigned value = fetch_add_hle_rel(m_value, 0u - c_one);
if (value != c_one) [[unlikely]] {
impl_unlock(value);
}
}
bool try_lock_upgrade() {
unsigned value = m_value.load();
// Conditional increment, try to convert a single reader into a writer,
// ignoring other writers
return (value + c_one - 1) % c_one == 0 &&
m_value.compare_exchange_strong(value, value + c_one - 1);
}
void lock_upgrade() {
if (!try_lock_upgrade()) [[unlikely]] {
impl_lock_upgrade();
}
}
void lock_downgrade() {
// Convert to reader lock (can result in broken state)
m_value -= c_one - 1;
}
// Check whether can immediately obtain an exclusive (writer) lock
[[nodiscard]] bool is_free() const { return m_value.load() == 0; }
// Check whether can immediately obtain a shared (reader) lock
[[nodiscard]] bool is_lockable() const { return m_value.load() < c_one - 1; }
private:
// For CV
bool lock_forced(int count = 1);
};
// Simplified shared (reader) lock implementation.
class reader_lock final {
shared_mutex &m_mutex;
bool m_upgraded = false;
public:
reader_lock(const reader_lock &) = delete;
reader_lock &operator=(const reader_lock &) = delete;
explicit reader_lock(shared_mutex &mutex) : m_mutex(mutex) {
m_mutex.lock_shared();
}
// One-way lock upgrade; note that the observed state could have been changed
void upgrade() {
if (!m_upgraded) {
m_mutex.lock_upgrade();
m_upgraded = true;
}
}
// Try to upgrade; if it succeeds, the observed state has NOT been changed
bool try_upgrade() {
return m_upgraded || (m_upgraded = m_mutex.try_lock_upgrade());
}
~reader_lock() { m_upgraded ? m_mutex.unlock() : m_mutex.unlock_shared(); }
};
class writer_lock final {
shared_mutex &m_mutex;
public:
writer_lock(const writer_lock &) = delete;
writer_lock &operator=(const writer_lock &) = delete;
explicit writer_lock(shared_mutex &mutex) : m_mutex(mutex) { m_mutex.lock(); }
~writer_lock() { m_mutex.unlock(); }
};
} // namespace utils
} // namespace orbis

8
kernel/orbis/src/KernelContext.cpp
View file

@ -2,7 +2,7 @@
#include "orbis/thread/Process.hpp"
#include "orbis/thread/ProcessOps.hpp"
#include "orbis/utils/Logs.hpp"
#include "utils/SharedAtomic.hpp"
#include "rx/SharedAtomic.hpp"
#include <bit>
#include <chrono>
#include <csignal>
@ -283,7 +283,7 @@ void KernelContext::kfree(void *ptr, std::size_t size) {
}
}
std::tuple<UmtxChain &, UmtxKey, std::unique_lock<shared_mutex>>
std::tuple<UmtxChain &, UmtxKey, std::unique_lock<rx::shared_mutex>>
KernelContext::getUmtxChainIndexed(int i, Thread *t, uint32_t flags,
void *ptr) {
auto pid = t->tproc->pid;
@ -406,7 +406,7 @@ bool Thread::block() {
scoped_unblock::scoped_unblock() {
if (g_currentThread && g_currentThread->context) {
g_scopedUnblock = [](bool unblock) {
rx::g_scopedUnblock = [](bool unblock) {
if (unblock) {
return g_currentThread->unblock();
}
@ -416,5 +416,5 @@ scoped_unblock::scoped_unblock() {
}
}
scoped_unblock::~scoped_unblock() { g_scopedUnblock = nullptr; }
scoped_unblock::~scoped_unblock() { rx::g_scopedUnblock = nullptr; }
} // namespace orbis

7
kernel/orbis/src/evf.cpp
View file

@ -1,7 +1,6 @@
#include "evf.hpp"
#include "error/ErrorCode.hpp"
#include "utils/Logs.hpp"
#include "utils/SharedCV.hpp"
#include "rx/SharedCV.hpp"
#include <atomic>
orbis::ErrorCode orbis::EventFlag::wait(Thread *thread, std::uint8_t waitMode,
@ -99,7 +98,7 @@ orbis::ErrorCode orbis::EventFlag::wait(Thread *thread, std::uint8_t waitMode,
orbis::ErrorCode orbis::EventFlag::tryWait(Thread *thread,
std::uint8_t waitMode,
std::uint64_t bitPattern) {
writer_lock lock(queueMtx);
rx::writer_lock lock(queueMtx);
if (isDeleted) {
return ErrorCode::ACCES;
@ -120,7 +119,7 @@ orbis::ErrorCode orbis::EventFlag::tryWait(Thread *thread,
}
std::size_t orbis::EventFlag::notify(NotifyType type, std::uint64_t bits) {
writer_lock lock(queueMtx);
rx::writer_lock lock(queueMtx);
auto patValue = value.load(std::memory_order::relaxed);
if (type == NotifyType::Destroy) {

2
kernel/orbis/src/sys/sys_event.cpp
View file

@ -78,7 +78,7 @@ static SysResult keventChange(KQueue *kq, KEvent &change, Thread *thread) {
nodeIt = kq->notes.end();
}
std::unique_lock<shared_mutex> noteLock;
std::unique_lock<rx::shared_mutex> noteLock;
if (change.flags & kEvAdd) {
if (nodeIt == kq->notes.end()) {
auto &note = kq->notes.emplace_front();

118
kernel/orbis/src/utils/SharedAtomic.cpp
View file

@ -1,118 +0,0 @@
#include "utils/SharedAtomic.hpp"
using namespace orbis;
#ifdef ORBIS_HAS_FUTEX
#include <linux/futex.h>
std::errc shared_atomic32::wait_impl(std::uint32_t oldValue,
std::chrono::microseconds usec_timeout) {
auto usec_timeout_count = usec_timeout.count();
struct timespec timeout{};
bool useTimeout = usec_timeout != std::chrono::microseconds::max();
if (useTimeout) {
timeout.tv_nsec = (usec_timeout_count % 1000'000) * 1000;
timeout.tv_sec = (usec_timeout_count / 1000'000);
}
bool unblock = (!useTimeout || usec_timeout.count() > 1000) &&
g_scopedUnblock != nullptr;
if (unblock) {
if (!g_scopedUnblock(true)) {
return std::errc::interrupted;
}
}
int result = syscall(SYS_futex, this, FUTEX_WAIT, oldValue,
useTimeout ? &timeout : nullptr);
auto errorCode = result < 0 ? static_cast<std::errc>(errno) : std::errc{};
if (unblock) {
if (!g_scopedUnblock(false)) {
if (result < 0) {
return std::errc::interrupted;
}
return {};
}
}
if (result < 0) {
return errorCode;
}
return {};
}
int shared_atomic32::notify_n(int count) const {
return syscall(SYS_futex, this, FUTEX_WAKE, count);
}
#elif defined(ORBIS_HAS_ULOCK)
#include <limits>
#define UL_COMPARE_AND_WAIT 1
#define UL_UNFAIR_LOCK 2
#define UL_COMPARE_AND_WAIT_SHARED 3
#define UL_UNFAIR_LOCK64_SHARED 4
#define UL_COMPARE_AND_WAIT64 5
#define UL_COMPARE_AND_WAIT64_SHARED 6
#define ULF_WAKE_ALL 0x00000100
#define ULF_WAKE_THREAD 0x00000200
#define ULF_WAKE_ALLOW_NON_OWNER 0x00000400
#define ULF_WAIT_WORKQ_DATA_CONTENTION 0x00010000
#define ULF_WAIT_CANCEL_POINT 0x00020000
#define ULF_WAIT_ADAPTIVE_SPIN 0x00040000
#define ULF_NO_ERRNO 0x01000000
#define UL_OPCODE_MASK 0x000000FF
#define UL_FLAGS_MASK 0xFFFFFF00
#define ULF_GENERIC_MASK 0xFFFF0000
extern int __ulock_wait(uint32_t operation, void *addr, uint64_t value,
uint32_t timeout);
extern int __ulock_wake(uint32_t operation, void *addr, uint64_t wake_value);
std::errc shared_atomic32::wait_impl(std::uint32_t oldValue,
std::chrono::microseconds usec_timeout) {
int result = __ulock_wait(UL_COMPARE_AND_WAIT_SHARED, (void *)this, oldValue,
usec_timeout.count());
if (result < 0) {
return static_cast<std::errc>(errno);
}
return {};
}
int shared_atomic32::notify_n(int count) const {
int result = 0;
uint32_t operation = UL_COMPARE_AND_WAIT_SHARED | ULF_NO_ERRNO;
if (count == 1) {
result = __ulock_wake(operation, (void *)this, 0);
} else if (count == std::numeric_limits<int>::max()) {
result = __ulock_wake(ULF_WAKE_ALL | operation, (void *)this, 0);
} else {
for (int i = 0; i < count; ++i) {
auto ret = __ulock_wake(operation, (void *)this, 0);
if (ret != 0) {
if (result == 0) {
result = ret;
}
break;
}
result++;
}
}
return result;
}
#else
#error Unimplemented atomic for this platform
#endif

158
kernel/orbis/src/utils/SharedCV.cpp
View file

@ -1,158 +0,0 @@
#include "orbis/utils/SharedCV.hpp"
#include <chrono>
#ifdef ORBIS_HAS_FUTEX
#include <linux/futex.h>
#include <syscall.h>
#include <unistd.h>
#endif
namespace orbis::utils {
std::errc shared_cv::impl_wait(shared_mutex &mutex, unsigned _val,
std::uint64_t usec_timeout) noexcept {
// Not supposed to fail
if (!_val) {
std::abort();
}
std::errc result = {};
bool useTimeout = usec_timeout != static_cast<std::uint64_t>(-1);
while (true) {
result =
m_value.wait(_val, useTimeout ? std::chrono::microseconds(usec_timeout)
: std::chrono::microseconds::max());
bool spurious = result == std::errc::resource_unavailable_try_again;
// Cleanup
const auto old = m_value.fetch_op([&](unsigned &value) {
// Remove waiter if no signals
if ((value & ~c_waiter_mask) == 0) {
if (!spurious) {
value -= 1;
}
}
// Try to remove signal
if (value & c_signal_mask) {
value -= c_signal_one;
}
#ifdef ORBIS_HAS_FUTEX
if (value & c_locked_mask) {
value -= c_locked_mask;
}
#endif
});
#ifdef ORBIS_HAS_FUTEX
// Lock is already acquired
if (old & c_locked_mask) {
return {};
}
// Wait directly (waiter has been added)
if (old & c_signal_mask) {
return mutex.impl_wait();
}
#else
if (old & c_signal_mask) {
result = {};
break;
}
#endif
// Possibly spurious wakeup
if (!spurious) {
break;
}
_val = old;
}
mutex.lock();
return result;
}
void shared_cv::impl_wake(shared_mutex &mutex, int _count) noexcept {
#ifdef ORBIS_HAS_FUTEX
while (true) {
unsigned _old = m_value.load();
const bool is_one = _count == 1;
// Enqueue _count waiters
_count = std::min<int>(_count, _old & c_waiter_mask);
if (_count <= 0)
return;
// Try to lock the mutex
const bool locked = mutex.lock_forced(_count);
const int max_sig = m_value.op([&](unsigned &value) {
// Verify the number of waiters
int max_sig = std::min<int>(_count, value & c_waiter_mask);
// Add lock signal (mutex was immediately locked)
if (locked && max_sig)
value |= c_locked_mask;
else if (locked)
std::abort();
// Add normal signals
value += c_signal_one * max_sig;
// Remove waiters
value -= max_sig;
_old = value;
return max_sig;
});
if (max_sig < _count) {
// Fixup mutex
mutex.lock_forced(max_sig - _count);
_count = max_sig;
}
if (_count) {
// Wake up one thread + requeue remaining waiters
unsigned awake_count = locked ? 1 : 0;
if (auto r = syscall(SYS_futex, &m_value, FUTEX_REQUEUE, awake_count,
_count - awake_count, &mutex, 0);
r < _count) {
// Keep awaking waiters
_count = is_one ? 1 : INT_MAX;
continue;
}
}
break;
}
#else
unsigned _old = m_value.load();
_count = std::min<int>(_count, _old & c_waiter_mask);
if (_count <= 0)
return;
mutex.lock_forced(1);
const int wakeupWaiters = m_value.op([&](unsigned &value) {
int max_sig = std::min<int>(_count, value & c_waiter_mask);
// Add normal signals
value += c_signal_one * max_sig;
// Remove waiters
value -= max_sig;
_old = value;
return max_sig;
});
if (wakeupWaiters > 0) {
m_value.notify_n(wakeupWaiters);
}
mutex.unlock();
#endif
}
} // namespace orbis::utils

182
kernel/orbis/src/utils/SharedMutex.cpp
View file

@ -1,182 +0,0 @@
#include "utils/SharedMutex.hpp"
#include "utils/Logs.hpp"
#include <syscall.h>
#include <unistd.h>
#include <xmmintrin.h>
static void busy_wait(unsigned long long cycles = 3000) {
const auto stop = __builtin_ia32_rdtsc() + cycles;
do
_mm_pause();
while (__builtin_ia32_rdtsc() < stop);
}
namespace orbis::utils {
void shared_mutex::impl_lock_shared(unsigned val) {
if (val >= c_err)
std::abort(); // "shared_mutex underflow"
// Try to steal the notification bit
unsigned _old = val;
if (val & c_sig && m_value.compare_exchange_strong(_old, val - c_sig + 1)) {
return;
}
for (int i = 0; i < 10; i++) {
if (try_lock_shared()) {
return;
}
unsigned old = m_value;
if (old & c_sig && m_value.compare_exchange_strong(old, old - c_sig + 1)) {
return;
}
busy_wait();
}
// Acquire writer lock and downgrade
const unsigned old = m_value.fetch_add(c_one);
if (old == 0) {
lock_downgrade();
return;
}
if ((old % c_sig) + c_one >= c_sig)
std::abort(); // "shared_mutex overflow"
while (impl_wait() != std::errc{}) {
}
lock_downgrade();
}
void shared_mutex::impl_unlock_shared(unsigned old) {
if (old - 1 >= c_err)
std::abort(); // "shared_mutex underflow"
// Check reader count, notify the writer if necessary
if ((old - 1) % c_one == 0) {
impl_signal();
}
}
std::errc shared_mutex::impl_wait() {
while (true) {
const auto [old, ok] = m_value.fetch_op([](unsigned &value) {
if (value >= c_sig) {
value -= c_sig;
return true;
}
return false;
});
if (ok) {
break;
}
auto result = m_value.wait(old);
if (result == std::errc::interrupted) {
return result;
}
}
return {};
}
void shared_mutex::impl_signal() {
m_value += c_sig;
m_value.notify_one();
}
void shared_mutex::impl_lock(unsigned val) {
if (val >= c_err)
std::abort(); // "shared_mutex underflow"
// Try to steal the notification bit
unsigned _old = val;
if (val & c_sig &&
m_value.compare_exchange_strong(_old, val - c_sig + c_one)) {
return;
}
for (int i = 0; i < 10; i++) {
busy_wait();
unsigned old = m_value;
if (!old && try_lock()) {
return;
}
if (old & c_sig &&
m_value.compare_exchange_strong(old, old - c_sig + c_one)) {
return;
}
}
const unsigned old = m_value.fetch_add(c_one);
if (old == 0) {
return;
}
if ((old % c_sig) + c_one >= c_sig)
std::abort(); // "shared_mutex overflow"
while (impl_wait() != std::errc{}) {
}
}
void shared_mutex::impl_unlock(unsigned old) {
if (old - c_one >= c_err)
std::abort(); // "shared_mutex underflow"
// 1) Notify the next writer if necessary
// 2) Notify all readers otherwise if necessary (currently indistinguishable
// from writers)
if (old - c_one) {
impl_signal();
}
}
void shared_mutex::impl_lock_upgrade() {
for (int i = 0; i < 10; i++) {
busy_wait();
if (try_lock_upgrade()) {
return;
}
}
// Convert to writer lock
const unsigned old = m_value.fetch_add(c_one - 1);
if ((old % c_sig) + c_one - 1 >= c_sig)
std::abort(); // "shared_mutex overflow"
if (old % c_one == 1) {
return;
}
while (impl_wait() != std::errc{}) {
}
}
bool shared_mutex::lock_forced(int count) {
if (count == 0)
return false;
if (count > 0) {
// Lock
return m_value.op([&](std::uint32_t &v) {
if (v & c_sig) {
v -= c_sig;
v += c_one * count;
return true;
}
bool firstLock = v == 0;
v += c_one * count;
return firstLock;
});
}
// Remove waiters
m_value.fetch_add(c_one * count);
return true;
}
} // namespace orbis::utils