diff --git a/kernel/orbis/CMakeLists.txt b/kernel/orbis/CMakeLists.txt index c86604d3d..5b5d4c584 100644 --- a/kernel/orbis/CMakeLists.txt +++ b/kernel/orbis/CMakeLists.txt @@ -1,10 +1,5 @@ set(CMAKE_POSITION_INDEPENDENT_CODE on) -add_library(obj.orbis-utils-ipc OBJECT - src/utils/SharedMutex.cpp - src/utils/SharedCV.cpp - src/utils/SharedAtomic.cpp -) add_library(obj.orbis-kernel OBJECT src/module.cpp src/pipe.cpp @@ -72,7 +67,7 @@ add_library(obj.orbis-kernel OBJECT src/utils/Logs.cpp ) -target_link_libraries(obj.orbis-kernel PUBLIC orbis::kernel::config $) +target_link_libraries(obj.orbis-kernel PUBLIC orbis::kernel::config rx) target_include_directories(obj.orbis-kernel PUBLIC @@ -82,27 +77,10 @@ target_include_directories(obj.orbis-kernel ${CMAKE_CURRENT_SOURCE_DIR}/include/orbis ) -target_include_directories(obj.orbis-utils-ipc - PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include - - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/include/orbis -) - -if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") - target_compile_definitions(obj.orbis-utils-ipc PUBLIC ORBIS_HAS_FUTEX) -elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") - target_compile_definitions(obj.orbis-utils-ipc PUBLIC ORBIS_HAS_ULOCK) -endif() - -add_library(orbis-utils-ipc STATIC) add_library(orbis-kernel STATIC) add_library(orbis-kernel-shared SHARED) -add_library(orbis::utils::ipc ALIAS orbis-utils-ipc) add_library(orbis::kernel ALIAS orbis-kernel) add_library(orbis::kernel-shared ALIAS orbis-kernel-shared) -target_link_libraries(orbis-utils-ipc PUBLIC obj.orbis-utils-ipc) target_link_libraries(orbis-kernel PUBLIC obj.orbis-kernel) target_link_libraries(orbis-kernel-shared PUBLIC obj.orbis-kernel) diff --git a/kernel/orbis/include/orbis/Budget.hpp b/kernel/orbis/include/orbis/Budget.hpp index 553453f08..8f661bded 100644 --- a/kernel/orbis/include/orbis/Budget.hpp +++ b/kernel/orbis/include/orbis/Budget.hpp @@ -1,12 +1,13 @@ #pragma once #include "orbis-config.hpp" +#include "rx/SharedMutex.hpp" #include "utils/BitSet.hpp" #include "utils/Rc.hpp" -#include "utils/SharedMutex.hpp" #include #include #include +#include #include namespace orbis { @@ -132,7 +133,7 @@ public: [[nodiscard]] ProcessType processType() const { return mProcessType; } private: - mutable shared_mutex mMtx; + mutable rx::shared_mutex mMtx; orbis::BitSet(BudgetResource::_count)> mUsed; ProcessType mProcessType{}; BudgetList mList; diff --git a/kernel/orbis/include/orbis/KernelContext.hpp b/kernel/orbis/include/orbis/KernelContext.hpp index fcca34885..e95443328 100644 --- a/kernel/orbis/include/orbis/KernelContext.hpp +++ b/kernel/orbis/include/orbis/KernelContext.hpp @@ -6,11 +6,11 @@ #include "ipmi.hpp" #include "orbis/note.hpp" #include "osem.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" #include "thread/types.hpp" #include "utils/IdMap.hpp" #include "utils/LinkedNode.hpp" -#include "utils/SharedCV.hpp" -#include "utils/SharedMutex.hpp" #include #include @@ -32,13 +32,13 @@ struct UmtxKey { struct UmtxCond { Thread *thr; - utils::shared_cv cv; + rx::shared_cv cv; UmtxCond(Thread *thr) : thr(thr) {} }; struct UmtxChain { - utils::shared_mutex mtx; + rx::shared_mutex mtx; using queue_type = utils::kmultimap; queue_type sleep_queue; queue_type spare_queue; @@ -158,7 +158,7 @@ public: } std::tuple &, - std::unique_lock> + std::unique_lock> getKernelEnv() { std::unique_lock lock(m_kenv_mtx); return {m_kenv, std::move(lock)}; @@ -178,7 +178,7 @@ public: }; // Use getUmtxChain0 or getUmtxChain1 - std::tuple> + std::tuple> getUmtxChainIndexed(int i, Thread *t, uint32_t flags, void *ptr); // Internal Umtx: Wait/Cv/Sem @@ -197,7 +197,7 @@ public: Ref blockpoolDevice; Ref gpuDevice; Ref dceDevice; - shared_mutex gpuDeviceMtx; + rx::shared_mutex gpuDeviceMtx; uint sdkVersion{}; uint fwSdkVersion{}; uint safeMode{}; @@ -206,7 +206,7 @@ public: RcIdMap budgets; Ref processTypeBudgets[4]; - shared_mutex regMgrMtx; + rx::shared_mutex regMgrMtx; kmap regMgrInt; std::vector> dialogs{}; @@ -227,8 +227,8 @@ public: } private: - shared_mutex m_heap_mtx; - shared_mutex m_heap_map_mtx; + rx::shared_mutex m_heap_mtx; + rx::shared_mutex m_heap_map_mtx; void *m_heap_next = this + 1; utils::kmultimap m_free_heap; @@ -238,21 +238,21 @@ private: std::atomic m_tsc_freq{0}; - shared_mutex m_thread_id_mtx; + rx::shared_mutex m_thread_id_mtx; OwningIdMap m_thread_id_map; - mutable shared_mutex m_proc_mtx; + mutable rx::shared_mutex m_proc_mtx; utils::LinkedNode *m_processes = nullptr; - shared_mutex m_evf_mtx; + rx::shared_mutex m_evf_mtx; utils::kmap> m_event_flags; - shared_mutex m_sem_mtx; + rx::shared_mutex m_sem_mtx; utils::kmap> m_semaphores; - shared_mutex mIpmiServerMtx; + rx::shared_mutex mIpmiServerMtx; utils::kmap> mIpmiServers; - shared_mutex m_kenv_mtx; + rx::shared_mutex m_kenv_mtx; utils::kmap m_kenv; // max size: 127 + '\0' }; diff --git a/kernel/orbis/include/orbis/event.hpp b/kernel/orbis/include/orbis/event.hpp index 8c993483d..4cc0b8a31 100644 --- a/kernel/orbis/include/orbis/event.hpp +++ b/kernel/orbis/include/orbis/event.hpp @@ -1,12 +1,12 @@ #pragma once #include "file.hpp" #include "note.hpp" -#include "utils/SharedCV.hpp" +#include "rx/SharedCV.hpp" #include namespace orbis { struct KQueue : orbis::File { - shared_cv cv; + rx::shared_cv cv; kstring name; kvector triggeredEvents; std::list> notes; diff --git a/kernel/orbis/include/orbis/evf.hpp b/kernel/orbis/include/orbis/evf.hpp index 9e265b087..bdc23ca52 100644 --- a/kernel/orbis/include/orbis/evf.hpp +++ b/kernel/orbis/include/orbis/evf.hpp @@ -1,7 +1,7 @@ #pragma once #include "KernelAllocator.hpp" #include "thread/Thread.hpp" -#include "utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include namespace orbis { @@ -56,7 +56,7 @@ struct EventFlag final { } }; - utils::shared_mutex queueMtx; + rx::shared_mutex queueMtx; utils::kvector waitingThreads; enum class NotifyType { Set, Cancel, Destroy }; @@ -80,7 +80,7 @@ struct EventFlag final { std::size_t set(std::uint64_t bits) { return notify(NotifyType::Set, bits); } void clear(std::uint64_t bits) { - writer_lock lock(queueMtx); + rx::writer_lock lock(queueMtx); value.fetch_and(bits, std::memory_order::relaxed); } diff --git a/kernel/orbis/include/orbis/file.hpp b/kernel/orbis/include/orbis/file.hpp index 4949866bb..39ec19882 100644 --- a/kernel/orbis/include/orbis/file.hpp +++ b/kernel/orbis/include/orbis/file.hpp @@ -3,9 +3,9 @@ #include "KernelAllocator.hpp" #include "error/ErrorCode.hpp" #include "note.hpp" +#include "rx/SharedMutex.hpp" #include "stat.hpp" #include "utils/Rc.hpp" -#include "utils/SharedMutex.hpp" #include namespace orbis { @@ -74,7 +74,7 @@ struct FileOps { }; struct File : RcBase { - shared_mutex mtx; + rx::shared_mutex mtx; Ref event; const FileOps *ops = nullptr; Ref device; diff --git a/kernel/orbis/include/orbis/ipmi.hpp b/kernel/orbis/include/orbis/ipmi.hpp index 101e8983c..6411630fc 100644 --- a/kernel/orbis/include/orbis/ipmi.hpp +++ b/kernel/orbis/include/orbis/ipmi.hpp @@ -3,8 +3,8 @@ #include "KernelAllocator.hpp" #include "evf.hpp" #include "orbis-config.hpp" -#include "orbis/utils/SharedCV.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" #include "utils/Rc.hpp" #include #include @@ -43,8 +43,8 @@ struct IpmiServer : RcBase { ptr serverImpl; ptr eventHandler; ptr userData; - shared_mutex mutex; - shared_cv receiveCv; + rx::shared_mutex mutex; + rx::shared_cv receiveCv; sint pid; kdeque packets; std::list> @@ -55,7 +55,7 @@ struct IpmiServer : RcBase { struct IpmiClient : RcBase { struct MessageQueue { - shared_cv messageCv; + rx::shared_cv messageCv; kdeque> messages; }; @@ -69,10 +69,10 @@ struct IpmiClient : RcBase { ptr clientImpl; ptr userData; Ref session; - shared_mutex mutex; - shared_cv sessionCv; - shared_cv asyncResponseCv; - shared_cv connectCv; + rx::shared_mutex mutex; + rx::shared_cv sessionCv; + rx::shared_cv asyncResponseCv; + rx::shared_cv connectCv; std::optional connectionStatus{}; Process *process; kdeque messageQueues; @@ -93,8 +93,8 @@ struct IpmiSession : RcBase { ptr userData; Ref client; Ref server; - shared_mutex mutex; - shared_cv responseCv; + rx::shared_mutex mutex; + rx::shared_cv responseCv; kdeque syncResponses; uint expectedOutput{0}; }; diff --git a/kernel/orbis/include/orbis/note.hpp b/kernel/orbis/include/orbis/note.hpp index 5a8a37a8a..01612f535 100644 --- a/kernel/orbis/include/orbis/note.hpp +++ b/kernel/orbis/include/orbis/note.hpp @@ -3,7 +3,7 @@ #include "KernelAllocator.hpp" #include "orbis-config.hpp" #include "orbis/utils/Rc.hpp" -#include "utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include #include @@ -75,7 +75,7 @@ struct KEvent { struct EventEmitter; struct KQueue; struct KNote { - shared_mutex mutex; + rx::shared_mutex mutex; KQueue *queue; Ref file; KEvent event{}; @@ -88,7 +88,7 @@ struct KNote { }; struct EventEmitter : orbis::RcBase { - shared_mutex mutex; + rx::shared_mutex mutex; std::set, kallocator> notes; void emit(sshort filter, uint fflags = 0, intptr_t data = 0, diff --git a/kernel/orbis/include/orbis/osem.hpp b/kernel/orbis/include/orbis/osem.hpp index ab68af84b..7eb6be061 100644 --- a/kernel/orbis/include/orbis/osem.hpp +++ b/kernel/orbis/include/orbis/osem.hpp @@ -1,11 +1,10 @@ #pragma once #include "KernelAllocator.hpp" -#include "thread/Thread.hpp" -#include "utils/SharedCV.hpp" -#include "utils/SharedMutex.hpp" +#include "orbis-config.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" #include -#include namespace orbis { enum { @@ -22,8 +21,8 @@ struct Semaphore final { std::atomic references{0}; std::atomic value; const sint maxValue; - utils::shared_mutex mtx; - utils::shared_cv cond; + rx::shared_mutex mtx; + rx::shared_cv cond; Semaphore(uint attrs, sint value, sint max) : attrs(attrs), value(value), maxValue(max) {} diff --git a/kernel/orbis/include/orbis/pipe.hpp b/kernel/orbis/include/orbis/pipe.hpp index 50ae1077c..0043da543 100644 --- a/kernel/orbis/include/orbis/pipe.hpp +++ b/kernel/orbis/include/orbis/pipe.hpp @@ -2,14 +2,14 @@ #include "KernelAllocator.hpp" #include "file.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" #include "utils/Rc.hpp" -#include "utils/SharedCV.hpp" -#include "utils/SharedMutex.hpp" #include namespace orbis { struct Pipe final : File { - shared_cv cv; + rx::shared_cv cv; kvector data; Ref other; }; diff --git a/kernel/orbis/include/orbis/thread/Process.hpp b/kernel/orbis/include/orbis/thread/Process.hpp index 2068cfb2f..a218359c8 100644 --- a/kernel/orbis/include/orbis/thread/Process.hpp +++ b/kernel/orbis/include/orbis/thread/Process.hpp @@ -15,7 +15,7 @@ #include "orbis/file.hpp" #include "orbis/module/Module.hpp" #include "orbis/utils/IdMap.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include namespace orbis { @@ -59,7 +59,7 @@ struct Process final { sysentvec *sysent = nullptr; ProcessState state = ProcessState::NEW; Process *parentProcess = nullptr; - shared_mutex mtx; + rx::shared_mutex mtx; int vmId = -1; ProcessType type = ProcessType::FreeBsd; void (*onSysEnter)(Thread *thread, int id, uint64_t *args, @@ -92,14 +92,14 @@ struct Process final { utils::RcIdMap fileDescriptors; // Named objects for debugging - utils::shared_mutex namedObjMutex; + rx::shared_mutex namedObjMutex; utils::kmap namedObjNames; utils::OwningIdMap namedObjIds; utils::kmap sigActions; // Named memory ranges for debugging - utils::shared_mutex namedMemMutex; + rx::shared_mutex namedMemMutex; utils::kmap namedMem; }; } // namespace orbis diff --git a/kernel/orbis/include/orbis/thread/Thread.hpp b/kernel/orbis/include/orbis/thread/Thread.hpp index 6872f4372..ae53e4f50 100644 --- a/kernel/orbis/include/orbis/thread/Thread.hpp +++ b/kernel/orbis/include/orbis/thread/Thread.hpp @@ -7,9 +7,9 @@ #include "../KernelAllocator.hpp" #include "../ucontext.hpp" -#include "../utils/SharedAtomic.hpp" -#include "../utils/SharedCV.hpp" -#include "../utils/SharedMutex.hpp" +#include "rx/SharedAtomic.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" #include namespace orbis { @@ -17,7 +17,7 @@ struct Process; static constexpr std::uint32_t kThreadSuspendFlag = 1 << 31; struct Thread { - utils::shared_mutex mtx; + rx::shared_mutex mtx; Process *tproc = nullptr; uint64_t retval[2]{}; void *context{}; @@ -34,14 +34,14 @@ struct Thread { .type = 2, .prio = 10, }; - utils::shared_mutex suspend_mtx; - utils::shared_cv suspend_cv; + rx::shared_mutex suspend_mtx; + rx::shared_cv suspend_cv; kvector sigReturns; kvector blockedSignals; kvector queuedSignals; - shared_atomic32 suspendFlags{0}; + rx::shared_atomic32 suspendFlags{0}; - utils::shared_atomic32 interruptedMtx{0}; + rx::shared_atomic32 interruptedMtx{0}; std::int64_t hostTid = -1; lwpid_t tid = -1; @@ -51,7 +51,7 @@ struct Thread { std::thread::native_handle_type nativeHandle; // Used to wake up thread in sleep queue - utils::shared_cv sync_cv; + rx::shared_cv sync_cv; uint64_t evfResultPattern; uint64_t evfIsCancelled; diff --git a/kernel/orbis/include/orbis/utils/IdMap.hpp b/kernel/orbis/include/orbis/utils/IdMap.hpp index 92e6ba279..59dcdb3ee 100644 --- a/kernel/orbis/include/orbis/utils/IdMap.hpp +++ b/kernel/orbis/include/orbis/utils/IdMap.hpp @@ -2,7 +2,7 @@ #include "BitSet.hpp" #include "Rc.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include #include @@ -66,7 +66,7 @@ class RcIdMap { public: static constexpr auto npos = static_cast(~static_cast(0)); - mutable shared_mutex mutex; + mutable rx::shared_mutex mutex; struct end_iterator {}; diff --git a/kernel/orbis/src/KernelContext.cpp b/kernel/orbis/src/KernelContext.cpp index 6b1a94947..bc305611e 100644 --- a/kernel/orbis/src/KernelContext.cpp +++ b/kernel/orbis/src/KernelContext.cpp @@ -2,7 +2,7 @@ #include "orbis/thread/Process.hpp" #include "orbis/thread/ProcessOps.hpp" #include "orbis/utils/Logs.hpp" -#include "utils/SharedAtomic.hpp" +#include "rx/SharedAtomic.hpp" #include #include #include @@ -283,7 +283,7 @@ void KernelContext::kfree(void *ptr, std::size_t size) { } } -std::tuple> +std::tuple> KernelContext::getUmtxChainIndexed(int i, Thread *t, uint32_t flags, void *ptr) { auto pid = t->tproc->pid; @@ -406,7 +406,7 @@ bool Thread::block() { scoped_unblock::scoped_unblock() { if (g_currentThread && g_currentThread->context) { - g_scopedUnblock = [](bool unblock) { + rx::g_scopedUnblock = [](bool unblock) { if (unblock) { return g_currentThread->unblock(); } @@ -416,5 +416,5 @@ scoped_unblock::scoped_unblock() { } } -scoped_unblock::~scoped_unblock() { g_scopedUnblock = nullptr; } +scoped_unblock::~scoped_unblock() { rx::g_scopedUnblock = nullptr; } } // namespace orbis diff --git a/kernel/orbis/src/evf.cpp b/kernel/orbis/src/evf.cpp index 2612078c1..0a31027d7 100644 --- a/kernel/orbis/src/evf.cpp +++ b/kernel/orbis/src/evf.cpp @@ -1,7 +1,6 @@ #include "evf.hpp" #include "error/ErrorCode.hpp" -#include "utils/Logs.hpp" -#include "utils/SharedCV.hpp" +#include "rx/SharedCV.hpp" #include orbis::ErrorCode orbis::EventFlag::wait(Thread *thread, std::uint8_t waitMode, @@ -99,7 +98,7 @@ orbis::ErrorCode orbis::EventFlag::wait(Thread *thread, std::uint8_t waitMode, orbis::ErrorCode orbis::EventFlag::tryWait(Thread *thread, std::uint8_t waitMode, std::uint64_t bitPattern) { - writer_lock lock(queueMtx); + rx::writer_lock lock(queueMtx); if (isDeleted) { return ErrorCode::ACCES; @@ -120,7 +119,7 @@ orbis::ErrorCode orbis::EventFlag::tryWait(Thread *thread, } std::size_t orbis::EventFlag::notify(NotifyType type, std::uint64_t bits) { - writer_lock lock(queueMtx); + rx::writer_lock lock(queueMtx); auto patValue = value.load(std::memory_order::relaxed); if (type == NotifyType::Destroy) { diff --git a/kernel/orbis/src/sys/sys_event.cpp b/kernel/orbis/src/sys/sys_event.cpp index 23749d511..dbf7d2bfc 100644 --- a/kernel/orbis/src/sys/sys_event.cpp +++ b/kernel/orbis/src/sys/sys_event.cpp @@ -78,7 +78,7 @@ static SysResult keventChange(KQueue *kq, KEvent &change, Thread *thread) { nodeIt = kq->notes.end(); } - std::unique_lock noteLock; + std::unique_lock noteLock; if (change.flags & kEvAdd) { if (nodeIt == kq->notes.end()) { auto ¬e = kq->notes.emplace_front(); diff --git a/orbis-kernel/include/orbis/KernelContext.hpp b/orbis-kernel/include/orbis/KernelContext.hpp new file mode 100644 index 000000000..dcd68edee --- /dev/null +++ b/orbis-kernel/include/orbis/KernelContext.hpp @@ -0,0 +1,260 @@ +#pragma once +#include "AppInfo.hpp" +#include "Budget.hpp" +#include "KernelAllocator.hpp" +#include "evf.hpp" +#include "ipmi.hpp" +#include "orbis/note.hpp" +#include "osem.hpp" +#include "thread/types.hpp" +#include "utils/IdMap.hpp" +#include "utils/LinkedNode.hpp" +#include "utils/SharedCV.hpp" +#include "utils/SharedMutex.hpp" + +#include +#include +#include +#include +#include + +namespace orbis { +struct Process; +struct Thread; + +struct UmtxKey { + // TODO: may contain a reference to a shared memory + std::uintptr_t addr; + orbis::pid_t pid; + + auto operator<=>(const UmtxKey &) const = default; +}; + +struct UmtxCond { + Thread *thr; + rx::shared_cv cv; + + UmtxCond(Thread *thr) : thr(thr) {} +}; + +struct UmtxChain { + rx::shared_mutex mtx; + using queue_type = utils::kmultimap; + queue_type sleep_queue; + queue_type spare_queue; + + std::pair *enqueue(UmtxKey &key, Thread *thr); + void erase(std::pair *obj); + queue_type::iterator erase(queue_type::iterator it); + uint notify_one(const UmtxKey &key); + uint notify_all(const UmtxKey &key); + uint notify_n(const UmtxKey &key, sint count); +}; + +enum class FwType : std::uint8_t { + Unknown, + Ps4, + Ps5, +}; + +struct RcAppInfo : RcBase, AppInfoEx { + orbis::uint32_t appState = 0; +}; + +class alignas(__STDCPP_DEFAULT_NEW_ALIGNMENT__) KernelContext final { +public: + KernelContext(); + ~KernelContext(); + + Process *createProcess(pid_t pid); + void deleteProcess(Process *proc); + Process *findProcessById(pid_t pid) const; + Process *findProcessByHostId(std::uint64_t pid) const; + + utils::LinkedNode *getProcessList() { return m_processes; } + + long allocatePid() { + std::lock_guard lock(m_thread_id_mtx); + return m_thread_id_map.emplace(0).first; + } + + long getTscFreq(); + + void *kalloc(std::size_t size, + std::size_t align = __STDCPP_DEFAULT_NEW_ALIGNMENT__); + void kfree(void *ptr, std::size_t size); + + std::pair createEventFlag(utils::kstring name, + std::int32_t flags, + std::uint64_t initPattern) { + std::lock_guard lock(m_evf_mtx); + + auto [it, inserted] = m_event_flags.try_emplace(std::move(name), nullptr); + if (inserted) { + it->second = knew(flags, initPattern); + std::strncpy(it->second->name, it->first.c_str(), 32); + } + + return {it->second.get(), inserted}; + } + + Ref findEventFlag(std::string_view name) { + std::lock_guard lock(m_evf_mtx); + + if (auto it = m_event_flags.find(name); it != m_event_flags.end()) { + return it->second; + } + + return {}; + } + + std::pair createSemaphore(utils::kstring name, + std::uint32_t attrs, + std::int32_t initCount, + std::int32_t maxCount) { + std::lock_guard lock(m_sem_mtx); + auto [it, inserted] = m_semaphores.try_emplace(std::move(name), nullptr); + if (inserted) { + it->second = knew(attrs, initCount, maxCount); + } + + return {it->second.get(), inserted}; + } + + Ref findSemaphore(std::string_view name) { + std::lock_guard lock(m_sem_mtx); + if (auto it = m_semaphores.find(name); it != m_semaphores.end()) { + return it->second; + } + + return {}; + } + + std::pair, ErrorCode> createIpmiServer(utils::kstring name) { + std::lock_guard lock(m_sem_mtx); + auto [it, inserted] = mIpmiServers.try_emplace(std::move(name), nullptr); + + if (!inserted) { + return {it->second, ErrorCode::EXIST}; + } + + it->second = knew(it->first); + + if (it->second == nullptr) { + mIpmiServers.erase(it); + return {nullptr, ErrorCode::NOMEM}; + } + + return {it->second, {}}; + } + + Ref findIpmiServer(std::string_view name) { + std::lock_guard lock(m_sem_mtx); + if (auto it = mIpmiServers.find(name); it != mIpmiServers.end()) { + return it->second; + } + + return {}; + } + + std::tuple &, + std::unique_lock> + getKernelEnv() { + std::unique_lock lock(m_kenv_mtx); + return {m_kenv, std::move(lock)}; + } + + void setKernelEnv(std::string_view key, std::string_view value) { + auto &kenvValue = m_kenv[utils::kstring(key)]; + auto len = std::min(sizeof(kenvValue) - 1, value.size()); + std::memcpy(kenvValue, value.data(), len); + kenvValue[len] = '0'; + } + + enum { + c_golden_ratio_prime = 2654404609u, + c_umtx_chains = 512, + c_umtx_shifts = 23, + }; + + // Use getUmtxChain0 or getUmtxChain1 + std::tuple> + getUmtxChainIndexed(int i, Thread *t, uint32_t flags, void *ptr); + + // Internal Umtx: Wait/Cv/Sem + auto getUmtxChain0(Thread *t, uint32_t flags, void *ptr) { + return getUmtxChainIndexed(0, t, flags, ptr); + } + + // Internal Umtx: Mutex/Umtx/Rwlock + auto getUmtxChain1(Thread *t, uint32_t flags, void *ptr) { + return getUmtxChainIndexed(1, t, flags, ptr); + } + + Ref deviceEventEmitter; + Ref shmDevice; + Ref dmemDevice; + Ref blockpoolDevice; + Ref gpuDevice; + Ref dceDevice; + rx::shared_mutex gpuDeviceMtx; + uint sdkVersion{}; + uint fwSdkVersion{}; + uint safeMode{}; + utils::RcIdMap ipmiMap; + RcIdMap appInfos; + RcIdMap budgets; + Ref processTypeBudgets[4]; + + rx::shared_mutex regMgrMtx; + kmap regMgrInt; + std::vector> dialogs{}; + + FwType fwType = FwType::Unknown; + bool isDevKit = false; + + Ref createProcessTypeBudget(Budget::ProcessType processType, + std::string_view name, + std::span items) { + auto budget = orbis::knew(name, processType, items); + processTypeBudgets[static_cast(processType)] = + orbis::knew(name, processType, items); + return budget; + } + + Ref getProcessTypeBudget(Budget::ProcessType processType) { + return processTypeBudgets[static_cast(processType)]; + } + +private: + rx::shared_mutex m_heap_mtx; + rx::shared_mutex m_heap_map_mtx; + void *m_heap_next = this + 1; + + utils::kmultimap m_free_heap; + utils::kmultimap m_used_node; + + UmtxChain m_umtx_chains[2][c_umtx_chains]{}; + + std::atomic m_tsc_freq{0}; + + rx::shared_mutex m_thread_id_mtx; + OwningIdMap m_thread_id_map; + mutable rx::shared_mutex m_proc_mtx; + utils::LinkedNode *m_processes = nullptr; + + rx::shared_mutex m_evf_mtx; + utils::kmap> m_event_flags; + + rx::shared_mutex m_sem_mtx; + utils::kmap> m_semaphores; + + rx::shared_mutex mIpmiServerMtx; + utils::kmap> mIpmiServers; + + rx::shared_mutex m_kenv_mtx; + utils::kmap m_kenv; // max size: 127 + '\0' +}; + +extern KernelContext &g_context; +} // namespace orbis diff --git a/rpcsx/gpu/Device.hpp b/rpcsx/gpu/Device.hpp index bfcda25b3..c7795f67c 100644 --- a/rpcsx/gpu/Device.hpp +++ b/rpcsx/gpu/Device.hpp @@ -6,7 +6,7 @@ #include "amdgpu/tiler_vulkan.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/utils/Rc.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include "rx/MemoryTable.hpp" #include "shader/SemanticInfo.hpp" #include "shader/SpvConverter.hpp" @@ -82,7 +82,7 @@ struct Device : orbis::RcBase, DeviceContext { CommandPipe commandPipe; FlipPipeline flipPipeline; - orbis::shared_mutex writeCommandMtx; + rx::shared_mutex writeCommandMtx; uint32_t imageIndex = 0; bool isImageAcquired = false; diff --git a/rpcsx/gpu/DeviceContext.hpp b/rpcsx/gpu/DeviceContext.hpp index dc2f37d9f..085b113e8 100644 --- a/rpcsx/gpu/DeviceContext.hpp +++ b/rpcsx/gpu/DeviceContext.hpp @@ -1,6 +1,6 @@ #pragma once -#include "orbis/utils/SharedAtomic.hpp" +#include "rx/SharedAtomic.hpp" #include #include @@ -69,9 +69,9 @@ struct DeviceContext { PadState kbPadState{}; std::atomic cpuCacheCommands[kMaxProcessCount][4]{}; - orbis::shared_atomic32 cpuCacheCommandsIdle[kMaxProcessCount]{}; - orbis::shared_atomic32 gpuCacheCommand[kMaxProcessCount]{}; - orbis::shared_atomic32 gpuCacheCommandIdle{}; + rx::shared_atomic32 cpuCacheCommandsIdle[kMaxProcessCount]{}; + rx::shared_atomic32 gpuCacheCommand[kMaxProcessCount]{}; + rx::shared_atomic32 gpuCacheCommandIdle{}; std::atomic *cachePages[kMaxProcessCount]{}; volatile std::uint32_t flipBuffer[kMaxProcessCount]; @@ -79,4 +79,4 @@ struct DeviceContext { volatile std::uint64_t flipCount[kMaxProcessCount]; volatile std::uint64_t bufferInUseAddress[kMaxProcessCount]; }; -} // namespace amdgpu \ No newline at end of file +} // namespace amdgpu diff --git a/rpcsx/gpu/Pipe.cpp b/rpcsx/gpu/Pipe.cpp index e46ad54b5..e1adfb206 100644 --- a/rpcsx/gpu/Pipe.cpp +++ b/rpcsx/gpu/Pipe.cpp @@ -200,7 +200,7 @@ void ComputePipe::setIndirectRing(int queueId, int indirectLevel, Ring ring) { } void ComputePipe::mapQueue(int queueId, Ring ring, - std::unique_lock &lock) { + std::unique_lock &lock) { if (ring.indirectLevel < 0 || ring.indirectLevel > 1) { rx::die("unexpected compute ring indirect level {}", ring.indirectLevel); } @@ -216,7 +216,7 @@ void ComputePipe::mapQueue(int queueId, Ring ring, } void ComputePipe::waitForIdle(int queueId, - std::unique_lock &lock) { + std::unique_lock &lock) { auto &ring = queues[1][queueId]; while (true) { diff --git a/rpcsx/gpu/Pipe.hpp b/rpcsx/gpu/Pipe.hpp index 77d9edeeb..c97f90a49 100644 --- a/rpcsx/gpu/Pipe.hpp +++ b/rpcsx/gpu/Pipe.hpp @@ -1,7 +1,7 @@ #pragma once #include "Registers.hpp" #include "Scheduler.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include #include @@ -42,7 +42,7 @@ struct ComputePipe { using CommandHandler = bool (ComputePipe::*)(Ring &); CommandHandler commandHandlers[255]; - orbis::shared_mutex queueMtx[kQueueCount]; + rx::shared_mutex queueMtx[kQueueCount]; int index; int currentQueueId; Ring queues[kRingsPerQueue][kQueueCount]; @@ -54,12 +54,12 @@ struct ComputePipe { bool processRing(Ring &ring); void setIndirectRing(int queueId, int level, Ring ring); void mapQueue(int queueId, Ring ring, - std::unique_lock &lock); - void waitForIdle(int queueId, std::unique_lock &lock); + std::unique_lock &lock); + void waitForIdle(int queueId, std::unique_lock &lock); void submit(int queueId, std::uint32_t offset); - std::unique_lock lockQueue(int queueId) { - return std::unique_lock(queueMtx[queueId]); + std::unique_lock lockQueue(int queueId) { + return std::unique_lock(queueMtx[queueId]); } bool setShReg(Ring &ring); @@ -109,7 +109,7 @@ struct GraphicsPipe { Ring deQueues[3]; Ring ceQueue; - orbis::shared_mutex eopFlipMtx; + rx::shared_mutex eopFlipMtx; std::uint32_t eopFlipRequestCount{0}; EopFlipRequest eopFlipRequests[kEopFlipRequestMax]; diff --git a/rpcsx/iodev/ajm.cpp b/rpcsx/iodev/ajm.cpp index b978b141e..41269ac23 100644 --- a/rpcsx/iodev/ajm.cpp +++ b/rpcsx/iodev/ajm.cpp @@ -41,7 +41,7 @@ enum { }; struct AjmDevice : IoDevice { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; orbis::uint32_t batchId = 1; // temp orbis::uint32_t instanceIds[AJM_CODEC_COUNT]{}; diff --git a/rpcsx/iodev/ajm.hpp b/rpcsx/iodev/ajm.hpp index e9cef20da..71fa45267 100644 --- a/rpcsx/iodev/ajm.hpp +++ b/rpcsx/iodev/ajm.hpp @@ -3,7 +3,7 @@ #include "libatrac9/libatrac9.h" #include "orbis-config.hpp" #include "orbis/KernelAllocator.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" // #include "orbis/utils/Logs.hpp" #include extern "C" { @@ -326,7 +326,7 @@ struct AJMAACCodecInfoSideband { }; struct Instance { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; AJMCodecs codec; AJMChannels maxChannels; AJMFormat outputFormat; diff --git a/rpcsx/iodev/blockpool.hpp b/rpcsx/iodev/blockpool.hpp index 6d6e94581..c10f2a111 100644 --- a/rpcsx/iodev/blockpool.hpp +++ b/rpcsx/iodev/blockpool.hpp @@ -4,12 +4,12 @@ #include "orbis/error/ErrorCode.hpp" #include "orbis/file.hpp" #include "orbis/utils/Rc.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include "rx/MemoryTable.hpp" #include struct BlockPoolDevice : public IoDevice { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; rx::MemoryAreaTable<> pool; orbis::ErrorCode open(orbis::Ref *file, const char *path, diff --git a/rpcsx/iodev/dce.hpp b/rpcsx/iodev/dce.hpp index 38a6b3d3d..5dbd5fee5 100644 --- a/rpcsx/iodev/dce.hpp +++ b/rpcsx/iodev/dce.hpp @@ -6,12 +6,12 @@ #include "orbis/file.hpp" #include "orbis/thread/Process.hpp" #include "orbis/utils/Rc.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" static constexpr auto kVmIdCount = 6; struct DceDevice : IoDevice { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; std::uint32_t eopCount = 0; std::uint32_t freeVmIds = (1 << (kVmIdCount + 1)) - 1; orbis::uint64_t dmemOffset = ~static_cast(0); diff --git a/rpcsx/iodev/dmem.hpp b/rpcsx/iodev/dmem.hpp index 308bb2625..6f4b0955d 100644 --- a/rpcsx/iodev/dmem.hpp +++ b/rpcsx/iodev/dmem.hpp @@ -5,13 +5,13 @@ #include "orbis/error/ErrorCode.hpp" #include "orbis/file.hpp" #include "orbis/utils/Rc.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include #include #include struct DmemDevice : public IoDevice { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; int index; int shmFd = -1; std::size_t dmemTotalSize; diff --git a/rpcsx/iodev/gc.cpp b/rpcsx/iodev/gc.cpp index 7fddddef4..c75900905 100644 --- a/rpcsx/iodev/gc.cpp +++ b/rpcsx/iodev/gc.cpp @@ -8,7 +8,7 @@ #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include "rx/die.hpp" #include "rx/print.hpp" #include "vm.hpp" @@ -25,7 +25,7 @@ struct ComputeQueue { }; struct GcDevice : public IoDevice { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; orbis::kmap clients; orbis::kmap computeQueues; void *submitArea = nullptr; diff --git a/rpcsx/iodev/mbus.hpp b/rpcsx/iodev/mbus.hpp index b2445cbb9..cfa7ef360 100644 --- a/rpcsx/iodev/mbus.hpp +++ b/rpcsx/iodev/mbus.hpp @@ -2,12 +2,12 @@ #include "io-device.hpp" #include "iodev/MBusEvent.hpp" -#include "orbis/utils/SharedCV.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" struct MBusDevice : IoDevice { - orbis::shared_mutex mtx; - orbis::shared_cv cv; + rx::shared_mutex mtx; + rx::shared_cv cv; orbis::kdeque events; orbis::Ref eventEmitter = orbis::knew(); diff --git a/rpcsx/iodev/mbus_av.cpp b/rpcsx/iodev/mbus_av.cpp index c48f677a9..610b43ffd 100644 --- a/rpcsx/iodev/mbus_av.cpp +++ b/rpcsx/iodev/mbus_av.cpp @@ -5,8 +5,6 @@ #include "orbis/note.hpp" #include "orbis/uio.hpp" #include "orbis/utils/Logs.hpp" -#include "orbis/utils/SharedCV.hpp" -#include "orbis/utils/SharedMutex.hpp" #include struct MBusAVFile : orbis::File {}; diff --git a/rpcsx/iodev/mbus_av.hpp b/rpcsx/iodev/mbus_av.hpp index 744eb86b1..3211ebaf7 100644 --- a/rpcsx/iodev/mbus_av.hpp +++ b/rpcsx/iodev/mbus_av.hpp @@ -2,12 +2,12 @@ #include "io-device.hpp" #include "iodev/MBusEvent.hpp" -#include "orbis/utils/SharedCV.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedCV.hpp" +#include "rx/SharedMutex.hpp" struct MBusAVDevice : IoDevice { - orbis::shared_mutex mtx; - orbis::shared_cv cv; + rx::shared_mutex mtx; + rx::shared_cv cv; orbis::kdeque events; orbis::Ref eventEmitter = orbis::knew(); diff --git a/rpcsx/iodev/notification.cpp b/rpcsx/iodev/notification.cpp index f648641ae..0afaf9487 100644 --- a/rpcsx/iodev/notification.cpp +++ b/rpcsx/iodev/notification.cpp @@ -4,7 +4,7 @@ #include "orbis/thread/Thread.hpp" #include "orbis/uio.hpp" #include "orbis/utils/Logs.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include #include #include @@ -14,7 +14,7 @@ struct NotificationFile : orbis::File {}; struct NotificationDevice : IoDevice { int index; - orbis::shared_mutex mutex; + rx::shared_mutex mutex; orbis::kvector data; NotificationDevice(int index) : index(index) {} diff --git a/rpcsx/iodev/sbl_srv.cpp b/rpcsx/iodev/sbl_srv.cpp index b018eceb7..7591264f1 100644 --- a/rpcsx/iodev/sbl_srv.cpp +++ b/rpcsx/iodev/sbl_srv.cpp @@ -4,14 +4,13 @@ #include "orbis/file.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" -#include "orbis/utils/SharedMutex.hpp" +#include "rx/SharedMutex.hpp" #include "vm.hpp" -#include struct SblSrvFile : public orbis::File {}; struct SblSrvDevice : IoDevice { - orbis::shared_mutex mtx; + rx::shared_mutex mtx; orbis::ErrorCode open(orbis::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override; diff --git a/rpcsx/vfs.cpp b/rpcsx/vfs.cpp index c262e882d..54e80e964 100644 --- a/rpcsx/vfs.cpp +++ b/rpcsx/vfs.cpp @@ -59,7 +59,7 @@ struct ProcFs : IoDevice { } }; -static orbis::shared_mutex gMountMtx; +static rx::shared_mutex gMountMtx; static std::map, std::greater<>> gMountsMap; static orbis::Ref gDevFs; diff --git a/rx/CMakeLists.txt b/rx/CMakeLists.txt index 246fb2490..7ef83bc46 100644 --- a/rx/CMakeLists.txt +++ b/rx/CMakeLists.txt @@ -5,10 +5,13 @@ find_package(Git) add_library(${PROJECT_NAME} OBJECT src/debug.cpp src/die.cpp + src/FileLock.cpp src/hexdump.cpp src/mem.cpp + src/SharedAtomic.cpp + src/SharedCV.cpp + src/SharedMutex.cpp src/Version.cpp - src/FileLock.cpp ) target_include_directories(${PROJECT_NAME} diff --git a/rx/include/rx/AtomicOp.hpp b/rx/include/rx/AtomicOp.hpp new file mode 100644 index 000000000..f314f7fb1 --- /dev/null +++ b/rx/include/rx/AtomicOp.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include + +namespace rx { +// Atomic operation; returns old value, or pair of old value and return value +// (cancel op if evaluates to false) +template > +inline std::conditional_t, T, std::pair> +atomic_fetch_op(std::atomic &v, F func) { + T _new, old = v.load(); + while (true) { + _new = old; + if constexpr (std::is_void_v) { + std::invoke(func, _new); + if (v.compare_exchange_strong(old, _new)) [[likely]] { + return old; + } + } else { + RT ret = std::invoke(func, _new); + if (!ret || v.compare_exchange_strong(old, _new)) [[likely]] { + return {old, std::move(ret)}; + } + } + } +} + +// Atomic operation; returns function result value, function is the lambda +template > +inline RT atomic_op(std::atomic &v, F func) { + T _new, old = v.load(); + while (true) { + _new = old; + if constexpr (std::is_void_v) { + std::invoke(func, _new); + if (v.compare_exchange_strong(old, _new)) [[likely]] { + return; + } + } else { + RT result = std::invoke(func, _new); + if (v.compare_exchange_strong(old, _new)) [[likely]] { + return result; + } + } + } +} + +#if defined(__ATOMIC_HLE_ACQUIRE) && defined(__ATOMIC_HLE_RELEASE) +static constexpr int s_hle_ack = __ATOMIC_SEQ_CST | __ATOMIC_HLE_ACQUIRE; +static constexpr int s_hle_rel = __ATOMIC_SEQ_CST | __ATOMIC_HLE_RELEASE; +#else +static constexpr int s_hle_ack = __ATOMIC_SEQ_CST; +static constexpr int s_hle_rel = __ATOMIC_SEQ_CST; +#endif + +template +inline bool compare_exchange_hle_acq(std::atomic &dest, T &comp, T exch) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8); + static_assert(std::atomic::is_always_lock_free); + return __atomic_compare_exchange(reinterpret_cast(&dest), &comp, &exch, + false, s_hle_ack, s_hle_ack); +} + +template +inline T fetch_add_hle_rel(std::atomic &dest, T value) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8); + static_assert(std::atomic::is_always_lock_free); + return __atomic_fetch_add(reinterpret_cast(&dest), value, s_hle_rel); +} +} // namespace rx diff --git a/kernel/orbis/include/orbis/utils/SharedAtomic.hpp b/rx/include/rx/SharedAtomic.hpp similarity index 97% rename from kernel/orbis/include/orbis/utils/SharedAtomic.hpp rename to rx/include/rx/SharedAtomic.hpp index 0a8f6308a..6422f50f4 100644 --- a/kernel/orbis/include/orbis/utils/SharedAtomic.hpp +++ b/rx/include/rx/SharedAtomic.hpp @@ -9,7 +9,7 @@ #include #include -namespace orbis { +namespace rx { inline void yield() { std::this_thread::yield(); } inline void relax() { #if defined(__GNUC__) && (defined __i386__ || defined __x86_64__) @@ -22,7 +22,6 @@ inline void relax() { static constexpr auto kRelaxSpinCount = 12; static constexpr auto kSpinCount = 16; -inline namespace utils { inline thread_local bool (*g_scopedUnblock)(bool) = nullptr; bool try_spin_wait(auto &&pred) { @@ -163,5 +162,4 @@ private: std::chrono::microseconds usec_timeout = std::chrono::microseconds::max()); }; -} // namespace utils -} // namespace orbis +} // namespace rx diff --git a/kernel/orbis/include/orbis/utils/SharedCV.hpp b/rx/include/rx/SharedCV.hpp similarity index 90% rename from kernel/orbis/include/orbis/utils/SharedCV.hpp rename to rx/include/rx/SharedCV.hpp index 6f8152f24..3c328933e 100644 --- a/kernel/orbis/include/orbis/utils/SharedCV.hpp +++ b/rx/include/rx/SharedCV.hpp @@ -1,21 +1,19 @@ #pragma once -#include "orbis/utils/SharedAtomic.hpp" +#include "SharedAtomic.hpp" +#include "SharedMutex.hpp" #include #include #include -#include -#include #include -namespace orbis { -inline namespace utils { +namespace rx { // IPC-ready lightweight condition variable class shared_cv final { enum : unsigned { c_waiter_mask = 0xffff, c_signal_mask = 0x7fff0000, -#ifdef ORBIS_HAS_FUTEX +#ifdef __linux c_locked_mask = 0x80000000, #endif c_signal_one = c_waiter_mask + 1, @@ -88,5 +86,4 @@ public: } } }; -} // namespace utils -} // namespace orbis +} // namespace rx diff --git a/kernel/orbis/include/orbis/utils/SharedMutex.hpp b/rx/include/rx/SharedMutex.hpp similarity index 95% rename from kernel/orbis/include/orbis/utils/SharedMutex.hpp rename to rx/include/rx/SharedMutex.hpp index 8beebae76..4b0d35b32 100644 --- a/kernel/orbis/include/orbis/utils/SharedMutex.hpp +++ b/rx/include/rx/SharedMutex.hpp @@ -1,12 +1,10 @@ #pragma once -#include -#include -#include +#include "AtomicOp.hpp" +#include "SharedAtomic.hpp" #include -namespace orbis { -inline namespace utils { +namespace rx { // IPC-ready shared mutex, using only writer lock is recommended class shared_mutex final { friend class shared_cv; @@ -147,5 +145,4 @@ public: explicit writer_lock(shared_mutex &mutex) : m_mutex(mutex) { m_mutex.lock(); } ~writer_lock() { m_mutex.unlock(); } }; -} // namespace utils -} // namespace orbis +} // namespace rx diff --git a/kernel/orbis/src/utils/SharedAtomic.cpp b/rx/src/SharedAtomic.cpp similarity index 57% rename from kernel/orbis/src/utils/SharedAtomic.cpp rename to rx/src/SharedAtomic.cpp index 846845e37..201253dfb 100644 --- a/kernel/orbis/src/utils/SharedAtomic.cpp +++ b/rx/src/SharedAtomic.cpp @@ -1,7 +1,7 @@ -#include "utils/SharedAtomic.hpp" -using namespace orbis; +#include "SharedAtomic.hpp" +using namespace rx; -#ifdef ORBIS_HAS_FUTEX +#ifdef __linux__ #include std::errc shared_atomic32::wait_impl(std::uint32_t oldValue, @@ -40,6 +40,10 @@ std::errc shared_atomic32::wait_impl(std::uint32_t oldValue, } if (result < 0) { + if (errorCode == std::errc::interrupted) { + return std::errc::resource_unavailable_try_again; + } + return errorCode; } @@ -49,7 +53,7 @@ std::errc shared_atomic32::wait_impl(std::uint32_t oldValue, int shared_atomic32::notify_n(int count) const { return syscall(SYS_futex, this, FUTEX_WAKE, count); } -#elif defined(ORBIS_HAS_ULOCK) +#elif defined(__APPLE__) #include #define UL_COMPARE_AND_WAIT 1 @@ -79,9 +83,29 @@ extern int __ulock_wake(uint32_t operation, void *addr, uint64_t wake_value); std::errc shared_atomic32::wait_impl(std::uint32_t oldValue, std::chrono::microseconds usec_timeout) { + bool useTimeout = usec_timeout != std::chrono::microseconds::max(); + bool unblock = (!useTimeout || usec_timeout.count() > 1000) && + g_scopedUnblock != nullptr; + + if (unblock) { + if (!g_scopedUnblock(true)) { + return std::errc::interrupted; + } + } + int result = __ulock_wait(UL_COMPARE_AND_WAIT_SHARED, (void *)this, oldValue, usec_timeout.count()); + if (unblock) { + if (!g_scopedUnblock(false)) { + if (result < 0) { + return std::errc::interrupted; + } + + return {}; + } + } + if (result < 0) { return static_cast(errno); } @@ -113,6 +137,68 @@ int shared_atomic32::notify_n(int count) const { return result; } +#elif defined(_WIN32) +#include +#include + +std::errc shared_atomic32::wait_impl(std::uint32_t oldValue, + std::chrono::microseconds usec_timeout) { + + bool useTimeout = usec_timeout != std::chrono::microseconds::max(); + + bool unblock = (!useTimeout || usec_timeout.count() > 1000) && + g_scopedUnblock != nullptr; + + if (unblock) { + if (!g_scopedUnblock(true)) { + return std::errc::interrupted; + } + } + + BOOL result = WaitOnAddress( + this, &oldValue, sizeof(std::uint32_t), + useTimeout + ? std::chrono::duration_cast(usec_timeout) + .count() + : INFINITY); + + DWORD error = 0; + if (!result) { + error = GetLastError(); + } else { + if (load(std::memory_order::relaxed) == oldValue) { + error = ERROR_ALERTED; // dummy error + } + } + + if (unblock) { + if (!g_scopedUnblock(false)) { + if (result != TRUE) { + return std::errc::interrupted; + } + + return {}; + } + } + + if (error == ERROR_TIMEOUT) { + return std::errc::timed_out; + } + + return std::errc::resource_unavailable_try_again; +} + +int shared_atomic32::notify_n(int count) const { + if (count == 1) { + WakeByAddressSingle(const_cast(this)); + } else if (count == std::numeric_limits::max()) { + WakeByAddressAll(const_cast(this)); + } else { + for (int i = 0; i < count; ++i) { + WakeByAddressSingle(const_cast(this)); + } + } +} #else #error Unimplemented atomic for this platform #endif diff --git a/kernel/orbis/src/utils/SharedCV.cpp b/rx/src/SharedCV.cpp similarity index 95% rename from kernel/orbis/src/utils/SharedCV.cpp rename to rx/src/SharedCV.cpp index 044a19b7f..781b4cc42 100644 --- a/kernel/orbis/src/utils/SharedCV.cpp +++ b/rx/src/SharedCV.cpp @@ -1,13 +1,13 @@ -#include "orbis/utils/SharedCV.hpp" +#include "SharedCV.hpp" #include -#ifdef ORBIS_HAS_FUTEX +#ifdef __linux #include #include #include #endif -namespace orbis::utils { +namespace rx { std::errc shared_cv::impl_wait(shared_mutex &mutex, unsigned _val, std::uint64_t usec_timeout) noexcept { // Not supposed to fail @@ -39,14 +39,14 @@ std::errc shared_cv::impl_wait(shared_mutex &mutex, unsigned _val, value -= c_signal_one; } -#ifdef ORBIS_HAS_FUTEX +#ifdef __linux if (value & c_locked_mask) { value -= c_locked_mask; } #endif }); -#ifdef ORBIS_HAS_FUTEX +#ifdef __linux // Lock is already acquired if (old & c_locked_mask) { return {}; @@ -76,7 +76,7 @@ std::errc shared_cv::impl_wait(shared_mutex &mutex, unsigned _val, } void shared_cv::impl_wake(shared_mutex &mutex, int _count) noexcept { -#ifdef ORBIS_HAS_FUTEX +#ifdef __linux while (true) { unsigned _old = m_value.load(); const bool is_one = _count == 1; diff --git a/kernel/orbis/src/utils/SharedMutex.cpp b/rx/src/SharedMutex.cpp similarity index 97% rename from kernel/orbis/src/utils/SharedMutex.cpp rename to rx/src/SharedMutex.cpp index 23f68c214..a7d7458ab 100644 --- a/kernel/orbis/src/utils/SharedMutex.cpp +++ b/rx/src/SharedMutex.cpp @@ -1,5 +1,4 @@ -#include "utils/SharedMutex.hpp" -#include "utils/Logs.hpp" +#include "SharedMutex.hpp" #include #include #include @@ -11,7 +10,7 @@ static void busy_wait(unsigned long long cycles = 3000) { while (__builtin_ia32_rdtsc() < stop); } -namespace orbis::utils { +namespace rx { void shared_mutex::impl_lock_shared(unsigned val) { if (val >= c_err) std::abort(); // "shared_mutex underflow" @@ -179,4 +178,4 @@ bool shared_mutex::lock_forced(int count) { m_value.fetch_add(c_one * count); return true; } -} // namespace orbis::utils +} // namespace rx