From d7ad77b406705f9d6c0fcd7e522cd2a0edbb4053 Mon Sep 17 00:00:00 2001 From: DH Date: Sun, 30 Nov 2025 15:46:37 +0300 Subject: [PATCH] orbis: implement physical memory emulation level & utils improvement fix blockpool & dmem implementation modernize blockpool & dmem io devices use budgets per allocation add serialization support for MemoryTableWithPayload and AddressRange utils add format support for EnumBitSet util implemented trace formatter per syscall increased allowed reference count for Ref --- kernel/include/kernel/KernelObject.hpp | 2 + kernel/include/kernel/MemoryResource.hpp | 49 +- kernel/orbis/CMakeLists.txt | 2 + kernel/orbis/include/orbis/Budget.hpp | 41 + kernel/orbis/include/orbis/IoDevice.hpp | 11 +- kernel/orbis/include/orbis/KernelContext.hpp | 4 +- kernel/orbis/include/orbis/MemoryType.hpp | 12 + kernel/orbis/include/orbis/blockpool.hpp | 32 + kernel/orbis/include/orbis/dmem.hpp | 62 + kernel/orbis/include/orbis/file.hpp | 9 +- kernel/orbis/include/orbis/fmem.hpp | 12 +- kernel/orbis/include/orbis/pmem.hpp | 18 +- kernel/orbis/include/orbis/sys/sysproto.hpp | 61 +- kernel/orbis/include/orbis/thread/Process.hpp | 8 +- .../orbis/include/orbis/thread/ProcessOps.hpp | 29 +- kernel/orbis/include/orbis/thread/Thread.hpp | 5 +- kernel/orbis/include/orbis/thread/sysent.hpp | 2 + kernel/orbis/include/orbis/thread/types.hpp | 2 +- kernel/orbis/include/orbis/ucontext.hpp | 2 +- kernel/orbis/include/orbis/vm.hpp | 13 - kernel/orbis/include/orbis/vmem.hpp | 153 ++- kernel/orbis/src/IoDevice.cpp | 20 + kernel/orbis/src/blockpool.cpp | 370 +++++ kernel/orbis/src/dmem.cpp | 610 ++++++++ kernel/orbis/src/fmem.cpp | 49 +- kernel/orbis/src/pmem.cpp | 53 +- kernel/orbis/src/sys/sys_event.cpp | 24 +- kernel/orbis/src/sys/sys_sce.cpp | 215 ++- kernel/orbis/src/sys/sys_sysctl.cpp | 6 +- kernel/orbis/src/sys/sys_vm_mmap.cpp | 219 ++- kernel/orbis/src/sysvec.cpp | 94 +- kernel/orbis/src/thread/Process.cpp | 6 + kernel/orbis/src/thread/Thread.cpp | 32 + kernel/orbis/src/vmem.cpp | 1018 ++++++++++++-- rpcsx/CMakeLists.txt | 1 - rpcsx/gpu/Device.cpp | 158 +-- rpcsx/gpu/Device.hpp | 22 +- rpcsx/gpu/DeviceCtl.cpp | 24 + rpcsx/gpu/DeviceCtl.hpp | 12 + rpcsx/gpu/Pipe.cpp | 16 +- rpcsx/io-device.cpp | 131 +- rpcsx/io-device.hpp | 2 +- rpcsx/io-devices.hpp | 3 +- rpcsx/iodev/ajm.cpp | 9 +- rpcsx/iodev/blockpool.cpp | 220 ++- rpcsx/iodev/blockpool.hpp | 22 - rpcsx/iodev/dce.cpp | 180 +-- rpcsx/iodev/dce.hpp | 11 +- rpcsx/iodev/dmem.cpp | 665 +++++---- rpcsx/iodev/dmem.hpp | 47 - rpcsx/iodev/gc.cpp | 115 +- rpcsx/iodev/hmd_mmap.cpp | 17 - rpcsx/iodev/rng.cpp | 19 +- rpcsx/iodev/sbl_srv.cpp | 17 - rpcsx/ipmi.cpp | 55 +- rpcsx/ipmi.hpp | 2 + rpcsx/linker.cpp | 466 +++++-- rpcsx/linker.hpp | 2 - rpcsx/main.cpp | 223 +-- rpcsx/ops.cpp | 197 +-- rpcsx/thread.cpp | 2 +- rpcsx/vm.cpp | 1221 ----------------- rpcsx/vm.hpp | 84 -- rx/include/rx/EnumBitSet.hpp | 69 +- rx/include/rx/FunctionRef.hpp | 2 +- rx/include/rx/Mappable.hpp | 12 +- rx/include/rx/MemoryTable.hpp | 117 +- rx/include/rx/Rc.hpp | 2 +- rx/include/rx/Serializer.hpp | 10 +- rx/include/rx/refl.hpp | 28 +- 70 files changed, 4268 insertions(+), 3160 deletions(-) create mode 100644 kernel/orbis/include/orbis/MemoryType.hpp create mode 100644 kernel/orbis/include/orbis/blockpool.hpp create mode 100644 kernel/orbis/include/orbis/dmem.hpp delete mode 100644 kernel/orbis/include/orbis/vm.hpp create mode 100644 kernel/orbis/src/blockpool.cpp create mode 100644 kernel/orbis/src/dmem.cpp delete mode 100644 rpcsx/iodev/blockpool.hpp delete mode 100644 rpcsx/iodev/dmem.hpp delete mode 100644 rpcsx/vm.cpp delete mode 100644 rpcsx/vm.hpp diff --git a/kernel/include/kernel/KernelObject.hpp b/kernel/include/kernel/KernelObject.hpp index 7947b0c2f..d84a02f88 100644 --- a/kernel/include/kernel/KernelObject.hpp +++ b/kernel/include/kernel/KernelObject.hpp @@ -65,6 +65,7 @@ struct LockableKernelObject : KernelObjectBase, StateT { void lock() const { mtx.lock(); } void unlock() const { mtx.unlock(); } + bool try_lock() const { return mtx.try_lock(); } }; namespace detail { @@ -175,6 +176,7 @@ private: template struct StaticObjectRef { + using type = T; std::uint32_t offset; T *get(std::byte *storage) { return reinterpret_cast(storage + offset); } diff --git a/kernel/include/kernel/MemoryResource.hpp b/kernel/include/kernel/MemoryResource.hpp index 31afe59b9..2a78eabdc 100644 --- a/kernel/include/kernel/MemoryResource.hpp +++ b/kernel/include/kernel/MemoryResource.hpp @@ -75,11 +75,13 @@ concept AllocationInfo = requires(T &t, rx::AddressRange range) { requires rx::Serializable; }; -template typename Allocator, rx::Serializable Resource = ExternalResource> struct AllocableResource : Resource { - mutable rx::MemoryTableWithPayload allocations; - using iterator = typename rx::MemoryTableWithPayload::iterator; + mutable rx::MemoryTableWithPayload allocations; + using BaseResource = AllocableResource; + using iterator = + typename rx::MemoryTableWithPayload::iterator; struct AllocationResult { iterator it; @@ -133,6 +135,10 @@ struct AllocableResource : Resource { } } + iterator lowerBound(std::uint64_t address) { + return allocations.lowerBound(address); + } + iterator query(std::uint64_t address) { return allocations.queryArea(address); } @@ -164,7 +170,14 @@ struct AllocableResource : Resource { if (flags & AllocationFlags::Fixed) { it = allocations.queryArea(addressHint); } else { - it = allocations.lowerBound(addressHint); + if (addressHint == 0 && (flags & AllocationFlags::Stack)) { + it = allocations.end(); + if (it != allocations.begin()) { + --it; + } + } else { + it = allocations.lowerBound(addressHint); + } } if (it == allocations.end()) { @@ -245,7 +258,7 @@ struct AllocableResource : Resource { } } else { auto hasEnoughSpace = [=](rx::AddressRange range) { - if (range.contains(addressHint)) { + if (addressHint != 0 && range.contains(addressHint)) { if (flags & AllocationFlags::Stack) { range = rx::AddressRange::fromBeginEnd( rx::alignDown(range.beginAddress(), alignment), addressHint); @@ -253,6 +266,11 @@ struct AllocableResource : Resource { range = rx::AddressRange::fromBeginEnd(addressHint, range.endAddress()); } + } else if (flags & AllocationFlags::Stack) { + auto alignedStackRange = rx::AddressRange::fromBeginEnd( + rx::alignDown(range.endAddress() - size, alignment), + range.endAddress()); + range = range.intersection(alignedStackRange); } auto alignedAddress = rx::AddressRange::fromBeginEnd( @@ -261,7 +279,7 @@ struct AllocableResource : Resource { return alignedAddress.isValid() && alignedAddress.size() >= size; }; - if (addressHint != 0 && (flags & AllocationFlags::Stack)) { + if (flags & AllocationFlags::Stack) { while (it != begin()) { if (!it->isAllocated() && hasEnoughSpace(it.range())) { break; @@ -288,23 +306,28 @@ struct AllocableResource : Resource { } // now `it` points to region that meets requirements, create fixed range - if (it.range().contains(addressHint)) { + if (addressHint != 0 && it.range().contains(addressHint)) { if (flags & AllocationFlags::Stack) { - fixedRange = - rx::AddressRange::fromBeginSize(rx::alignDown(addressHint - size, alignment), size); + fixedRange = rx::AddressRange::fromBeginSize( + rx::alignDown(addressHint - size, alignment), size); } else { fixedRange = rx::AddressRange::fromBeginEnd(addressHint, it.endAddress()); } } else { - fixedRange = rx::AddressRange::fromBeginEnd( - rx::alignUp(it.beginAddress(), alignment), it.endAddress()); + if (flags & AllocationFlags::Stack) { + fixedRange = rx::AddressRange::fromBeginSize( + rx::alignDown(it.endAddress() - size, alignment), + size); + } else { + fixedRange = rx::AddressRange::fromBeginSize( + rx::alignUp(it.beginAddress(), alignment), size); + } } } if (fixedRange.size() > size) { - if ((flags & AllocationFlags::Stack) && - !it.range().contains(addressHint)) { + if (flags & AllocationFlags::Stack) { fixedRange = rx::AddressRange::fromBeginSize( rx::alignDown(fixedRange.endAddress() - size, alignment), size); } else { diff --git a/kernel/orbis/CMakeLists.txt b/kernel/orbis/CMakeLists.txt index bcc80bb9c..98eab3cb4 100644 --- a/kernel/orbis/CMakeLists.txt +++ b/kernel/orbis/CMakeLists.txt @@ -1,6 +1,8 @@ set(CMAKE_POSITION_INDEPENDENT_CODE on) add_library(obj.orbis-kernel OBJECT + src/blockpool.cpp + src/dmem.cpp src/event.cpp src/evf.cpp src/fmem.cpp diff --git a/kernel/orbis/include/orbis/Budget.hpp b/kernel/orbis/include/orbis/Budget.hpp index 0bc352f06..f2a6cb72b 100644 --- a/kernel/orbis/include/orbis/Budget.hpp +++ b/kernel/orbis/include/orbis/Budget.hpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace orbis { enum class BudgetResource : uint32_t { @@ -139,4 +140,44 @@ private: BudgetList mList; char mName[32]{}; }; + +class ScopedBudgetAcquire { + Budget *mBudget = nullptr; + std::uint64_t mSize = 0; + BudgetResource mResourceId = {}; + +public: + ScopedBudgetAcquire() = default; + + ScopedBudgetAcquire(const ScopedBudgetAcquire &) = delete; + ScopedBudgetAcquire &operator=(const ScopedBudgetAcquire &) = delete; + ScopedBudgetAcquire(Budget *budget, BudgetResource resourceId, + std::uint64_t size) + : mBudget(budget), mSize(size), mResourceId(resourceId) { + if (!budget->acquire(resourceId, size)) { + mBudget = nullptr; + } + } + + ScopedBudgetAcquire(ScopedBudgetAcquire &&other) noexcept { + *this = std::move(other); + } + + ScopedBudgetAcquire &operator=(ScopedBudgetAcquire &&other) noexcept { + std::swap(mBudget, other.mBudget); + std::swap(mSize, other.mSize); + std::swap(mResourceId, other.mResourceId); + return *this; + } + + ~ScopedBudgetAcquire() { + if (mBudget) { + mBudget->release(mResourceId, mSize); + } + } + + explicit operator bool() const { return mBudget != nullptr; } + + void commit() { mBudget = nullptr; } +}; } // namespace orbis diff --git a/kernel/orbis/include/orbis/IoDevice.hpp b/kernel/orbis/include/orbis/IoDevice.hpp index 114bdbf6b..b2571f3f9 100644 --- a/kernel/orbis/include/orbis/IoDevice.hpp +++ b/kernel/orbis/include/orbis/IoDevice.hpp @@ -6,6 +6,7 @@ #include "rx/EnumBitSet.hpp" #include "rx/Rc.hpp" #include "rx/mem.hpp" +#include "vmem.hpp" #include #include @@ -34,6 +35,8 @@ struct Process; struct Stat; struct StatFs; struct IoDevice : rx::RcBase { + rx::EnumBitSet blockFlags{}; + virtual ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, Thread *thread) = 0; @@ -67,14 +70,12 @@ struct IoDevice : rx::RcBase { return ErrorCode::NOTSUP; } - virtual ErrorCode ioctl(std::uint64_t request, orbis::ptr argp, + virtual ErrorCode ioctl(std::uint64_t request, ptr argp, Thread *thread); virtual ErrorCode map(rx::AddressRange range, std::int64_t offset, - rx::EnumBitSet protection, - Process *process) { - return ErrorCode::NOTSUP; - } + rx::EnumBitSet protection, File *file, + Process *process); }; namespace ioctl { diff --git a/kernel/orbis/include/orbis/KernelContext.hpp b/kernel/orbis/include/orbis/KernelContext.hpp index 7ecd665db..4e4334601 100644 --- a/kernel/orbis/include/orbis/KernelContext.hpp +++ b/kernel/orbis/include/orbis/KernelContext.hpp @@ -122,8 +122,8 @@ public: rx::Ref deviceEventEmitter; rx::Ref shmDevice; - rx::Ref dmemDevice; - rx::Ref blockpoolDevice; + rx::Ref dmem; + rx::Ref blockpool; rx::Ref gpuDevice; rx::Ref dceDevice; rx::shared_mutex gpuDeviceMtx; diff --git a/kernel/orbis/include/orbis/MemoryType.hpp b/kernel/orbis/include/orbis/MemoryType.hpp new file mode 100644 index 000000000..fae3dd018 --- /dev/null +++ b/kernel/orbis/include/orbis/MemoryType.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace orbis { +enum class MemoryType : std::uint32_t { + Invalid = -1u, + WbOnion = 0, // write back, CPU bus + WCGarlic = 3, // combining, GPU bus + WbGarlic = 10, // write back, GPU bus +}; +} diff --git a/kernel/orbis/include/orbis/blockpool.hpp b/kernel/orbis/include/orbis/blockpool.hpp new file mode 100644 index 000000000..5bdaefc02 --- /dev/null +++ b/kernel/orbis/include/orbis/blockpool.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "dmem.hpp" +#include "error/ErrorCode.hpp" +#include "orbis-config.hpp" +#include "rx/AddressRange.hpp" + +namespace orbis { +struct Process; +} + +namespace orbis::blockpool { +#pragma pack(push, 1) +struct BlockStats { + sint availFlushedBlocks; + sint availCachedBlocks; + sint commitFlushedBlocks; + sint commitCachedBlocks; +}; +static_assert(sizeof(BlockStats) == 16); +#pragma pack(pop) + +void clear(); +ErrorCode expand(rx::AddressRange dmemRange); +ErrorCode allocateControlBlock(); +ErrorCode releaseControlBlock(); +ErrorCode commit(Process *process, rx::AddressRange vmemRange, MemoryType type, + rx::EnumBitSet protection); +void decommit(Process *process, rx::AddressRange vmemRange); +std::optional getType(std::uint64_t address); +BlockStats stats(); +} // namespace orbis::blockpool diff --git a/kernel/orbis/include/orbis/dmem.hpp b/kernel/orbis/include/orbis/dmem.hpp new file mode 100644 index 000000000..3dd91c639 --- /dev/null +++ b/kernel/orbis/include/orbis/dmem.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include "MemoryType.hpp" +#include "error/ErrorCode.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" +#include "vmem.hpp" +#include + +namespace orbis { +struct Process; +struct IoDevice; +} // namespace orbis + +namespace orbis::dmem { +static constexpr auto kPageSize = 64 * 1024; + +enum class QueryFlags { + LowerBound, + Pooled, + + bitset_last = Pooled, +}; + +struct QueryResult { + rx::AddressRange range; + MemoryType memoryType; +}; + +ErrorCode initialize(); +ErrorCode clear(unsigned dmemIndex); + +std::pair +allocate(unsigned dmemIndex, rx::AddressRange searchRange, std::uint64_t len, + MemoryType memoryType, std::uint64_t alignment = kPageSize, + bool pooled = false); + +ErrorCode release(unsigned dmemIndex, rx::AddressRange range, + bool pooled = false); + +std::pair reserveSystem(unsigned dmemIndex, + std::uint64_t size); +std::pair reservePooled(unsigned dmemIndex, + rx::AddressRange searchRange, + std::uint64_t size, + std::uint64_t alignment); + +ErrorCode setType(unsigned dmemIndex, rx::AddressRange range, MemoryType type, + rx::EnumBitSet flags = {}); +std::optional query(unsigned dmemIndex, std::uint64_t dmemOffset, + rx::EnumBitSet flags = {}); +std::uint64_t getSize(unsigned dmemIndex); +std::pair +getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t alignment); + +ErrorCode map(unsigned dmemIndex, rx::AddressRange range, std::uint64_t offset, + rx::EnumBitSet protection); + +std::pair getPmemOffset(unsigned dmemIndex, + std::uint64_t dmemOffset); +} // namespace orbis::dmem diff --git a/kernel/orbis/include/orbis/file.hpp b/kernel/orbis/include/orbis/file.hpp index 1be2edb5e..a6639bb13 100644 --- a/kernel/orbis/include/orbis/file.hpp +++ b/kernel/orbis/include/orbis/file.hpp @@ -4,6 +4,7 @@ #include "KernelAllocator.hpp" #include "error/ErrorCode.hpp" #include "note.hpp" +#include "rx/Mappable.hpp" #include "rx/Rc.hpp" #include "rx/SharedMutex.hpp" #include "stat.hpp" @@ -42,12 +43,6 @@ struct FileOps { // TODO: chown // TODO: chmod - ErrorCode (*mmap)(File *file, void **address, std::uint64_t size, - std::int32_t prot, std::int32_t flags, std::int64_t offset, - Thread *thread) = nullptr; - ErrorCode (*munmap)(File *file, void **address, std::uint64_t size, - Thread *thread) = nullptr; - ErrorCode (*bind)(orbis::File *file, SocketAddress *address, std::size_t addressLen, Thread *thread) = nullptr; ErrorCode (*listen)(orbis::File *file, int backlog, Thread *thread) = nullptr; @@ -84,7 +79,7 @@ struct File : rx::RcBase { std::uint64_t nextOff = 0; int flags = 0; int mode = 0; - int hostFd = -1; + rx::Mappable hostFd; kvector dirEntries; bool noBlock() const { return (flags & 4) != 0; } diff --git a/kernel/orbis/include/orbis/fmem.hpp b/kernel/orbis/include/orbis/fmem.hpp index b6edf1206..10f37f306 100644 --- a/kernel/orbis/include/orbis/fmem.hpp +++ b/kernel/orbis/include/orbis/fmem.hpp @@ -4,13 +4,9 @@ #include "rx/AddressRange.hpp" #include -namespace orbis { -struct Process; -} namespace orbis::fmem { -ErrorCode initialize(Process *process, std::uint64_t size); -void destroy(Process *process); -std::pair allocate(Process *process, - std::uint64_t size); -ErrorCode deallocate(Process *process, rx::AddressRange range); +ErrorCode initialize(std::uint64_t size); +void destroy(); +std::pair allocate(std::uint64_t size); +ErrorCode deallocate(rx::AddressRange range); } // namespace orbis::fmem diff --git a/kernel/orbis/include/orbis/pmem.hpp b/kernel/orbis/include/orbis/pmem.hpp index 516354116..f0994007b 100644 --- a/kernel/orbis/include/orbis/pmem.hpp +++ b/kernel/orbis/include/orbis/pmem.hpp @@ -9,30 +9,20 @@ namespace orbis { using kernel::AllocationFlags; struct IoDevice; +struct File; } // namespace orbis namespace orbis::pmem { -enum class MemoryType : std::uint32_t { - Invalid = -1u, - WbOnion = 0, // write back, CPU bus - WCGarlic = 3, // combining, GPU bus - WbGarlic = 10, // write back, GPU bus -}; - -struct AllocatedMemory { - rx::AddressRange range; - MemoryType memoryType; -}; - ErrorCode initialize(std::uint64_t size); void destroy(); std::pair -allocate(std::uint64_t addressHint, std::uint64_t size, MemoryType memoryType, +allocate(std::uint64_t addressHint, std::uint64_t size, rx::EnumBitSet flags, std::uint64_t alignment); ErrorCode deallocate(rx::AddressRange range); -std::optional query(std::uint64_t address); +std::optional query(std::uint64_t address); ErrorCode map(std::uint64_t virtualAddress, rx::AddressRange range, rx::EnumBitSet protection); std::size_t getSize(); IoDevice *getDevice(); +File *getFile(); } // namespace orbis::pmem diff --git a/kernel/orbis/include/orbis/sys/sysproto.hpp b/kernel/orbis/include/orbis/sys/sysproto.hpp index 6925c6185..d2f0b8ff9 100644 --- a/kernel/orbis/include/orbis/sys/sysproto.hpp +++ b/kernel/orbis/include/orbis/sys/sysproto.hpp @@ -1,4 +1,7 @@ #include "orbis-config.hpp" +#include "orbis/dmem.hpp" +#include "orbis/vmem.hpp" +#include "rx/EnumBitSet.hpp" #include #include #include @@ -18,7 +21,9 @@ using id_t = uint32_t; struct Thread; struct AuthInfo; +namespace vmem { struct MemoryProtection; +} struct ModuleInfo; struct ModuleInfoEx; struct KEvent; @@ -105,16 +110,16 @@ SysResult sys_execve(Thread *thread, ptr fname, ptr> argv, ptr> envv); SysResult sys_umask(Thread *thread, sint newmask); SysResult sys_chroot(Thread *thread, ptr path); -SysResult sys_msync(Thread *thread, ptr addr, size_t len, sint flags); +SysResult sys_msync(Thread *thread, uintptr_t addr, size_t len, sint flags); SysResult sys_vfork(Thread *thread); SysResult sys_sbrk(Thread *thread, sint incr); SysResult sys_sstk(Thread *thread, sint incr); SysResult sys_ovadvise(Thread *thread, sint anom); -SysResult sys_munmap(Thread *thread, ptr addr, size_t len); -SysResult sys_mprotect(Thread *thread, ptr addr, size_t len, - sint prot); -SysResult sys_madvise(Thread *thread, ptr addr, size_t len, sint behav); -SysResult sys_mincore(Thread *thread, ptr addr, size_t len, +SysResult sys_munmap(Thread *thread, uintptr_t addr, size_t len); +SysResult sys_mprotect(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot); +SysResult sys_madvise(Thread *thread, uintptr_t addr, size_t len, sint behav); +SysResult sys_mincore(Thread *thread, uintptr_t addr, size_t len, ptr vec); SysResult sys_getgroups(Thread *thread, uint gidsetsize, ptr gidset); SysResult sys_setgroups(Thread *thread, uint gidsetsize, ptr gidset); @@ -198,8 +203,10 @@ SysResult sys_getrlimit(Thread *thread, uint which, ptr rlp); SysResult sys_setrlimit(Thread *thread, uint which, ptr rlp); SysResult sys_getdirentries(Thread *thread, sint fd, ptr buf, uint count, ptr basep); -SysResult sys_freebsd6_mmap(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags, sint fd, sint pad, off_t pos); +SysResult sys_freebsd6_mmap(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, + sint pad, off_t pos); SysResult sys_freebsd6_lseek(Thread *thread, sint fd, sint pad, off_t offset, sint whence); SysResult sys_freebsd6_truncate(Thread *thread, ptr path, sint pad, @@ -209,8 +216,8 @@ SysResult sys_freebsd6_ftruncate(Thread *thread, sint fd, sint pad, SysResult sys___sysctl(Thread *thread, ptr name, uint namelen, ptr old, ptr oldenp, ptr new_, size_t newlen); -SysResult sys_mlock(Thread *thread, ptr addr, size_t len); -SysResult sys_munlock(Thread *thread, ptr addr, size_t len); +SysResult sys_mlock(Thread *thread, uintptr_t addr, size_t len); +SysResult sys_munlock(Thread *thread, uintptr_t addr, size_t len); SysResult sys_undelete(Thread *thread, ptr path); SysResult sys_futimes(Thread *thread, sint fd, ptr tptr); SysResult sys_getpgid(Thread *thread, pid_t pid); @@ -246,7 +253,7 @@ SysResult sys_ktimer_getoverrun(Thread *thread, sint timerid); SysResult sys_nanosleep(Thread *thread, cptr rqtp, ptr rmtp); SysResult sys_ntp_gettime(Thread *thread, ptr ntvp); -SysResult sys_minherit(Thread *thread, ptr addr, size_t len, +SysResult sys_minherit(Thread *thread, uintptr_t addr, size_t len, sint inherit); SysResult sys_rfork(Thread *thread, sint flags); SysResult sys_openbsd_poll(Thread *thread, ptr fds, uint nfds, @@ -519,8 +526,9 @@ SysResult sys_pread(Thread *thread, sint fd, ptr buf, size_t nbyte, off_t offset); SysResult sys_pwrite(Thread *thread, sint fd, ptr buf, size_t nbyte, off_t offset); -SysResult sys_mmap(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags, sint fd, off_t pos); +SysResult sys_mmap(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, off_t pos); SysResult sys_lseek(Thread *thread, sint fd, off_t offset, sint whence); SysResult sys_truncate(Thread *thread, ptr path, off_t length); SysResult sys_ftruncate(Thread *thread, sint fd, off_t length); @@ -621,7 +629,9 @@ SysResult sys_socketex(Thread *thread, ptr name, sint domain, SysResult sys_socketclose(Thread *thread, sint fd); SysResult sys_netgetiflist(Thread *thread /* TODO */); SysResult sys_kqueueex(Thread *thread, ptr name, sint flags); -SysResult sys_mtypeprotect(Thread *thread /* TODO */); +SysResult sys_mtypeprotect(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, + rx::EnumBitSet prot); SysResult sys_regmgr_call(Thread *thread, uint32_t op, uint32_t id, ptr result, ptr value, uint64_t len); SysResult sys_jitshm_create(Thread *thread /* TODO */); @@ -644,9 +654,10 @@ SysResult sys_evf_set(Thread *thread, sint id, uint64_t value); SysResult sys_evf_clear(Thread *thread, sint id, uint64_t value); SysResult sys_evf_cancel(Thread *thread, sint id, uint64_t value, ptr pNumWaitThreads); -SysResult sys_query_memory_protection(Thread *thread, ptr address, - ptr protection); -SysResult sys_batch_map(Thread *thread, sint unk, sint flags, +SysResult sys_query_memory_protection(Thread *thread, uintptr_t address, + ptr protection); +SysResult sys_batch_map(Thread *thread, sint unk, + rx::EnumBitSet flags, ptr entries, sint entriesCount, ptr processedCount); SysResult sys_osem_create(Thread *thread, ptr name, uint attrs, @@ -679,7 +690,7 @@ SysResult sys_budget_delete(Thread *thread, sint budget); SysResult sys_budget_get(Thread *thread, sint id, ptr budgetInfo, ptr count); SysResult sys_budget_set(Thread *thread, sint budget); -SysResult sys_virtual_query(Thread *thread, ptr addr, uint64_t unk, +SysResult sys_virtual_query(Thread *thread, uintptr_t addr, uint64_t unk, ptr info, size_t infosz); SysResult sys_mdbg_call(Thread *thread /* TODO */); SysResult sys_obs_sblock_create(Thread *thread /* TODO */); @@ -752,8 +763,10 @@ SysResult sys_opmc_set_hw(Thread *thread /* TODO */); SysResult sys_opmc_get_hw(Thread *thread /* TODO */); SysResult sys_get_cpu_usage_all(Thread *thread, uint32_t unk, ptr result); -SysResult sys_mmap_dmem(Thread *thread, caddr_t addr, size_t len, - sint memoryType, sint prot, sint flags, +SysResult sys_mmap_dmem(Thread *thread, uintptr_t addr, size_t len, + MemoryType memoryType, + rx::EnumBitSet prot, + rx::EnumBitSet flags, off_t directMemoryStart); SysResult sys_physhm_open(Thread *thread /* TODO */); SysResult sys_physhm_unlink(Thread *thread /* TODO */); @@ -785,9 +798,11 @@ SysResult sys_budget_get_ptype_of_budget(Thread *thread, sint budgetId); SysResult sys_prepare_to_resume_process(Thread *thread /* TODO */); SysResult sys_process_terminate(Thread *thread /* TODO */); SysResult sys_blockpool_open(Thread *thread); -SysResult sys_blockpool_map(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags); -SysResult sys_blockpool_unmap(Thread *thread, caddr_t addr, size_t len, +SysResult sys_blockpool_map(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, + rx::EnumBitSet prot, + rx::EnumBitSet flags); +SysResult sys_blockpool_unmap(Thread *thread, uintptr_t addr, size_t len, sint flags); SysResult sys_dynlib_get_info_for_libdbg(Thread *thread /* TODO */); SysResult sys_blockpool_batch(Thread *thread /* TODO */); diff --git a/kernel/orbis/include/orbis/thread/Process.hpp b/kernel/orbis/include/orbis/thread/Process.hpp index 245ddb7ae..6a8d8d19f 100644 --- a/kernel/orbis/include/orbis/thread/Process.hpp +++ b/kernel/orbis/include/orbis/thread/Process.hpp @@ -102,6 +102,8 @@ struct Process final { rx::RcIdMap threadsMap; rx::RcIdMap fileDescriptors; + rx::AddressRange libkernelRange; + // Named objects for debugging rx::shared_mutex namedObjMutex; kmap namedObjNames; @@ -109,10 +111,6 @@ struct Process final { kmap sigActions; - // Named memory ranges for debugging - rx::shared_mutex namedMemMutex; - kmap namedMem; - // FIXME: implement process destruction void incRef() {} void decRef() {} @@ -126,6 +124,8 @@ struct Process final { ref) { return ref.get(storage); } + + Budget *getBudget() const; }; pid_t allocatePid(); diff --git a/kernel/orbis/include/orbis/thread/ProcessOps.hpp b/kernel/orbis/include/orbis/thread/ProcessOps.hpp index 717fb8c55..7a3eeb4f6 100644 --- a/kernel/orbis/include/orbis/thread/ProcessOps.hpp +++ b/kernel/orbis/include/orbis/thread/ProcessOps.hpp @@ -10,34 +10,13 @@ struct Thread; struct Module; struct timespec; struct File; +namespace vmem { struct MemoryProtection; +} struct IoVec; struct UContext; struct ProcessOps { - SysResult (*mmap)(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags, sint fd, off_t pos); - SysResult (*dmem_mmap)(Thread *thread, caddr_t addr, size_t len, - sint memoryType, sint prot, sint flags, - off_t directMemoryStart); - SysResult (*munmap)(Thread *thread, ptr addr, size_t len); - SysResult (*msync)(Thread *thread, ptr addr, size_t len, sint flags); - SysResult (*mprotect)(Thread *thread, ptr addr, size_t len, - sint prot); - SysResult (*minherit)(Thread *thread, ptr addr, size_t len, - sint inherit); - SysResult (*madvise)(Thread *thread, ptr addr, size_t len, sint behav); - SysResult (*mincore)(Thread *thread, ptr addr, size_t len, - ptr vec); - SysResult (*mlock)(Thread *thread, ptr addr, size_t len); - SysResult (*mlockall)(Thread *thread, sint how); - SysResult (*munlockall)(Thread *thread); - SysResult (*munlock)(Thread *thread, ptr addr, size_t len); - SysResult (*virtual_query)(Thread *thread, ptr addr, sint flags, - ptr info, ulong infoSize); - SysResult (*query_memory_protection)(Thread *thread, ptr address, - ptr protection); - SysResult (*open)(Thread *thread, ptr path, sint flags, sint mode, rx::Ref *file); SysResult (*shm_open)(Thread *thread, const char *path, sint flags, sint mode, @@ -46,10 +25,6 @@ struct ProcessOps { SysResult (*mkdir)(Thread *thread, ptr path, sint mode); SysResult (*rmdir)(Thread *thread, ptr path); SysResult (*rename)(Thread *thread, ptr from, ptr to); - SysResult (*blockpool_open)(Thread *thread, rx::Ref *file); - SysResult (*blockpool_map)(Thread *thread, caddr_t addr, size_t len, - sint prot, sint flags); - SysResult (*blockpool_unmap)(Thread *thread, caddr_t addr, size_t len); SysResult (*socket)(Thread *thread, ptr name, sint domain, sint type, sint protocol, rx::Ref *file); SysResult (*socketpair)(Thread *thread, sint domain, sint type, sint protocol, diff --git a/kernel/orbis/include/orbis/thread/Thread.hpp b/kernel/orbis/include/orbis/thread/Thread.hpp index 7963c4335..9df49bf45 100644 --- a/kernel/orbis/include/orbis/thread/Thread.hpp +++ b/kernel/orbis/include/orbis/thread/Thread.hpp @@ -32,8 +32,8 @@ struct Thread final { uint64_t retval[2]{}; void *context{}; kvector altStack; - ptr stackStart; - ptr stackEnd; + uint64_t stackStart; + uint64_t stackEnd; uint64_t fsBase{}; uint64_t gsBase{}; rx::StaticString<31> name; @@ -97,6 +97,7 @@ struct Thread final { }; Thread *createThread(Process *process, std::string_view name); +uintptr_t getCallerAddress(Thread *thread); extern thread_local Thread *g_currentThread; diff --git a/kernel/orbis/include/orbis/thread/sysent.hpp b/kernel/orbis/include/orbis/thread/sysent.hpp index 01f37e1b3..719f45c70 100644 --- a/kernel/orbis/include/orbis/thread/sysent.hpp +++ b/kernel/orbis/include/orbis/thread/sysent.hpp @@ -1,6 +1,7 @@ #pragma once #include "orbis-config.hpp" +#include namespace orbis { struct Thread; @@ -9,6 +10,7 @@ using sy_call_t = SysResult(Thread *, uint64_t *); struct sysent { sint narg; sy_call_t *call; + std::string (*format)(uint64_t *); }; struct sysentvec { diff --git a/kernel/orbis/include/orbis/thread/types.hpp b/kernel/orbis/include/orbis/thread/types.hpp index 438b6fa01..c509d8943 100644 --- a/kernel/orbis/include/orbis/thread/types.hpp +++ b/kernel/orbis/include/orbis/thread/types.hpp @@ -15,7 +15,7 @@ struct rtprio { struct thr_param { ptr start_func; ptr arg; - ptr stack_base; + uint64_t stack_base; size_t stack_size; ptr tls_base; size_t tls_size; diff --git a/kernel/orbis/include/orbis/ucontext.hpp b/kernel/orbis/include/orbis/ucontext.hpp index fa47726a3..9066510ee 100644 --- a/kernel/orbis/include/orbis/ucontext.hpp +++ b/kernel/orbis/include/orbis/ucontext.hpp @@ -47,7 +47,7 @@ struct MContext { }; struct Stack { - ptr sp; + uintptr_t sp; size_t size; sint flags; sint align; diff --git a/kernel/orbis/include/orbis/vm.hpp b/kernel/orbis/include/orbis/vm.hpp deleted file mode 100644 index 8e55bc27d..000000000 --- a/kernel/orbis/include/orbis/vm.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "orbis-config.hpp" - -namespace orbis { -struct MemoryProtection { - uint64_t startAddress; - uint64_t endAddress; - int32_t prot; -}; - -static_assert(sizeof(MemoryProtection) == 24); -} // namespace orbis diff --git a/kernel/orbis/include/orbis/vmem.hpp b/kernel/orbis/include/orbis/vmem.hpp index 3a9724467..461eb274c 100644 --- a/kernel/orbis/include/orbis/vmem.hpp +++ b/kernel/orbis/include/orbis/vmem.hpp @@ -1,21 +1,22 @@ #pragma once +#include "MemoryType.hpp" #include "kernel/MemoryResource.hpp" #include "orbis-config.hpp" #include "rx/AddressRange.hpp" #include "rx/EnumBitSet.hpp" #include "rx/StaticString.hpp" +#include "rx/mem.hpp" #include namespace orbis { using kernel::AllocationFlags; -} -namespace orbis { -struct IoDevice; +struct File; struct Process; +} // namespace orbis -namespace vmem { +namespace orbis::vmem { static constexpr auto kPageSize = 16 * 1024; enum class Protection { @@ -28,33 +29,85 @@ enum class Protection { bitset_last = GpuWrite }; -enum class BlockFlags { +enum class BlockFlags : std::uint8_t { FlexibleMemory, DirectMemory, Stack, PooledMemory, Commited, - Allocated, - bitset_last = Allocated + bitset_last = Commited }; +enum class BlockFlagsEx : std::uint8_t { + Allocated, + Private, + Shared, + PoolControl, + Reserved, + + bitset_last = Reserved +}; + +enum class MapFlags { + Shared = 0, + Private = 1, + Fixed = 4, + Rename = 5, + NoReserve = 6, + NoOverwrite = 7, + Void = 8, + HasSemaphore = 9, + Stack = 10, + NoSync = 11, + Anon = 12, + System = 13, + AllAvailable = 14, + NoCore = 17, + PrefaultRead = 18, + Self = 19, + NoCoalesce = 22, + + bitset_last = NoCoalesce +}; + +inline constexpr std::uint32_t kMapFlagsAlignShift = 24; +inline constexpr std::uint32_t kMapFlagsAlignMask = 0x1f << kMapFlagsAlignShift; + inline constexpr auto kProtCpuReadWrite = Protection::CpuRead | Protection::CpuWrite; inline constexpr auto kProtCpuAll = Protection::CpuRead | Protection::CpuWrite | Protection::CpuExec; inline constexpr auto kProtGpuAll = Protection::GpuRead | Protection::GpuWrite; +inline std::pair> +unpackMapFlags(rx::EnumBitSet flags, std::uint64_t minAlignment) { + std::uint64_t alignment = minAlignment; + + if (auto align = + (flags.toUnderlying() & kMapFlagsAlignMask) >> kMapFlagsAlignShift) { + alignment = std::uint64_t(1) << align; + flags = rx::EnumBitSet::fromUnderlying( + flags.toUnderlying() & ~kMapFlagsAlignMask); + + if (alignment < minAlignment) { + alignment = minAlignment; + } + } + + return {alignment, flags}; +} + #pragma pack(push, 1) struct QueryResult { uint64_t start; uint64_t end; uint64_t offset; - uint32_t protection; - uint32_t memoryType; - uint32_t flags; + rx::EnumBitSet protection; + MemoryType memoryType; + rx::EnumBitSet flags; rx::StaticCString<32> name; - uint32_t _padding; + char _padding[7]; }; static_assert(sizeof(QueryResult) == 72); @@ -69,25 +122,81 @@ struct MemoryProtection { static_assert(sizeof(MemoryProtection) == 24); #pragma pack(pop) +inline constexpr rx::EnumBitSet +toCpuProtection(rx::EnumBitSet prot) { + rx::EnumBitSet result{}; + if (prot & Protection::CpuRead) { + result |= rx::mem::Protection::R; + } + if (prot & Protection::CpuWrite) { + result |= rx::mem::Protection::W; + } + if (prot & Protection::CpuExec) { + result |= rx::mem::Protection::X; + } + return result; +} + +inline constexpr rx::EnumBitSet +toGpuProtection(rx::EnumBitSet prot) { + rx::EnumBitSet result{}; + if (prot & Protection::GpuRead) { + result |= rx::mem::Protection::R; + } + if (prot & Protection::GpuWrite) { + result |= rx::mem::Protection::W; + } + return result; +} + void initialize(Process *process, bool force = false); void fork(Process *process, Process *parentThread); std::pair reserve(Process *process, std::uint64_t addressHint, std::uint64_t size, - rx::EnumBitSet allocFlags); + rx::EnumBitSet allocFlags, + rx::EnumBitSet blockFlagsEx = {}, + std::uint64_t alignment = kPageSize); std::pair -map(Process *process, std::uint64_t addressHint, std::uint64_t size, - rx::EnumBitSet allocFlags, - rx::EnumBitSet prot = {}, - rx::EnumBitSet blockFlags = {}, - std::uint64_t alignment = kPageSize, std::string_view name = {}, - IoDevice *device = nullptr, std::int64_t deviceOffset = 0); +mapFile(Process *process, std::uint64_t addressHint, std::uint64_t size, + rx::EnumBitSet allocFlags, + rx::EnumBitSet prot, rx::EnumBitSet blockFlags, + rx::EnumBitSet blockFlagsEx, File *file, + std::uint64_t fileOffset, std::string_view name = {}, + std::uint64_t alignment = kPageSize, + MemoryType type = MemoryType::Invalid); + +std::pair +mapDirect(Process *process, std::uint64_t addressHint, + rx::AddressRange directRange, rx::EnumBitSet prot, + rx::EnumBitSet allocFlags, + std::string_view name = {}, std::uint64_t alignment = kPageSize, + MemoryType type = MemoryType::Invalid); + +std::pair +mapFlex(Process *process, std::uint64_t size, rx::EnumBitSet prot, + std::uint64_t addressHint = 0, + rx::EnumBitSet allocFlags = {}, + rx::EnumBitSet blockFlags = {}, std::string_view name = {}, + std::uint64_t alignment = kPageSize); + +std::pair +commitPooled(Process *process, rx::AddressRange addressRange, MemoryType type, + rx::EnumBitSet prot); +ErrorCode decommitPooled(Process *process, rx::AddressRange addressRange); + +ErrorCode protect(Process *process, rx::AddressRange range, + rx::EnumBitSet prot); ErrorCode unmap(Process *process, rx::AddressRange range); ErrorCode setName(Process *process, rx::AddressRange range, std::string_view name); -std::optional query(Process *process, std::uint64_t address); +ErrorCode setType(Process *process, rx::AddressRange range, MemoryType type); +ErrorCode setTypeAndProtect(Process *process, rx::AddressRange range, + MemoryType type, rx::EnumBitSet prot); +std::optional query(Process *process, std::uint64_t address, + bool lowerBound = false); std::optional queryProtection(Process *process, - std::uint64_t address); -} // namespace vmem -} // namespace orbis + std::uint64_t address, + bool lowerBound = false); +} // namespace orbis::vmem diff --git a/kernel/orbis/src/IoDevice.cpp b/kernel/orbis/src/IoDevice.cpp index 0840f2a2e..bd0dfecd1 100644 --- a/kernel/orbis/src/IoDevice.cpp +++ b/kernel/orbis/src/IoDevice.cpp @@ -1,6 +1,9 @@ #include "IoDevice.hpp" +#include "file.hpp" +#include "rx/Mappable.hpp" #include "thread/Thread.hpp" #include "utils/Logs.hpp" +#include "vmem.hpp" static std::string iocGroupToString(unsigned iocGroup) { if (iocGroup >= 128) { @@ -64,6 +67,23 @@ static std::string iocGroupToString(unsigned iocGroup) { return "'?'"; } +orbis::ErrorCode +orbis::IoDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, File *file, + Process *) { + if (!file->dirEntries.empty()) { + return orbis::ErrorCode::ISDIR; + } + + if (!file->hostFd) { + return ErrorCode::NOTSUP; + } + + auto errc = file->hostFd.map(range, offset, vmem::toCpuProtection(protection), + orbis::vmem::kPageSize); + return orbis::toErrorCode(errc); +} + orbis::ErrorCode orbis::IoDevice::ioctl(std::uint64_t request, orbis::ptr argp, Thread *thread) { auto group = iocGroupToString(ioctl::group(request)); diff --git a/kernel/orbis/src/blockpool.cpp b/kernel/orbis/src/blockpool.cpp new file mode 100644 index 000000000..c2389e954 --- /dev/null +++ b/kernel/orbis/src/blockpool.cpp @@ -0,0 +1,370 @@ +#include "blockpool.hpp" +#include "KernelAllocator.hpp" +#include "KernelObject.hpp" +#include "dmem.hpp" +#include "pmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/MemoryTable.hpp" +#include "rx/die.hpp" +#include "thread/Process.hpp" +#include "vmem.hpp" +#include +#include +#include + +// FIXME: remove +namespace amdgpu { +void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset); +void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange); +void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot); +} // namespace amdgpu + +struct PooledMemoryResource { + struct CommitedBlock { + std::uint64_t pmemAddress; + orbis::MemoryType type; + bool operator==(const CommitedBlock &) const = default; + }; + + std::uint64_t total = 0; + std::uint64_t used = 0; + orbis::kvector freeBlocks; + rx::MemoryTableWithPayload usedBlocks; + orbis::kvector reservedPages; + + void clear() { + total = 0; + used = 0; + freeBlocks.clear(); + usedBlocks.clear(); + reservedPages.clear(); + } + + void addFreeBlock(rx::AddressRange dmemRange) { + if (freeBlocks.empty()) { + freeBlocks.push_back(dmemRange); + return; + } + + auto it = std::upper_bound( + freeBlocks.begin(), freeBlocks.end(), dmemRange.beginAddress(), + [](auto lhs, auto rhs) { + if constexpr (requires { lhs.beginAddress() < rhs; }) { + return lhs.beginAddress() < rhs; + } else { + return lhs < rhs.beginAddress(); + } + }); + + if (it != freeBlocks.end() && + dmemRange.endAddress() == it->beginAddress()) { + *it = rx::AddressRange::fromBeginEnd(it->beginAddress(), + dmemRange.endAddress()); + return; + } + + if (it != freeBlocks.begin()) { + auto prev = std::prev(it); + + if (prev->endAddress() == dmemRange.beginAddress()) { + *prev = rx::AddressRange::fromBeginEnd(prev->beginAddress(), + dmemRange.endAddress()); + return; + } + } + + freeBlocks.insert(it, dmemRange); + return; + } + + void expand(rx::AddressRange dmemRange) { + addFreeBlock(dmemRange); + total += dmemRange.size(); + } + + orbis::ErrorCode reserve(std::size_t size) { + if (size > total) { + return orbis::ErrorCode::INVAL; + } + + used += size; + return {}; + } + + std::pair reservePage() { + if (total - used < orbis::dmem::kPageSize) { + return {{}, orbis::ErrorCode::INVAL}; + } + + used += orbis::dmem::kPageSize; + auto &block = freeBlocks.back(); + auto allocatedPage = block.beginAddress(); + + if (block.endAddress() == allocatedPage + orbis::dmem::kPageSize) { + freeBlocks.pop_back(); + } else { + block = rx::AddressRange::fromBeginEnd( + allocatedPage + orbis::dmem::kPageSize, block.endAddress()); + } + + reservedPages.push_back(allocatedPage); + return {allocatedPage, {}}; + } + + std::pair releasePage() { + if (reservedPages.empty()) { + return {{}, orbis::ErrorCode::INVAL}; + } + + used -= orbis::dmem::kPageSize; + total -= orbis::dmem::kPageSize; + + auto address = reservedPages.back(); + reservedPages.pop_back(); + return {address, {}}; + } + + void moveTo(PooledMemoryResource &other, std::size_t size) { + while (size > 0 && total > 0) { + auto &block = freeBlocks.back(); + + if (block.size() > size) { + total -= size; + + auto moveRange = + rx::AddressRange::fromBeginSize(block.beginAddress(), size); + + other.expand(moveRange); + + block = rx::AddressRange::fromBeginEnd(moveRange.endAddress(), + block.endAddress()); + break; + } + + total -= block.size(); + size -= block.size(); + other.expand(block); + freeBlocks.pop_back(); + } + } + + void commit(orbis::Process *process, rx::AddressRange virtualRange, + orbis::MemoryType type, + rx::EnumBitSet protection) { + while (virtualRange.isValid()) { + auto &block = freeBlocks.back(); + + if (block.size() >= virtualRange.size()) [[likely]] { + auto mapPhysicalRange = rx::AddressRange::fromBeginSize( + block.beginAddress(), virtualRange.size()); + auto errc = + orbis::pmem::map(virtualRange.beginAddress(), mapPhysicalRange, + orbis::vmem::toCpuProtection(protection)); + + rx::dieIf(errc != orbis::ErrorCode{}, + "blockpool: failed to map physical memory"); + amdgpu::mapMemory(process->pid, virtualRange, type, protection, + block.beginAddress()); + + if (mapPhysicalRange.endAddress() == block.endAddress()) { + freeBlocks.pop_back(); + } else { + block = rx::AddressRange::fromBeginEnd(mapPhysicalRange.endAddress(), + block.endAddress()); + } + + usedBlocks.map(virtualRange, + {.pmemAddress = block.beginAddress(), .type = type}, + false); + used += virtualRange.size(); + break; + } + + auto mapVirtualRange = rx::AddressRange::fromBeginSize( + virtualRange.beginAddress(), block.size()); + + virtualRange = rx::AddressRange::fromBeginEnd( + mapVirtualRange.endAddress(), virtualRange.endAddress()); + + auto errc = orbis::pmem::map(mapVirtualRange.beginAddress(), block, + orbis::vmem::toCpuProtection(protection)); + + rx::dieIf(errc != orbis::ErrorCode{}, + "blockpool: failed to map physical memory"); + amdgpu::mapMemory(process->pid, mapVirtualRange, type, protection, + block.beginAddress()); + + usedBlocks.map(mapVirtualRange, + {.pmemAddress = block.beginAddress(), .type = type}, + false); + freeBlocks.pop_back(); + } + } + + void decommit(orbis::Process *process, rx::AddressRange virtualRange) { + auto it = usedBlocks.lowerBound(virtualRange.beginAddress()); + + if (it != usedBlocks.end() && + it.beginAddress() < virtualRange.beginAddress()) { + auto itRange = it.range(); + auto decommitRange = itRange.intersection(virtualRange); + used -= decommitRange.size(); + + auto decommitPmemRange = rx::AddressRange::fromBeginSize( + it->pmemAddress + + (virtualRange.beginAddress() - itRange.beginAddress()), + decommitRange.size()); + addFreeBlock(decommitPmemRange); + usedBlocks.unmap(decommitRange); + amdgpu::unmapMemory(process->pid, decommitRange); + ++it; + } + + while (it != usedBlocks.end() && + it.beginAddress() < virtualRange.endAddress()) { + auto itRange = it.range(); + auto decommitRange = itRange.intersection(virtualRange); + used -= decommitRange.size(); + + addFreeBlock(rx::AddressRange::fromBeginSize(it->pmemAddress, + decommitRange.size())); + amdgpu::unmapMemory(process->pid, decommitRange); + + if (itRange == decommitRange) { + it = usedBlocks.unmap(it); + } else { + usedBlocks.unmap(decommitRange); + break; + } + } + } + + std::optional getMemoryType(std::uint64_t address) { + auto it = usedBlocks.queryArea(address); + if (it == usedBlocks.end()) { + return {}; + } + return it->type; + } +}; + +static auto g_blockpool = orbis::createGlobalObject< + kernel::LockableKernelObject>(); + +static auto g_cachedBlockpool = orbis::createGlobalObject< + kernel::LockableKernelObject>(); + +void orbis::blockpool::clear() { + std::scoped_lock lock(*g_blockpool, *g_cachedBlockpool); + + g_blockpool->clear(); + g_cachedBlockpool->clear(); +} + +orbis::ErrorCode orbis::blockpool::expand(rx::AddressRange dmemRange) { + std::scoped_lock lock(*g_blockpool); + g_blockpool->expand(dmemRange); + return {}; +} + +orbis::ErrorCode orbis::blockpool::allocateControlBlock() { + std::scoped_lock cachedLock(*g_cachedBlockpool); + if (g_cachedBlockpool->used < g_cachedBlockpool->total) { + return g_cachedBlockpool->reservePage().second; + } + + std::scoped_lock lock(*g_blockpool); + + if (g_blockpool->total - g_blockpool->used < dmem::kPageSize) { + return ErrorCode::INVAL; + } + + g_blockpool->moveTo(*g_cachedBlockpool, dmem::kPageSize); + return g_cachedBlockpool->reservePage().second; +} + +orbis::ErrorCode orbis::blockpool::releaseControlBlock() { + std::scoped_lock lock(*g_cachedBlockpool); + + // control block is always cached + if (!g_cachedBlockpool->reservedPages.empty()) { + auto [page, errc] = g_cachedBlockpool->releasePage(); + if (errc != ErrorCode{}) { + return errc; + } + + g_cachedBlockpool->expand( + rx::AddressRange::fromBeginSize(page, dmem::kPageSize)); + return {}; + } + + return ErrorCode::INVAL; +} + +orbis::ErrorCode +orbis::blockpool::commit(Process *process, rx::AddressRange vmemRange, + MemoryType type, + rx::EnumBitSet protection) { + auto pool = + type == MemoryType::WbOnion ? g_cachedBlockpool : g_blockpool; + auto otherPool = + type == MemoryType::WbOnion ? g_blockpool : g_cachedBlockpool; + + std::scoped_lock lock(*pool); + + if (auto avail = pool->total - pool->used; avail < vmemRange.size()) { + // try to steal free blocks from other pool + + std::scoped_lock lock(*otherPool); + auto pullSize = vmemRange.size() - avail; + + if (otherPool->total - otherPool->used < pullSize) { + return ErrorCode::NOMEM; + } + + otherPool->moveTo(*pool, pullSize); + } + + pool->commit(process, vmemRange, type, protection); + return {}; +} + +void orbis::blockpool::decommit(Process *process, rx::AddressRange vmemRange) { + std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool); + g_cachedBlockpool->decommit(process, vmemRange); + g_blockpool->decommit(process, vmemRange); +} + +std::optional +orbis::blockpool::getType(std::uint64_t address) { + { + std::scoped_lock lock(*g_cachedBlockpool); + + if (auto result = g_cachedBlockpool->getMemoryType(address)) { + return result; + } + } + + std::scoped_lock lock(*g_blockpool); + return g_blockpool->getMemoryType(address); +} + +orbis::blockpool::BlockStats orbis::blockpool::stats() { + BlockStats result{}; + + { + std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool); + result.availFlushedBlocks = + (g_blockpool->total - g_blockpool->used) / dmem::kPageSize; + result.availCachedBlocks = + (g_cachedBlockpool->total - g_cachedBlockpool->used) / dmem::kPageSize; + result.commitFlushedBlocks = g_blockpool->used / dmem::kPageSize; + result.commitCachedBlocks = g_cachedBlockpool->used / dmem::kPageSize; + } + + return result; +} diff --git a/kernel/orbis/src/dmem.cpp b/kernel/orbis/src/dmem.cpp new file mode 100644 index 000000000..546c322d7 --- /dev/null +++ b/kernel/orbis/src/dmem.cpp @@ -0,0 +1,610 @@ +#include "dmem.hpp" +#include "KernelAllocator.hpp" +#include "KernelObject.hpp" +#include "error.hpp" +#include "kernel/KernelObject.hpp" +#include "pmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/FileLock.hpp" +#include "rx/Serializer.hpp" +#include "rx/die.hpp" +#include "rx/print.hpp" +#include "utils/Logs.hpp" +#include "vmem.hpp" +#include +#include +#include +#include + +struct DirectMemoryAllocation { + static constexpr std::uint32_t kAllocatedBit = 1 << 31; + static constexpr std::uint32_t kPooledBit = 1 << 30; + std::uint32_t type = 0; + + [[nodiscard]] bool isPooled() const { return type & kPooledBit; } + void markAsPooled() { type |= kPooledBit | kAllocatedBit; } + + void setMemoryType(orbis::MemoryType memoryType) { + type = (std::to_underlying(memoryType) & ~(kAllocatedBit | kPooledBit)) | + (type & kPooledBit) | kAllocatedBit; + } + + [[nodiscard]] orbis::MemoryType getMemoryType() const { + return static_cast(type & ~(kAllocatedBit | kPooledBit)); + } + + [[nodiscard]] bool isAllocated() const { return (type & kAllocatedBit) != 0; } + [[nodiscard]] bool isRelated(const DirectMemoryAllocation &other, + rx::AddressRange, rx::AddressRange) const { + return type == other.type; + } + + [[nodiscard]] DirectMemoryAllocation + merge(const DirectMemoryAllocation &other, rx::AddressRange, + rx::AddressRange) const { + return other; + } + + bool operator==(const DirectMemoryAllocation &) const = default; +}; + +struct DirectMemoryResourceState { + std::uint64_t pmemOffset{}; + std::uint64_t dmemTotalSize{}; + std::uint64_t dmemReservedSize{}; +}; + +struct DirectMemoryResource + : kernel::AllocableResource, + DirectMemoryResourceState { + using BaseResource = + kernel::AllocableResource; + + void create(std::size_t size) { + auto [pmemRange, errc] = orbis::pmem::allocate(0, size, {}, 64 * 1024); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to allocate direct memory: size {:x}, error {}", size, + static_cast(errc)); + + dmemTotalSize = pmemRange.size(); + pmemOffset = pmemRange.beginAddress(); + auto dmemResourceRange = + rx::AddressRange::fromBeginSize(0, pmemRange.size()); + + if (auto errc = BaseResource::create(dmemResourceRange); + errc != std::errc{}) { + rx::die("failed to create direct memory resource {:x}, error {}", + pmemRange.size(), errc); + } + } + + orbis::ErrorCode clear() { + auto result = BaseResource::create( + rx::AddressRange::fromBeginSize(0, dmemTotalSize + dmemReservedSize)); + + if (result != std::errc{}) { + return orbis::toErrorCode(result); + } + dmemReservedSize = 0; + return {}; + } + + void serialize(rx::Serializer &s) const { + s.serialize(static_cast(*this)); + BaseResource::serialize(s); + } + + void deserialize(rx::Deserializer &d) { + d.deserialize(static_cast(*this)); + BaseResource::deserialize(d); + } +}; + +static std::array g_dmemPools = { + orbis::createGlobalObject< + kernel::LockableKernelObject>(), + orbis::createGlobalObject< + kernel::LockableKernelObject>(), + orbis::createGlobalObject< + kernel::LockableKernelObject>(), +}; + +orbis::ErrorCode orbis::dmem::initialize() { + g_dmemPools[0]->create(0x120000000); + g_dmemPools[1]->create(0x1000000); + g_dmemPools[2]->create(0x1000000); + + return {}; +} + +orbis::ErrorCode orbis::dmem::clear(unsigned dmemIndex) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + { + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + auto result = dmem->clear(); + if (result != orbis::ErrorCode{}) { + return result; + } + } + + return {}; +} + +static void dmemDump(unsigned dmemIndex, std::string_view message = {}) { + auto dmem = g_dmemPools[dmemIndex]; + + rx::ScopedFileLock lock(stderr); + + rx::println(stderr, "dmem0 {}", message); + for (auto alloc : *dmem) { + rx::println(stderr, " {:012x}-{:012x}: {}{} {}", alloc.beginAddress(), + alloc.endAddress(), "_A"[alloc->isAllocated()], + "_P"[alloc->isPooled()], alloc->getMemoryType()); + } +} + +std::pair +orbis::dmem::allocate(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t len, MemoryType memoryType, + std::uint64_t alignment, bool pooled) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + if (searchRange.endAddress() != 0 && + (!searchRange.isValid() || searchRange.size() < len)) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress(), len); + return {{}, orbis::ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + alignment = alignment == 0 ? kPageSize : alignment; + len = rx::alignUp(len, dmem::kPageSize); + + if (searchRange.endAddress() == 0) { + searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(), + dmem->dmemTotalSize - + dmem->dmemReservedSize); + } + + if (!searchRange.isValid() || searchRange.size() < len) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress(), len); + return {{}, orbis::ErrorCode::INVAL}; + } + + if (len == 0) { + ORBIS_LOG_ERROR(__FUNCTION__, "len is 0", searchRange.beginAddress(), + searchRange.endAddress(), len); + + return {{}, ErrorCode::INVAL}; + } + + if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.beginAddress(), searchRange.endAddress(), + dmem->dmemTotalSize); + return {{}, ErrorCode::INVAL}; + } + + DirectMemoryAllocation allocation; + allocation.setMemoryType(memoryType); + if (pooled) { + allocation.markAsPooled(); + } + + auto allocResult = dmem->map(searchRange.beginAddress(), len, allocation, + AllocationFlags::Dry, alignment); + if (allocResult.errc != std::errc{}) { + return {{}, toErrorCode(allocResult.errc)}; + } + + auto result = allocResult.range.beginAddress(); + + auto commitResult = + dmem->map(result, len, allocation, AllocationFlags::Fixed, alignment); + + rx::dieIf(commitResult.errc != std::errc{}, + "dmem: failed to commit memory, error {}", commitResult.errc); + + dmemDump(dmemIndex, + rx::format("allocated {:x}-{:x}", allocResult.range.beginAddress(), + allocResult.range.endAddress())); + + return {result, {}}; +} + +orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex, + rx::AddressRange range, bool pooled) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + if (range.beginAddress() % kPageSize || range.endAddress() % kPageSize || + !range.isValid()) { + return ErrorCode::INVAL; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + constexpr auto razorGpuMemory = + rx::AddressRange::fromBeginSize(0x3000000000, 0x20000000); + + if (dmemIndex == 0 && razorGpuMemory.contains(range.beginAddress())) { + return ErrorCode::OPNOTSUPP; + } + + auto it = dmem->query(range.beginAddress()); + + if (it == dmem->end() || !it->isAllocated()) { + return ErrorCode::NOENT; + } + + if (it->isPooled() && !pooled) { + return ErrorCode::NOENT; + } + + DirectMemoryAllocation allocation{}; + auto result = dmem->map(range.beginAddress(), range.size(), allocation, + AllocationFlags::Fixed, vmem::kPageSize); + + if (result.errc != std::errc{}) { + return toErrorCode(result.errc); + } + + return {}; +} + +std::pair +orbis::dmem::reserveSystem(unsigned dmemIndex, std::uint64_t size) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + if (size == 0 || size % vmem::kPageSize) { + return {{}, ErrorCode::INVAL}; + } + + if (dmem->dmemReservedSize + size > dmem->dmemTotalSize) { + return {{}, ErrorCode::NOMEM}; + } + + DirectMemoryAllocation alloc; + alloc.setMemoryType(MemoryType::WbOnion); + auto result = dmem->map(0, size, alloc, AllocationFlags::Stack, kPageSize); + + if (result.errc != std::errc{}) { + return {{}, toErrorCode(result.errc)}; + } + + dmem->dmemReservedSize += size; + return {result.range.beginAddress() | 0x4000000000, {}}; +} + +std::pair +orbis::dmem::reservePooled(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t size, std::uint64_t alignment) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + alignment = alignment == 0 ? kPageSize : alignment; + + if (alignment % kPageSize) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + + if (searchRange.endAddress() > dmem->dmemTotalSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize); + return {{}, ErrorCode::INVAL}; + } + + if (!searchRange.isValid() && + searchRange.beginAddress() != searchRange.endAddress()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, ErrorCode::INVAL}; + } + + std::lock_guard lock(*dmem); + + if (searchRange.beginAddress() == searchRange.endAddress()) { + searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(), + dmem->dmemTotalSize - + dmem->dmemReservedSize); + + if (!searchRange.isValid()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, ErrorCode::INVAL}; + } + } + + if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize, + dmem->dmemReservedSize); + return {{}, ErrorCode::INVAL}; + } + + auto it = dmem->lowerBound(searchRange.beginAddress()); + rx::AddressRange result; + + while (it != dmem->end() && it.beginAddress() < searchRange.endAddress()) { + if (!it->isAllocated()) { + auto viewRange = searchRange.intersection(it.range()); + + viewRange = rx::AddressRange::fromBeginEnd( + rx::alignUp(viewRange.beginAddress(), alignment), + viewRange.endAddress()); + + if (viewRange.isValid() && viewRange.size() >= size) { + result = + rx::AddressRange::fromBeginSize(viewRange.beginAddress(), size); + break; + } + } + + ++it; + } + + if (!result.isValid()) { + return {{}, ErrorCode::NOMEM}; + } + + DirectMemoryAllocation allocation; + allocation.markAsPooled(); + + auto commitResult = dmem->map(result.beginAddress(), result.size(), + allocation, AllocationFlags::Fixed, alignment); + + if (commitResult.errc != std::errc{}) { + return {{}, toErrorCode(commitResult.errc)}; + } + + return {result.beginAddress(), {}}; +} + +orbis::ErrorCode orbis::dmem::setType(unsigned dmemIndex, + rx::AddressRange range, MemoryType type, + rx::EnumBitSet flags) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + bool expectedPooled = (flags & QueryFlags::Pooled) == QueryFlags::Pooled; + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + auto it = dmem->query(range.beginAddress()); + + if (it == dmem->end() || !it->isAllocated() || + it->isPooled() != expectedPooled) { + return ErrorCode::ACCES; + } + + if (it->getMemoryType() == type && it.range().contains(range)) { + return {}; + } + + DirectMemoryAllocation allocation; + allocation.setMemoryType(type); + auto [_it, errc, _range] = + dmem->map(range.beginAddress(), range.size(), allocation, + AllocationFlags::Fixed, vmem::kPageSize); + + return toErrorCode(errc); +} + +std::optional +orbis::dmem::query(unsigned dmemIndex, std::uint64_t dmemOffset, + rx::EnumBitSet flags) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + auto it = dmem->lowerBound(dmemOffset); + + if (flags & QueryFlags::LowerBound) { + while (it != dmem->end() && !it->isAllocated()) { + ++it; + } + + if (it == dmem->end()) { + return {}; + } + } else if (it == dmem->end() || !it->isAllocated()) { + return {}; + } + + if (!(flags & QueryFlags::Pooled)) { + if (it->isPooled()) { + return {}; + } + } + + return QueryResult{.range = it.range(), .memoryType = it->getMemoryType()}; +} + +std::uint64_t orbis::dmem::getSize(unsigned dmemIndex) { + if (dmemIndex >= std::size(g_dmemPools)) { + return 0; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + return dmem->dmemTotalSize - dmem->dmemReservedSize; +} + +std::pair +orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t alignment) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + alignment = alignment == 0 ? kPageSize : alignment; + + if (alignment % kPageSize) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + + if (searchRange.endAddress() > dmem->dmemTotalSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize); + return {{}, orbis::ErrorCode::INVAL}; + } + + if (!searchRange.isValid() && + searchRange.beginAddress() != searchRange.endAddress()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, orbis::ErrorCode::INVAL}; + } + + std::lock_guard lock(*dmem); + + if (searchRange.beginAddress() == searchRange.endAddress()) { + searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(), + dmem->dmemTotalSize - + dmem->dmemReservedSize); + + if (!searchRange.isValid()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, orbis::ErrorCode::INVAL}; + } + } + + if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize, + dmem->dmemReservedSize); + return {{}, orbis::ErrorCode::INVAL}; + } + + auto it = dmem->lowerBound(searchRange.beginAddress()); + rx::AddressRange result = rx::AddressRange::fromBeginEnd( + searchRange.beginAddress(), searchRange.beginAddress()); + + while (it != dmem->end() && it.beginAddress() < searchRange.endAddress()) { + if (!it->isAllocated()) { + auto viewRange = searchRange.intersection(it.range()); + + viewRange = rx::AddressRange::fromBeginEnd( + rx::alignUp(viewRange.beginAddress(), alignment), + viewRange.endAddress()); + + if (viewRange.isValid() && + (!result.isValid() || viewRange.size() > result.size())) { + result = viewRange; + } + } + + ++it; + } + + return {result, {}}; +} + +orbis::ErrorCode orbis::dmem::map(unsigned dmemIndex, rx::AddressRange range, + std::uint64_t offset, + rx::EnumBitSet protection) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + if (offset < 0 || offset + range.size() < offset || + offset + range.size() > dmem->dmemTotalSize) { + return orbis::ErrorCode::INVAL; + } + + if (offset + range.size() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + return orbis::ErrorCode::ACCES; + } + + auto allocationInfoIt = dmem->query(offset); + + if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) { + if (allocationInfoIt != dmem->end()) { + dmemDump( + dmemIndex, + rx::format("map unallocated {:x}-{:x}, requested range {:x}-{:x}", + allocationInfoIt.beginAddress(), + allocationInfoIt.endAddress(), range.beginAddress(), + range.endAddress())); + } else { + dmemDump(dmemIndex, rx::format("map out of memory {:x}-{:x}", + range.beginAddress(), range.endAddress())); + } + return orbis::ErrorCode::ACCES; + } + + if (allocationInfoIt->isPooled()) { + return orbis::ErrorCode::ACCES; + } + + auto directRange = rx::AddressRange::fromBeginSize(offset, range.size()) + .intersection(allocationInfoIt.range()); + + if (range.size() > directRange.size()) { + return orbis::ErrorCode::INVAL; + } + + auto physicalRange = + rx::AddressRange::fromBeginSize(dmem->pmemOffset + offset, range.size()); + + return orbis::pmem::map(range.beginAddress(), physicalRange, + vmem::toCpuProtection(protection)); +} + +std::pair +orbis::dmem::getPmemOffset(unsigned dmemIndex, std::uint64_t dmemOffset) { + if (dmemIndex > std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + if (dmemOffset >= dmem->dmemTotalSize) { + return {{}, orbis::ErrorCode::INVAL}; + } + + if (dmemOffset >= dmem->dmemTotalSize - dmem->dmemReservedSize) { + return {{}, orbis::ErrorCode::ACCES}; + } + + auto allocationInfoIt = dmem->query(dmemOffset); + + if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) { + return {{}, orbis::ErrorCode::ACCES}; + } + + return {dmem->pmemOffset + dmemOffset, {}}; +} diff --git a/kernel/orbis/src/fmem.cpp b/kernel/orbis/src/fmem.cpp index cd6125e8d..47ed3bdf5 100644 --- a/kernel/orbis/src/fmem.cpp +++ b/kernel/orbis/src/fmem.cpp @@ -1,12 +1,14 @@ #include "fmem.hpp" +#include "KernelAllocator.hpp" #include "KernelObject.hpp" #include "error.hpp" #include "kernel/KernelObject.hpp" #include "kernel/MemoryResource.hpp" #include "pmem.hpp" #include "rx/AddressRange.hpp" -#include "thread/Process.hpp" +#include "rx/debug.hpp" +#include "rx/print.hpp" #include "vmem.hpp" #include #include @@ -30,43 +32,39 @@ struct FlexibleMemoryAllocation { }; using FlexibleMemoryResource = - kernel::AllocableResource; + kernel::AllocableResource; -static auto g_fmemInstance = orbis::createProcessLocalObject< +static auto g_fmemInstance = orbis::createGlobalObject< kernel::LockableKernelObject>(); -orbis::ErrorCode orbis::fmem::initialize(Process *process, std::uint64_t size) { +orbis::ErrorCode orbis::fmem::initialize(std::uint64_t size) { auto [range, errc] = - pmem::allocate(pmem::getSize() - 1, size, pmem::MemoryType::WbOnion, - kernel::AllocationFlags::Stack, vmem::kPageSize); + pmem::allocate(0, size, kernel::AllocationFlags::Stack, vmem::kPageSize); if (errc != ErrorCode{}) { return errc; } - auto fmem = process->get(g_fmemInstance); - std::lock_guard lock(*fmem); - return toErrorCode(fmem->create(range)); + rx::println("fmem: {:x}-{:x}", range.beginAddress(), range.endAddress()); + + std::lock_guard lock(*g_fmemInstance); + return toErrorCode(g_fmemInstance->create(range)); } -void orbis::fmem::destroy(Process *process) { - auto fmem = process->get(g_fmemInstance); +void orbis::fmem::destroy() { + std::lock_guard lock(*g_fmemInstance); - std::lock_guard lock(*fmem); - - for (auto allocation : fmem->allocations) { + for (auto allocation : g_fmemInstance->allocations) { pmem::deallocate(allocation); } - fmem->destroy(); + g_fmemInstance->destroy(); } std::pair -orbis::fmem::allocate(Process *process, std::uint64_t size) { - auto fmem = process->get(g_fmemInstance); - - std::lock_guard lock(*fmem); +orbis::fmem::allocate(std::uint64_t size) { + std::lock_guard lock(*g_fmemInstance); FlexibleMemoryAllocation allocation{.allocated = true}; - auto [it, errc, range] = fmem->map( + auto [it, errc, range] = g_fmemInstance->map( 0, size, allocation, kernel::AllocationFlags::NoMerge, vmem::kPageSize); if (errc != std::errc{}) { @@ -76,13 +74,12 @@ orbis::fmem::allocate(Process *process, std::uint64_t size) { return {range, {}}; } -orbis::ErrorCode orbis::fmem::deallocate(Process *process, - rx::AddressRange range) { +orbis::ErrorCode orbis::fmem::deallocate(rx::AddressRange range) { FlexibleMemoryAllocation allocation{}; - auto fmem = process->get(g_fmemInstance); - std::lock_guard lock(*fmem); - auto [it, errc, _] = fmem->map(range.beginAddress(), range.size(), allocation, - AllocationFlags::Fixed, 1); + std::lock_guard lock(*g_fmemInstance); + auto [it, errc, _] = + g_fmemInstance->map(range.beginAddress(), range.size(), allocation, + AllocationFlags::Fixed, 1); return toErrorCode(errc); } diff --git a/kernel/orbis/src/pmem.cpp b/kernel/orbis/src/pmem.cpp index 8a6b76e46..d7abc9bc1 100644 --- a/kernel/orbis/src/pmem.cpp +++ b/kernel/orbis/src/pmem.cpp @@ -1,30 +1,32 @@ #include "pmem.hpp" #include "IoDevice.hpp" +#include "KernelAllocator.hpp" #include "KernelObject.hpp" #include "error.hpp" #include "error/ErrorCode.hpp" +#include "file.hpp" #include "kernel/KernelObject.hpp" #include "kernel/MemoryResource.hpp" #include "rx/AddressRange.hpp" +#include "rx/Rc.hpp" +#include "rx/print.hpp" #include "vmem.hpp" #include #include struct PhysicalMemoryAllocation { - orbis::pmem::MemoryType type = orbis::pmem::MemoryType::Invalid; + bool allocated = false; - [[nodiscard]] bool isAllocated() const { - return type != orbis::pmem::MemoryType::Invalid; - } + [[nodiscard]] bool isAllocated() const { return allocated; } [[nodiscard]] bool isRelated(const PhysicalMemoryAllocation &left, rx::AddressRange, rx::AddressRange) const { - return type == left.type; + return allocated == left.allocated; } [[nodiscard]] PhysicalMemoryAllocation merge(const PhysicalMemoryAllocation &other, rx::AddressRange, rx::AddressRange) const { - assert(other.type == type); + assert(other.allocated == allocated); return other; } @@ -37,12 +39,22 @@ using MappableMemoryResource = })>; using PhysicalMemoryResource = - kernel::AllocableResource; + kernel::AllocableResource; static auto g_pmemInstance = orbis::createGlobalObject< kernel::LockableKernelObject>(); struct PhysicalMemory : orbis::IoDevice { + orbis::File file; + + PhysicalMemory() { + incRef(); // do not delete global object + + file.device = this; + file.incRef(); // do not delete property + } + orbis::ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override { @@ -50,11 +62,12 @@ struct PhysicalMemory : orbis::IoDevice { } orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, - rx::EnumBitSet protection, - orbis::Process *) override { + rx::EnumBitSet protection, + orbis::File *, orbis::Process *) override { return orbis::pmem::map( range.beginAddress(), - rx::AddressRange::fromBeginSize(offset, range.size()), protection); + rx::AddressRange::fromBeginSize(offset, range.size()), + orbis::vmem::toCpuProtection(protection)); } void serialize(rx::Serializer &s) const {} @@ -65,6 +78,8 @@ static auto g_phyMemory = orbis::createGlobalObject(); orbis::ErrorCode orbis::pmem::initialize(std::uint64_t size) { std::lock_guard lock(*g_pmemInstance); + rx::println("pmem: {:x}", size); + return toErrorCode( g_pmemInstance->create(rx::AddressRange::fromBeginSize(0, size))); } @@ -74,11 +89,12 @@ void orbis::pmem::destroy() { g_pmemInstance->destroy(); } -std::pair orbis::pmem::allocate( - std::uint64_t addressHint, std::uint64_t size, MemoryType memoryType, - rx::EnumBitSet flags, std::uint64_t alignment) { +std::pair +orbis::pmem::allocate(std::uint64_t addressHint, std::uint64_t size, + rx::EnumBitSet flags, + std::uint64_t alignment) { std::lock_guard lock(*g_pmemInstance); - PhysicalMemoryAllocation allocation{.type = memoryType}; + PhysicalMemoryAllocation allocation{.allocated = true}; auto [it, errc, range] = g_pmemInstance->map(addressHint, size, allocation, flags, alignment); @@ -99,8 +115,7 @@ orbis::ErrorCode orbis::pmem::deallocate(rx::AddressRange range) { return toErrorCode(errc); } -std::optional -orbis::pmem::query(std::uint64_t address) { +std::optional orbis::pmem::query(std::uint64_t address) { std::lock_guard lock(*g_pmemInstance); auto result = g_pmemInstance->query(address); @@ -108,7 +123,7 @@ orbis::pmem::query(std::uint64_t address) { return {}; } - return AllocatedMemory{.range = result.range(), .memoryType = result->type}; + return result.range(); } orbis::ErrorCode @@ -117,11 +132,11 @@ orbis::pmem::map(std::uint64_t virtualAddress, rx::AddressRange range, auto virtualRange = rx::AddressRange::fromBeginSize(virtualAddress, range.size()); auto errc = g_pmemInstance->mappable.map(virtualRange, range.beginAddress(), - protection, orbis::vmem::kPageSize); + protection, vmem::kPageSize); return toErrorCode(errc); } std::size_t orbis::pmem::getSize() { return g_pmemInstance->size; } orbis::IoDevice *orbis::pmem::getDevice() { return g_phyMemory.get(); } - +orbis::File *orbis::pmem::getFile() { return &g_phyMemory->file; } diff --git a/kernel/orbis/src/sys/sys_event.cpp b/kernel/orbis/src/sys/sys_event.cpp index eab4e0d7d..5e591ccf1 100644 --- a/kernel/orbis/src/sys/sys_event.cpp +++ b/kernel/orbis/src/sys/sys_event.cpp @@ -40,31 +40,33 @@ orbis::SysResult orbis::sys_kqueueex(Thread *thread, ptr name, return {}; } -static bool isReadEventTriggered(int hostFd) { +static bool isReadEventTriggered(const rx::Mappable &hostFd) { #ifdef __linux fd_set fds{}; - FD_SET(hostFd, &fds); + FD_SET(hostFd.native_handle(), &fds); timeval timeout{}; - if (::select(hostFd + 1, &fds, nullptr, nullptr, &timeout) < 0) { + if (::select(hostFd.native_handle() + 1, &fds, nullptr, nullptr, &timeout) < + 0) { return false; } - return FD_ISSET(hostFd, &fds); + return FD_ISSET(hostFd.native_handle(), &fds); #else #warning "Not implemented" return false; #endif } -static bool isWriteEventTriggered(int hostFd) { +static bool isWriteEventTriggered(const rx::Mappable &hostFd) { #ifdef __linux fd_set fds{}; - FD_SET(hostFd, &fds); + FD_SET(hostFd.native_handle(), &fds); timeval timeout{}; - if (::select(hostFd + 1, nullptr, &fds, nullptr, &timeout) < 0) { + if (::select(hostFd.native_handle() + 1, nullptr, &fds, nullptr, &timeout) < + 0) { return false; } - return FD_ISSET(hostFd, &fds); + return FD_ISSET(hostFd.native_handle(), &fds); #else #warning "Not implemented" return false; @@ -131,7 +133,7 @@ static SysResult keventChange(KQueue *kq, KEvent &change, Thread *thread) { eventEmitter->subscribe(&*nodeIt); nodeIt->triggered = true; kq->cv.notify_all(kq->mtx); - } else if (note.file->hostFd < 0) { + } else if (!note.file->hostFd) { ORBIS_LOG_ERROR("Unimplemented event emitter", change.ident); } } else if (change.filter == kEvFiltGraphicsCore || @@ -303,7 +305,7 @@ orbis::SysResult orbis::sys_kevent(Thread *thread, sint fd, if (!note.triggered) { if (note.event.filter == kEvFiltRead) { - if (note.file->hostFd >= 0) { + if (note.file->hostFd) { if (isReadEventTriggered(note.file->hostFd)) { note.triggered = true; } else { @@ -311,7 +313,7 @@ orbis::SysResult orbis::sys_kevent(Thread *thread, sint fd, } } } else if (note.event.filter == kEvFiltWrite) { - if (note.file->hostFd >= 0) { + if (note.file->hostFd) { if (isWriteEventTriggered(note.file->hostFd)) { note.triggered = true; } else { diff --git a/kernel/orbis/src/sys/sys_sce.cpp b/kernel/orbis/src/sys/sys_sce.cpp index a25d3ce66..e13530cc9 100644 --- a/kernel/orbis/src/sys/sys_sce.cpp +++ b/kernel/orbis/src/sys/sys_sce.cpp @@ -7,6 +7,9 @@ #include "module/ModuleInfoEx.hpp" #include "orbis/time.hpp" #include "osem.hpp" +#include "pmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/ProcessOps.hpp" @@ -14,6 +17,7 @@ #include "ucontext.hpp" #include "uio.hpp" #include "utils/Logs.hpp" +#include "vmem.hpp" #include #include #include @@ -69,8 +73,12 @@ orbis::SysResult orbis::sys_netgetiflist(Thread *thread /* TODO */) { return {}; } -orbis::SysResult orbis::sys_mtypeprotect(Thread *thread /* TODO */) { - return ErrorCode::NOSYS; +orbis::SysResult +orbis::sys_mtypeprotect(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, + rx::EnumBitSet prot) { + auto range = rx::AddressRange::fromBeginSize(addr, len); + return vmem::setTypeAndProtect(thread->tproc, range, type, prot); } orbis::SysResult orbis::sys_regmgr_call(Thread *thread, uint32_t op, uint32_t id, ptr result, @@ -393,19 +401,20 @@ orbis::SysResult orbis::sys_evf_cancel(Thread *thread, sint id, uint64_t value, return {}; } orbis::SysResult -orbis::sys_query_memory_protection(Thread *thread, ptr address, - ptr protection) { - if (auto query_memory_protection = - thread->tproc->ops->query_memory_protection) { - return query_memory_protection(thread, address, protection); +orbis::sys_query_memory_protection(Thread *thread, uintptr_t address, + ptr protection) { + auto result = vmem::queryProtection(thread->tproc, address); + + if (!result) { + return ErrorCode::ACCES; } - return ErrorCode::NOSYS; + return uwrite(protection, *result); } namespace orbis { struct BatchMapEntry { - ptr start; + uintptr_t start; off_t offset; size_t length; char protection; @@ -419,14 +428,14 @@ struct BatchMapEntry { }; } // namespace orbis -orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, sint flags, +orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, + rx::EnumBitSet flags, ptr entries, sint entriesCount, ptr processedCount) { - auto ops = thread->tproc->ops; SysResult result = ErrorCode{}; - ORBIS_LOG_ERROR(__FUNCTION__, unk, flags, entries, entriesCount, - processedCount); + ORBIS_LOG_ERROR(__FUNCTION__, unk, flags.toUnderlying(), entries, + entriesCount, processedCount); int processed = 0; for (int i = 0; i < entriesCount; ++i) { @@ -438,16 +447,29 @@ orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, sint flags, switch (_entry.operation) { case 0: - result = ops->dmem_mmap(thread, _entry.start, _entry.length, _entry.type, - _entry.protection, flags, _entry.offset); - break; + result = sys_mmap_dmem( + thread, _entry.start, _entry.length, + static_cast(_entry.type), + rx::EnumBitSet::fromUnderlying(_entry.protection), + flags, _entry.offset); case 1: - result = ops->munmap(thread, _entry.start, _entry.length); + result = sys_munmap(thread, _entry.start, _entry.length); break; case 2: - result = - ops->mprotect(thread, _entry.start, _entry.length, _entry.protection); + result = sys_mprotect( + thread, _entry.start, _entry.length, + rx::EnumBitSet::fromUnderlying(_entry.protection)); break; + case 3: + result = sys_mmap( + thread, _entry.start, _entry.length, + rx::EnumBitSet::fromUnderlying(_entry.protection), + vmem::MapFlags::Void | vmem::MapFlags::Anon | flags, -1, 0); + break; + case 4: + ORBIS_LOG_ERROR(__FUNCTION__, "unimplemented type protect"); + break; + default: result = ErrorCode::INVAL; break; @@ -880,14 +902,22 @@ orbis::SysResult orbis::sys_budget_set(Thread *thread, sint budgetId) { thread->tproc->budgetId = budgetId; return {}; } -orbis::SysResult orbis::sys_virtual_query(Thread *thread, ptr addr, - uint64_t unk, ptr info, + +orbis::SysResult orbis::sys_virtual_query(Thread *thread, uintptr_t addr, + uint64_t flags, ptr info, size_t infosz) { - if (auto virtual_query = thread->tproc->ops->virtual_query) { - return virtual_query(thread, addr, unk, info, infosz); + + if (infosz != sizeof(vmem::QueryResult)) { + return ErrorCode::INVAL; } - return ErrorCode::NOSYS; + auto result = vmem::query(thread->tproc, addr, flags & 1); + + if (!result || result->start >= 0x800000000) { + return ErrorCode::ACCES; + } + + return uwrite(ptr(info), *result); } orbis::SysResult orbis::sys_mdbg_call(Thread *thread /* TODO */) { return {}; } orbis::SysResult orbis::sys_obs_sblock_create(Thread *thread /* TODO */) { @@ -930,7 +960,7 @@ orbis::SysResult orbis::sys_is_in_sandbox(Thread *thread /* TODO */) { } orbis::SysResult orbis::sys_dmem_container(Thread *thread, uint id) { ORBIS_LOG_NOTICE(__FUNCTION__, id); - thread->retval[0] = 1; // returns default direct memory device + thread->retval[0] = 0; // returns default direct memory device if (id + 1) return ErrorCode::PERM; return {}; @@ -955,24 +985,11 @@ orbis::SysResult orbis::sys_mname(Thread *thread, uint64_t addr, uint64_t len, return result; } - NamedMemoryRange range; - range.begin = addr & ~0x3fffull; - range.end = range.begin; - range.end += ((addr & 0x3fff) + 0x3fff + len) & ~0x3fffull; - - std::lock_guard lock(thread->tproc->namedMemMutex); - auto [it, end] = thread->tproc->namedMem.equal_range(range); - while (it != end) { - auto [addr2, end2] = it->first; - auto len2 = end2 - addr2; - ORBIS_LOG_NOTICE("sys_mname: removed overlapped", it->second, addr2, len2); - it = thread->tproc->namedMem.erase(it); + auto range = rx::AddressRange::fromBeginSize(addr, len); + if (auto error = vmem::setName(thread->tproc, range, _name); + error != ErrorCode{}) { + ORBIS_LOG_ERROR(__FUNCTION__, "failure:", static_cast(error)); } - if (!thread->tproc->namedMem.try_emplace(range, _name).second) - std::abort(); - if (!thread->tproc->namedMem.count(addr)) - std::abort(); - return {}; } orbis::SysResult orbis::sys_dynlib_dlopen(Thread *thread /* TODO */) { @@ -1019,7 +1036,7 @@ orbis::SysResult orbis::sys_dynlib_get_info(Thread *thread, ModuleInfo result = {}; result.size = sizeof(ModuleInfo); - std::strncpy(result.name, module->moduleName, sizeof(result.name)); + std::strncpy(result.name, module->soName, sizeof(result.name)); std::memcpy(result.segments, module->segments, sizeof(ModuleSegment) * module->segmentCount); result.segmentCount = module->segmentCount; @@ -1485,14 +1502,56 @@ orbis::SysResult orbis::sys_get_cpu_usage_all(Thread *thread, uint32_t unk, ORBIS_LOG_TODO(__FUNCTION__, unk, result); return {}; } -orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, caddr_t addr, size_t len, - sint memoryType, sint prot, sint flags, +orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr, + size_t len, MemoryType memoryType, + rx::EnumBitSet prot, + rx::EnumBitSet flags, off_t directMemoryStart) { - if (auto dmem_mmap = thread->tproc->ops->dmem_mmap) { - return dmem_mmap(thread, addr, len, memoryType, prot, flags, - directMemoryStart); + + auto callerAddress = getCallerAddress(thread); + auto alignment = dmem::kPageSize; + + { + auto unpacked = unpackMapFlags(flags, dmem::kPageSize); + alignment = unpacked.first; + flags = unpacked.second; } - return ErrorCode::NOSYS; + + len = rx::alignUp(len, dmem::kPageSize); + + rx::EnumBitSet allocFlags{}; + + if (!prot) { + // HACK + // FIXME: implement protect for pid + prot = vmem::Protection::CpuRead | vmem::Protection::CpuWrite | + vmem::Protection::GpuRead | vmem::Protection::GpuWrite; + } + + if (flags & vmem::MapFlags::Fixed) { + allocFlags = AllocationFlags::Fixed; + } else if (addr == 0) { + addr = 0xfe0000000; + } + + if (flags & vmem::MapFlags::NoOverwrite) { + allocFlags |= AllocationFlags::NoOverwrite; + } + + auto name = + callerAddress ? rx::format("anon:{:012x}", callerAddress) : ""; + + auto [range, errc] = + vmem::mapDirect(thread->tproc, addr, + rx::AddressRange::fromBeginSize(directMemoryStart, len), + prot, allocFlags, name, alignment, memoryType); + + if (errc != ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress(); + return {}; } orbis::SysResult orbis::sys_physhm_open(Thread *thread /* TODO */) { return ErrorCode::NOSYS; @@ -1694,31 +1753,53 @@ orbis::SysResult orbis::sys_process_terminate(Thread *thread /* TODO */) { return ErrorCode::NOSYS; } orbis::SysResult orbis::sys_blockpool_open(Thread *thread) { - if (auto blockpool_open = thread->tproc->ops->blockpool_open) { - rx::Ref file; - auto result = blockpool_open(thread, &file); - if (result.isError()) { - return result; - } + if (!g_context->blockpool) { + return ErrorCode::NOSYS; + } - thread->retval[0] = thread->tproc->fileDescriptors.insert(file); - return {}; + auto fd = thread->tproc->fileDescriptors.insert(g_context->blockpool); + + if (fd == thread->tproc->fileDescriptors.npos) { + return ErrorCode::MFILE; } - return ErrorCode::NOSYS; + + thread->retval[0] = fd; + return {}; } -orbis::SysResult orbis::sys_blockpool_map(Thread *thread, caddr_t addr, - size_t len, sint prot, sint flags) { - if (auto blockpool_map = thread->tproc->ops->blockpool_map) { - return blockpool_map(thread, addr, len, prot, flags); +orbis::SysResult +orbis::sys_blockpool_map(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, rx::EnumBitSet prot, + rx::EnumBitSet flags) { + rx::EnumBitSet allocFlags{}; + + if (flags & vmem::MapFlags::Fixed) { + allocFlags |= AllocationFlags::Fixed; } - return ErrorCode::NOSYS; + if (flags & vmem::MapFlags::NoOverwrite) { + allocFlags |= AllocationFlags::NoOverwrite; + } + if (flags & vmem::MapFlags::NoCoalesce) { + allocFlags |= AllocationFlags::NoMerge; + } + + auto [range, errc] = vmem::commitPooled( + thread->tproc, rx::AddressRange::fromBeginSize(addr, len), type, prot); + + if (errc != ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress(); + return {}; } -orbis::SysResult orbis::sys_blockpool_unmap(Thread *thread, caddr_t addr, +orbis::SysResult orbis::sys_blockpool_unmap(Thread *thread, uintptr_t addr, size_t len, sint flags) { - if (auto blockpool_unmap = thread->tproc->ops->blockpool_unmap) { - return blockpool_unmap(thread, addr, len); + if (flags != 0) { + return ErrorCode::INVAL; } - return ErrorCode::NOSYS; + + return vmem::decommitPooled(thread->tproc, + rx::AddressRange::fromBeginSize(addr, len)); } orbis::SysResult orbis::sys_dynlib_get_info_for_libdbg(Thread *thread /* TODO */) { diff --git a/kernel/orbis/src/sys/sys_sysctl.cpp b/kernel/orbis/src/sys/sys_sysctl.cpp index 5015f136c..4515cd6b9 100644 --- a/kernel/orbis/src/sys/sys_sysctl.cpp +++ b/kernel/orbis/src/sys/sys_sysctl.cpp @@ -1,4 +1,5 @@ #include "KernelContext.hpp" +#include "rx/print.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/Thread.hpp" @@ -749,9 +750,8 @@ SysResult kern_sysctl(Thread *thread, ptr name, uint namelen, return ErrorCode::INVAL; } - std::printf("Reporting stack at %p\n", thread->stackEnd); - *(ptr *)old = thread->stackEnd; - return {}; + rx::println(stderr, "Reporting stack at {:x}", thread->stackEnd); + return uwrite(ptr(old), thread->stackEnd); } case sysctl_kern::smp_cpus: diff --git a/kernel/orbis/src/sys/sys_vm_mmap.cpp b/kernel/orbis/src/sys/sys_vm_mmap.cpp index 0c8fa67fe..309807761 100644 --- a/kernel/orbis/src/sys/sys_vm_mmap.cpp +++ b/kernel/orbis/src/sys/sys_vm_mmap.cpp @@ -1,8 +1,16 @@ +#include "KernelContext.hpp" #include "error.hpp" +#include "rx/AddressRange.hpp" +#include "rx/align.hpp" +#include "rx/debug.hpp" +#include "rx/format.hpp" +#include "rx/print.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/ProcessOps.hpp" #include "thread/Thread.hpp" +#include "utils/Logs.hpp" +#include "vmem.hpp" orbis::SysResult orbis::sys_sbrk(Thread *, sint) { return ErrorCode::OPNOTSUPP; @@ -11,94 +19,179 @@ orbis::SysResult orbis::sys_sstk(Thread *, sint) { return ErrorCode::OPNOTSUPP; } -orbis::SysResult orbis::sys_mmap(Thread *thread, caddr_t addr, size_t len, - sint prot, sint flags, sint fd, off_t pos) { - if (auto impl = thread->tproc->ops->mmap) { - return impl(thread, addr, len, prot, flags, fd, pos); +orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, + off_t pos) { + + std::uint64_t callerAddress = getCallerAddress(thread); + + auto shift = addr & (vmem::kPageSize - 1); + addr = rx::alignDown(addr, vmem::kPageSize); + rx::EnumBitSet allocFlags{}; + rx::EnumBitSet blockFlags{}; + rx::EnumBitSet blockFlagsEx{}; + std::uint64_t alignment = vmem::kPageSize; + + { + auto unpacked = unpackMapFlags(flags, vmem::kPageSize); + alignment = unpacked.first; + flags = unpacked.second; } - return ErrorCode::NOSYS; + len = rx::alignUp(len, vmem::kPageSize); + + if (flags & vmem::MapFlags::Stack) { + allocFlags |= AllocationFlags::Stack; + } + + if (flags & vmem::MapFlags::Fixed) { + allocFlags |= AllocationFlags::Fixed; + } + + if (flags & vmem::MapFlags::NoOverwrite) { + allocFlags |= AllocationFlags::NoOverwrite; + } + + if (flags & vmem::MapFlags::NoCoalesce) { + allocFlags |= AllocationFlags::NoMerge; + } + + if (flags & vmem::MapFlags::Shared) { + blockFlagsEx |= vmem::BlockFlagsEx::Shared; + + if (flags & vmem::MapFlags::Private) { + return ErrorCode::INVAL; + } + } + + if (flags & vmem::MapFlags::Private) { + blockFlagsEx |= vmem::BlockFlagsEx::Private; + } + + if (addr == 0) { + addr = flags & vmem::MapFlags::System ? 0xfc0000000 : 0x200000000; + } + + if (flags & vmem::MapFlags::Void) { + flags |= vmem::MapFlags::Anon; + + if (fd != -1 || pos != 0) { + return ErrorCode::INVAL; + } + } + + if (flags & vmem::MapFlags::Stack) { + flags |= vmem::MapFlags::Anon; + blockFlags |= vmem::BlockFlags::Stack; + + if (fd != -1 || pos != 0) { + return ErrorCode::INVAL; + } + + if ((prot & (vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) != + (vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) { + return ErrorCode::INVAL; + } + } + + auto name = callerAddress ? rx::format("anon:{:012x}", callerAddress) : ""; + + if (flags & vmem::MapFlags::Anon) { + if (fd != -1 || pos != 0) { + return ErrorCode::INVAL; + } + + if (prot & (vmem::Protection::GpuRead | vmem::Protection::GpuWrite)) { + return ErrorCode::INVAL; + } + + auto [range, errc] = vmem::mapFlex(thread->tproc, len, prot, addr, + allocFlags, blockFlags, name, alignment); + + if (errc != orbis::ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress() + shift; + return {}; + } + + auto file = thread->tproc->fileDescriptors.get(fd); + if (file == nullptr) { + return ErrorCode::BADF; + } + + if (!file->device->blockFlags) { + blockFlags |= vmem::BlockFlags::FlexibleMemory; + } + + prot &= ~vmem::Protection::CpuExec; + + auto [range, errc] = + vmem::mapFile(thread->tproc, addr, len, allocFlags, prot, blockFlags, + blockFlagsEx, file.get(), pos, name, alignment); + + if (errc != ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress() + shift; + return {}; } -orbis::SysResult orbis::sys_freebsd6_mmap(Thread *thread, caddr_t addr, - size_t len, sint prot, sint flags, +orbis::SysResult orbis::sys_freebsd6_mmap(Thread *thread, uintptr_t addr, + size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, sint, off_t pos) { return sys_mmap(thread, addr, len, prot, flags, fd, pos); } -orbis::SysResult orbis::sys_msync(Thread *thread, ptr addr, size_t len, +orbis::SysResult orbis::sys_msync(Thread *thread, uintptr_t addr, size_t len, sint flags) { - if (auto impl = thread->tproc->ops->msync) { - return impl(thread, addr, len, flags); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__, addr, len, flags); + return {}; } -orbis::SysResult orbis::sys_munmap(Thread *thread, ptr addr, size_t len) { - if (auto impl = thread->tproc->ops->munmap) { - return impl(thread, addr, len); - } +orbis::SysResult orbis::sys_munmap(Thread *thread, uintptr_t addr, size_t len) { + auto range = rx::AddressRange::fromBeginSize(addr, len); - return ErrorCode::NOSYS; + return vmem::unmap(thread->tproc, range); } -orbis::SysResult orbis::sys_mprotect(Thread *thread, ptr addr, - size_t len, sint prot) { - if (auto impl = thread->tproc->ops->mprotect) { - return impl(thread, addr, len, prot); - } - - return ErrorCode::NOSYS; +orbis::SysResult orbis::sys_mprotect(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot) { + auto range = rx::AddressRange::fromBeginSize(addr, len); + return vmem::protect(thread->tproc, range, prot); } -orbis::SysResult orbis::sys_minherit(Thread *thread, ptr addr, size_t len, +orbis::SysResult orbis::sys_minherit(Thread *thread, uintptr_t addr, size_t len, sint inherit) { - if (auto impl = thread->tproc->ops->minherit) { - return impl(thread, addr, len, inherit); - } + ORBIS_LOG_TODO(__FUNCTION__, addr, len, inherit); return ErrorCode::NOSYS; } -orbis::SysResult orbis::sys_madvise(Thread *thread, ptr addr, size_t len, +orbis::SysResult orbis::sys_madvise(Thread *thread, uintptr_t addr, size_t len, sint behav) { - if (auto impl = thread->tproc->ops->madvise) { - return impl(thread, addr, len, behav); - } - + ORBIS_LOG_TODO(__FUNCTION__, addr, len, behav); return ErrorCode::NOSYS; } -orbis::SysResult orbis::sys_mincore(Thread *thread, ptr addr, - size_t len, ptr vec) { - if (auto impl = thread->tproc->ops->mincore) { - return impl(thread, addr, len, vec); - } - +orbis::SysResult orbis::sys_mincore(Thread *thread, uintptr_t addr, size_t len, + ptr vec) { + ORBIS_LOG_TODO(__FUNCTION__, addr, len, vec); return ErrorCode::NOSYS; } -orbis::SysResult orbis::sys_mlock(Thread *thread, ptr addr, - size_t len) { - if (auto impl = thread->tproc->ops->mlock) { - return impl(thread, addr, len); - } - - return ErrorCode::NOSYS; +orbis::SysResult orbis::sys_mlock(Thread *thread, uintptr_t addr, size_t len) { + ORBIS_LOG_TODO(__FUNCTION__, addr, len); + return {}; } orbis::SysResult orbis::sys_mlockall(Thread *thread, sint how) { - if (auto impl = thread->tproc->ops->mlockall) { - return impl(thread, how); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__, how); + return {}; } orbis::SysResult orbis::sys_munlockall(Thread *thread) { - if (auto impl = thread->tproc->ops->munlockall) { - return impl(thread); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__); + return {}; } -orbis::SysResult orbis::sys_munlock(Thread *thread, ptr addr, +orbis::SysResult orbis::sys_munlock(Thread *thread, uintptr_t addr, size_t len) { - if (auto impl = thread->tproc->ops->munlock) { - return impl(thread, addr, len); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__, addr, len); + return {}; } diff --git a/kernel/orbis/src/sysvec.cpp b/kernel/orbis/src/sysvec.cpp index 786531131..c000ae180 100644 --- a/kernel/orbis/src/sysvec.cpp +++ b/kernel/orbis/src/sysvec.cpp @@ -1,10 +1,15 @@ +#include "orbis-config.hpp" +#include "rx/EnumBitSet.hpp" +#include "rx/format.hpp" #include "sys/syscall.hpp" #include "sys/sysentry.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/Thread.hpp" #include +#include #include +#include enum { PSL_C = 0x1 }; @@ -101,6 +106,72 @@ struct WrapImpl { sysent result; result.narg = sizeof...(Args); result.call = &WrapImpl::call; + result.format = [](uint64_t *values) -> std::string { + std::string result = getSysentName(&WrapImpl::call); + result += "("; + + auto formatArg = + [&](std::integral_constant, + T value, std::uint64_t raw) { + if (I != 0) { + result += ", "; + } + + using type = std::remove_cvref_t; + + if constexpr (std::is_same_v) { + if (value) { + result += "true"; + } else { + result += "false"; + } + } else if constexpr (std::is_integral_v) { + if (value > 9) { + result += rx::format("{:#x}", value); + } else { + result += rx::format("{}", value); + } + } else if constexpr (std::is_pointer_v) { + result += rx::format("{}", (void *)value); + + // using pointee = std::remove_cvref_t>; + // if constexpr (requires(rx::format_parse_context &ctx) { + // rx::formatter().parse(ctx); + // }) { + // if (value) { + // pointee kernelValue; + // auto errc = orbis::uread(kernelValue, value); + + // if (errc == ErrorCode{}) { + // result += rx::format("={}", kernelValue); + // } else { + // result += rx::format("={}", errc); + // } + // } + // } + } else if constexpr (requires(rx::format_parse_context &ctx) { + rx::formatter().parse(ctx); + }) { + result += rx::format("{}", value); + } else { + if (raw > 9) { + result += rx::format("{:#x}", raw); + } else { + result += rx::format("{}", raw); + } + } + }; + auto formatArgs = [&](std::index_sequence, + uint64_t *values) { + (formatArg(std::integral_constant{}, + makeArg(values[I]), values[I]), + ...); + }; + + formatArgs(std::make_index_sequence{}, values); + result += ")"; + return result; + }; return result; } @@ -113,7 +184,28 @@ private: template static SysResult callImpl(Thread *thread, uint64_t *args, std::index_sequence) { - return Fn(thread, Args(args[I])...); + return Fn(thread, makeArg(args[I])...); + } + + template + static T makeArg(uint64_t raw) + requires requires { std::bit_cast(raw); } + { + return std::bit_cast(raw); + } + + template + static T makeArg(uint64_t raw) + requires requires { T(raw); } && (!requires { std::bit_cast(raw); }) + { + return T(raw); + } + + template + static T makeArg(uint64_t raw) + requires requires { T(T::fromUnderlying(raw)); } + { + return T::fromUnderlying(raw); } }; } // namespace detail diff --git a/kernel/orbis/src/thread/Process.cpp b/kernel/orbis/src/thread/Process.cpp index 5d6cbc605..9d1bebdec 100644 --- a/kernel/orbis/src/thread/Process.cpp +++ b/kernel/orbis/src/thread/Process.cpp @@ -177,3 +177,9 @@ orbis::Process *orbis::createProcess(Process *parentProcess, pid_t pid) { g_processList->list = result; return &result->object; } + +orbis::Budget *orbis::Process::getBudget() const { + auto result = g_context->budgets.get(budgetId); + + return result != nullptr ? result.get() : nullptr; +} diff --git a/kernel/orbis/src/thread/Thread.cpp b/kernel/orbis/src/thread/Thread.cpp index 579a2cd47..124cab0d5 100644 --- a/kernel/orbis/src/thread/Thread.cpp +++ b/kernel/orbis/src/thread/Thread.cpp @@ -36,3 +36,35 @@ orbis::Thread *orbis::createThread(Process *process, std::string_view name) { return result; } + +uintptr_t orbis::getCallerAddress(Thread *thread) { + if (!thread->context) { + return 0; + } + + auto rbp = readRegister(thread->context, RegisterId::rbp); + std::uint64_t result = 0; + + while (rbp < 0x8000'0000'0000) { + auto framePtr = std::bit_cast>(rbp); + + std::uint64_t nextFrame; + std::uint64_t retAddress; + if (orbis::uread(retAddress, framePtr + 1) != orbis::ErrorCode{}) { + break; + } + + if (orbis::uread(nextFrame, framePtr) != orbis::ErrorCode{}) { + break; + } + + if (!thread->tproc->libkernelRange.contains(retAddress)) { + result = retAddress; + break; + } + + rbp = nextFrame; + } + + return result; +} diff --git a/kernel/orbis/src/vmem.cpp b/kernel/orbis/src/vmem.cpp index 70f663e7b..545b4ed83 100644 --- a/kernel/orbis/src/vmem.cpp +++ b/kernel/orbis/src/vmem.cpp @@ -1,11 +1,25 @@ #include "vmem.hpp" +#include "Budget.hpp" #include "IoDevice.hpp" +#include "KernelAllocator.hpp" +#include "KernelContext.hpp" #include "KernelObject.hpp" +#include "MemoryType.hpp" +#include "blockpool.hpp" +#include "dmem.hpp" #include "error.hpp" +#include "fmem.hpp" #include "pmem.hpp" -#include "rx/Mappable.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" +#include "rx/FileLock.hpp" +#include "rx/FunctionRef.hpp" #include "rx/Rc.hpp" +#include "rx/Serializer.hpp" #include "rx/StaticString.hpp" +#include "rx/align.hpp" +#include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/mem.hpp" #include "rx/print.hpp" #include "thread/Process.hpp" @@ -13,28 +27,48 @@ #include #include +// FIXME: remove +namespace amdgpu { +void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset); +void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange); +void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot); +} // namespace amdgpu + struct VirtualMemoryAllocation { rx::EnumBitSet flags{}; + rx::EnumBitSet flagsEx{}; rx::EnumBitSet prot{}; + orbis::MemoryType type = orbis::MemoryType::Invalid; rx::Ref device; std::uint64_t deviceOffset = 0; rx::StaticString<31> name; [[nodiscard]] bool isAllocated() const { - return (flags & orbis::vmem::BlockFlags::Allocated) == - orbis::vmem::BlockFlags::Allocated; + return (flagsEx & orbis::vmem::BlockFlagsEx::Allocated) == + orbis::vmem::BlockFlagsEx::Allocated; } [[nodiscard]] bool isRelated(const VirtualMemoryAllocation &other, rx::AddressRange selfRange, [[maybe_unused]] rx::AddressRange rightRange) const { - if (flags != other.flags || prot != other.prot || device != other.device || + if (flags != other.flags || flagsEx != other.flagsEx || + prot != other.prot || type != other.type || device != other.device || name != other.name) { return false; } + if (!isAllocated()) { + return true; + } - return !isAllocated() || - deviceOffset + selfRange.size() == other.deviceOffset; + if (device == nullptr || flags == orbis::vmem::BlockFlags::PooledMemory) { + return true; + } + + return deviceOffset + selfRange.size() == other.deviceOffset; } [[nodiscard]] VirtualMemoryAllocation merge(const VirtualMemoryAllocation &, @@ -43,10 +77,10 @@ struct VirtualMemoryAllocation { return *this; } - std::pair - truncate(rx::AddressRange selfRange, rx::AddressRange, - rx::AddressRange rightRange) { - if (!isAllocated() || !rightRange.isValid() || device == nullptr) { + [[nodiscard]] std::pair + truncate(rx::AddressRange selfRange, rx::AddressRange leftRange, + rx::AddressRange rightRange) const { + if (!rightRange.isValid() || device == nullptr) { return {}; } @@ -59,19 +93,197 @@ struct VirtualMemoryAllocation { } bool operator==(const VirtualMemoryAllocation &) const = default; + + void setName(orbis::Process *process, std::string_view newName) { + if (!newName.empty()) { + name = newName; + return; + } + + name = "(NoName)"; + + if (auto executable = process->modulesMap.get(orbis::ModuleHandle{})) { + name += executable->soName; + } + } + + void serialize(rx::Serializer &s) const {} + void deserialize(rx::Deserializer &d) {} }; -using MappableMemoryResource = - kernel::MappableResource; - struct VirtualMemoryResource - : kernel::AllocableResource {}; + : kernel::AllocableResource {}; static auto g_vmInstance = orbis::createProcessLocalObject< kernel::LockableKernelObject>(); +static void vmemDump(orbis::Process *process, std::string_view message = {}) { + rx::ScopedFileLock lock(stderr); + auto vmem = process->get(g_vmInstance); + + rx::println(stderr, "vmem {} {}", process->pid, message); + for (auto alloc : *vmem) { + if (!alloc->isAllocated()) { + continue; + } + + rx::print(stderr, " {:012x}-{:012x}: {:08x}", alloc.beginAddress(), + alloc.endAddress(), + alloc->flags & orbis::vmem::BlockFlags::FlexibleMemory + ? 0 + : alloc->deviceOffset); + + rx::print( + stderr, " {}{}{}{}{}", + "_P"[alloc->flags.any_of(orbis::vmem::BlockFlags::PooledMemory)], + "_D"[alloc->flags.any_of(orbis::vmem::BlockFlags::DirectMemory)], + "_F"[alloc->flags.any_of(orbis::vmem::BlockFlags::FlexibleMemory)], + "_S"[alloc->flags.any_of(orbis::vmem::BlockFlags::Stack)], + "_C"[alloc->flags.any_of(orbis::vmem::BlockFlags::Commited)]); + + rx::print(stderr, " {}{}{} {}{}", + "_R"[alloc->prot.any_of(orbis::vmem::Protection::CpuRead)], + "_W"[alloc->prot.any_of(orbis::vmem::Protection::CpuWrite)], + "_X"[alloc->prot.any_of(orbis::vmem::Protection::CpuExec)], + "_R"[alloc->prot.any_of(orbis::vmem::Protection::GpuRead)], + "_W"[alloc->prot.any_of(orbis::vmem::Protection::GpuWrite)]); + + rx::print( + stderr, " {}{}{}{}{} {}", + "_A"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Allocated)], + "_P"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Private)], + "_S"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Shared)], + "_C"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::PoolControl)], + "_R"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Reserved)], + alloc->type); + + rx::println(stderr, " {}", alloc->name); + } +} + +static orbis::ErrorCode validateOverwrite(decltype(g_vmInstance)::type *vmem, + rx::AddressRange range, + bool isUnmap) { + for (auto it = vmem->lowerBound(range.beginAddress()); + it != vmem->end() && it.beginAddress() < range.endAddress(); ++it) { + if (!it->isAllocated()) { + continue; + } + + if (it->flagsEx & orbis::vmem::BlockFlagsEx::Reserved) { + return orbis::ErrorCode::ACCES; + } + + if (it->flags & orbis::vmem::BlockFlags::PooledMemory) { + if (!isUnmap) { + return orbis::ErrorCode::ACCES; + } + + while (!(it->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl)) { + ++it; + + if (it == vmem->end()) { + return orbis::ErrorCode::ACCES; + } + + if (!it->isAllocated() || + !(it->flags & orbis::vmem::BlockFlags::PooledMemory)) { + --it; + break; + } + } + + if ((it->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl) && + it->deviceOffset != 0) { + // control block has allocations, verify that whole reserved range going + // to be unmapped + + auto reservedRange = rx::AddressRange::fromBeginEnd( + it.beginAddress() - it->deviceOffset, it.endAddress()); + + if (!range.contains(reservedRange)) { + return orbis::ErrorCode::ACCES; + } + } + } + } + + return {}; +} + +static void release(orbis::Process *process, decltype(g_vmInstance)::type *vmem, + orbis::Budget *budget, rx::AddressRange range) { + for (auto it = vmem->lowerBound(range.beginAddress()); + it != vmem->end() && it.beginAddress() < range.endAddress(); ++it) { + if (!it->isAllocated()) { + continue; + } + + auto blockRange = range.intersection(it.range()); + + if (it->flags & orbis::vmem::BlockFlags::FlexibleMemory) { + if (it->device == nullptr) { + orbis::fmem::deallocate(blockRange); + } + budget->release(orbis::BudgetResource::Fmem, blockRange.size()); + } + + if (it->flags & orbis::vmem::BlockFlags::DirectMemory) { + budget->release(orbis::BudgetResource::Dmem, blockRange.size()); + } + + if (it->flags & orbis::vmem::BlockFlags::PooledMemory) { + if (it->flags & orbis::vmem::BlockFlags::Commited) { + orbis::blockpool::decommit(process, it.range()); + } else if (it->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl) { + orbis::blockpool::releaseControlBlock(); + } + } + + if (orbis::vmem::toGpuProtection(it->prot) && + (it->flags & orbis::vmem::BlockFlags::DirectMemory | + orbis::vmem::BlockFlags::PooledMemory)) { + amdgpu::unmapMemory(process->pid, blockRange); + } + } +} + +static orbis::ErrorCode validateRange( + decltype(g_vmInstance)::type *vmem, + decltype(g_vmInstance)::type::iterator it, rx::AddressRange range, + rx::FunctionRef cb) { + while (it != vmem->end() && it.beginAddress() < range.endAddress()) { + if (auto errc = cb(it.get()); errc != orbis::ErrorCode{}) { + return errc; + } + + ++it; + } + + return {}; +} + +static void modifyRange( + decltype(g_vmInstance)::type *vmem, + decltype(g_vmInstance)::type::iterator it, rx::AddressRange range, + rx::FunctionRef cb) { + while (it != vmem->end() && it.beginAddress() < range.endAddress()) { + auto mapRange = range.intersection(it.range()); + auto allocInfo = it.get(); + + if (allocInfo.device != nullptr && + !(allocInfo.flags & orbis::vmem::BlockFlags::PooledMemory)) { + allocInfo.deviceOffset += mapRange.beginAddress() - it.beginAddress(); + } + + cb(allocInfo, mapRange); + vmem->map(mapRange.beginAddress(), mapRange.size(), allocInfo, + orbis::AllocationFlags::Fixed, orbis::vmem::kPageSize); + + ++it; + } +} + void orbis::vmem::initialize(Process *process, bool force) { auto vmem = process->get(g_vmInstance); @@ -125,37 +337,60 @@ void orbis::vmem::fork(Process *process, Process *parentThread) { // FIXME: implement } -std::pair orbis::vmem::map( +std::pair orbis::vmem::reserve( Process *process, std::uint64_t addressHint, std::uint64_t size, - rx::EnumBitSet allocFlags, rx::EnumBitSet prot, - rx::EnumBitSet blockFlags, std::uint64_t alignment, - std::string_view name, IoDevice *device, std::int64_t deviceOffset) { - auto vmem = process->get(g_vmInstance); + rx::EnumBitSet allocFlags, + rx::EnumBitSet blockFlagsEx, std::uint64_t alignment) { + VirtualMemoryAllocation allocationInfo; + allocationInfo.flagsEx = blockFlagsEx | BlockFlagsEx::Allocated; + auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - VirtualMemoryAllocation allocationInfo; - allocationInfo.flags = blockFlags | orbis::vmem::BlockFlags::Allocated; - allocationInfo.device = device; - allocationInfo.prot = prot; - allocationInfo.deviceOffset = deviceOffset; - allocationInfo.name = name; + auto [_, errc, range] = + vmem->map(addressHint, size, allocationInfo, allocFlags, alignment); - if (device == nullptr) { - if (deviceOffset != 0 || prot) { - return {{}, ErrorCode::INVAL}; - } - - auto [_, errc, range] = - vmem->map(addressHint, size, allocationInfo, allocFlags, alignment); - - if (errc != std::errc{}) { - return {{}, toErrorCode(errc)}; - } - - return {range, {}}; + if (errc != std::errc{}) { + return {{}, toErrorCode(errc)}; } + return {range, {}}; +} + +std::pair orbis::vmem::mapFile( + Process *process, std::uint64_t addressHint, std::uint64_t size, + rx::EnumBitSet allocFlags, rx::EnumBitSet prot, + rx::EnumBitSet blockFlags, + rx::EnumBitSet blockFlagsEx, File *file, + std::uint64_t fileOffset, std::string_view name, std::uint64_t alignment, + MemoryType type) { + blockFlags |= file->device->blockFlags; + + if (blockFlags & BlockFlags::PooledMemory) { + if (size < dmem::kPageSize * 2 || size % dmem::kPageSize) { + return {{}, orbis::ErrorCode::INVAL}; + } + } + + if (blockFlags & (BlockFlags::DirectMemory | BlockFlags::PooledMemory)) { + if (alignment < dmem::kPageSize) { + alignment = dmem::kPageSize; + } + } + + bool canOverwrite = (allocFlags & AllocationFlags::Fixed) && + !(allocFlags & AllocationFlags::NoOverwrite); + + VirtualMemoryAllocation allocationInfo; + allocationInfo.flagsEx = blockFlagsEx | BlockFlagsEx::Allocated; + allocationInfo.device = file->device; + allocationInfo.prot = prot; + allocationInfo.deviceOffset = fileOffset; + allocationInfo.setName(process, name); + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + auto [_, errc, range] = vmem->map(addressHint, size, allocationInfo, allocFlags | AllocationFlags::Dry, alignment); @@ -163,48 +398,515 @@ std::pair orbis::vmem::map( if (errc != std::errc{}) { if (errc == std::errc::not_enough_memory) { // virtual memory shouldn't care about physical memory - return {{}, orbis::ErrorCode::INVAL}; + return {{}, ErrorCode::INVAL}; } return {{}, toErrorCode(errc)}; } - rx::EnumBitSet deviceProtection = {}; - - if (prot & Protection::CpuRead) { - deviceProtection |= rx::mem::Protection::R; + if (canOverwrite) { + if (auto errc = validateOverwrite(vmem, range, false); + errc != ErrorCode{}) { + return {{}, errc}; + } } - if (prot & Protection::CpuWrite) { - deviceProtection |= rx::mem::Protection::W; + auto budget = process->getBudget(); + + if (blockFlags & BlockFlags::PooledMemory) { + prot = {}; + allocationInfo.prot = prot; + blockFlags &= ~BlockFlags::Commited; } - if (prot & Protection::CpuExec) { - deviceProtection |= rx::mem::Protection::X; + if (blockFlags & BlockFlags::FlexibleMemory) { + if (!budget->acquire(BudgetResource::Fmem, size)) { + return {{}, ErrorCode::INVAL}; + } + + if (prot) { + blockFlags |= BlockFlags::Commited; + } } - if (auto error = device->map(range, deviceOffset, deviceProtection, process); + allocFlags = AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge); + + if (blockFlags & BlockFlags::DirectMemory) { + if (!budget->acquire(BudgetResource::Dmem, size)) { + return {{}, ErrorCode::INVAL}; + } + + allocFlags |= AllocationFlags::NoMerge; + + if (prot) { + blockFlags |= BlockFlags::Commited; + } + } + + if (blockFlags & BlockFlags::PooledMemory) { + if (auto errc = blockpool::allocateControlBlock(); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + allocationInfo.flagsEx |= BlockFlagsEx::PoolControl; + } + + if (auto error = file->device->map(range, fileOffset, prot, file, process); error != ErrorCode{}) { + if (blockFlags & BlockFlags::FlexibleMemory) { + budget->release(BudgetResource::Fmem, size); + } + + if (blockFlags & BlockFlags::DirectMemory) { + budget->release(BudgetResource::Dmem, size); + } + + if (blockFlags & BlockFlags::PooledMemory) { + blockpool::releaseControlBlock(); + } + return {{}, error}; } - auto [it, _errc, _range] = - vmem->map(range.beginAddress(), range.size(), allocationInfo, - AllocationFlags::Fixed | AllocationFlags::NoMerge, alignment); - - if (name.empty()) { - it->name = rx::format("anon:{:012x}", it.beginAddress()); + if (canOverwrite) { + release(process, vmem, budget, range); } + allocationInfo.flags = blockFlags; + + if (type == MemoryType::Invalid) { + if (blockFlags & BlockFlags::FlexibleMemory) { + type = MemoryType::WbOnion; + } else if (blockFlags & BlockFlags::DirectMemory) { + auto queryResult = dmem::query(0, fileOffset); + if (queryResult) { + type = queryResult->memoryType; + } else { + type = MemoryType::WbOnion; + } + } + allocationInfo.type = type; + } else { + if (blockFlags & BlockFlags::DirectMemory) { + if (auto errc = dmem::setType( + 0, rx::AddressRange::fromBeginSize(fileOffset, size), type); + errc != ErrorCode{}) { + rx::die("mapDirect: failed to set dmem type"); + } + } + } + + { + auto [_it, errc, _range] = vmem->map(range.beginAddress(), range.size(), + allocationInfo, allocFlags, alignment); + + rx::dieIf(errc != std::errc{}, "failed to commit virtual memory {}", errc); + } + + // vmemDump(process, rx::format("mapped {:x}-{:x}", range.beginAddress(), + // range.endAddress())); + return {range, {}}; } +std::pair orbis::vmem::mapDirect( + Process *process, std::uint64_t addressHint, rx::AddressRange directRange, + rx::EnumBitSet prot, rx::EnumBitSet allocFlags, + std::string_view name, std::uint64_t alignment, MemoryType type) { + ScopedBudgetAcquire dmemResource(process->getBudget(), BudgetResource::Dmem, + directRange.size()); + if (!dmemResource) { + return {{}, ErrorCode::INVAL}; + } + + VirtualMemoryAllocation allocationInfo; + allocationInfo.flags = orbis::vmem::BlockFlags::DirectMemory; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.prot = prot; + allocationInfo.device = g_context->dmem->device; + allocationInfo.deviceOffset = directRange.beginAddress(); + allocationInfo.type = type; + allocationInfo.setName(process, name); + + if (prot) { + allocationInfo.flags |= orbis::vmem::BlockFlags::Commited; + } + + bool canOverwrite = (allocFlags & AllocationFlags::Fixed) && + !(allocFlags & AllocationFlags::NoOverwrite); + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + rx::AddressRange range; + + { + auto [_, errc, vmemRange] = + vmem->map(addressHint, directRange.size(), allocationInfo, + allocFlags | AllocationFlags::Dry, alignment); + if (errc != std::errc{}) { + return {{}, toErrorCode(errc)}; + } + + range = vmemRange; + } + + if (canOverwrite) { + if (auto errc = validateOverwrite(vmem, range, false); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + } + + if (auto errc = dmem::map(0, range, directRange.beginAddress(), prot); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + + if (canOverwrite) { + release(process, vmem, process->getBudget(), range); + } + + if (type == MemoryType::Invalid) { + auto queryResult = dmem::query(0, directRange.beginAddress()); + rx::dieIf(!queryResult, "mapDirect: failed to query dmem type"); + type = queryResult->memoryType; + allocationInfo.type = type; + } else { + if (auto errc = dmem::setType(0, directRange, type); errc != ErrorCode{}) { + rx::die("mapDirect: failed to set dmem type"); + } + } + + vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, alignment); + + dmemResource.commit(); + + auto [pmemOffset, errc] = dmem::getPmemOffset(0, directRange.beginAddress()); + rx::dieIf(errc != ErrorCode{}, + "mapDirect: failed to query physical offset {}", errc); + + amdgpu::mapMemory(process->pid, range, type, prot, pmemOffset); + + // vmemDump(process, + // rx::format("mapped dmem {:x}-{:x}", range.beginAddress(), + // range.endAddress())); + + return {range, {}}; +} + +std::pair +orbis::vmem::mapFlex(Process *process, std::uint64_t size, + rx::EnumBitSet prot, std::uint64_t addressHint, + rx::EnumBitSet allocFlags, + rx::EnumBitSet blockFlags, + std::string_view name, std::uint64_t alignment) { + ScopedBudgetAcquire fmemResource(process->getBudget(), BudgetResource::Fmem, + size); + if (!fmemResource) { + return {{}, ErrorCode::INVAL}; + } + + bool canOverwrite = (allocFlags & AllocationFlags::Fixed) && + !(allocFlags & AllocationFlags::NoOverwrite); + + VirtualMemoryAllocation allocationInfo; + allocationInfo.flags = orbis::vmem::BlockFlags::FlexibleMemory | blockFlags; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.prot = prot; + allocationInfo.setName(process, name); + + if (prot) { + allocationInfo.flags |= orbis::vmem::BlockFlags::Commited; + } + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + rx::AddressRange vmemRange; + + { + auto [_, errc, range] = + vmem->map(addressHint, size, allocationInfo, + allocFlags | AllocationFlags::Dry, alignment); + if (errc != std::errc{}) { + return {{}, toErrorCode(errc)}; + } + + vmemRange = range; + } + + if (canOverwrite) { + if (auto errc = validateOverwrite(vmem, vmemRange, false); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + } + + rx::AddressRange flexRange; + + { + auto [range, errc] = fmem::allocate(size); + + if (errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + + flexRange = range; + } + + allocationInfo.deviceOffset = flexRange.beginAddress(); + + if (auto errc = + pmem::map(vmemRange.beginAddress(), flexRange, toCpuProtection(prot)); + errc != orbis::ErrorCode{}) { + fmem::deallocate(flexRange); + return {{}, errc}; + } + + if (canOverwrite) { + release(process, vmem, process->getBudget(), vmemRange); + } + + auto [it, _errc, _range] = vmem->map( + vmemRange.beginAddress(), vmemRange.size(), allocationInfo, + AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge), + alignment); + + // vmemDump(process, rx::format("mapFlex {:x}-{:x}", vmemRange.beginAddress(), + // vmemRange.endAddress())); + fmemResource.commit(); + return {vmemRange, {}}; +} + +std::pair +orbis::vmem::commitPooled(Process *process, rx::AddressRange range, + MemoryType type, rx::EnumBitSet prot) { + VirtualMemoryAllocation allocationInfo; + allocationInfo.flags = BlockFlags::PooledMemory | BlockFlags::Commited; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.prot = prot; + allocationInfo.type = type; + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end() || !it->isAllocated() || + it->flags != BlockFlags::PooledMemory || !it.range().contains(range)) { + return {{}, ErrorCode::INVAL}; + } + + allocationInfo.name = it->name; + + auto controlBlockIt = it; + + while (controlBlockIt != vmem->end() && controlBlockIt->isAllocated() && + (controlBlockIt->flags & BlockFlags::PooledMemory)) { + if (!controlBlockIt->prot && + controlBlockIt->flags == BlockFlags::PooledMemory && + (controlBlockIt->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl)) { + break; + } + + ++controlBlockIt; + } + + if (controlBlockIt != vmem->end() && + controlBlockIt->flags != BlockFlags::PooledMemory) { + controlBlockIt = vmem->end(); + } + + if (controlBlockIt == vmem->end()) { + auto rangeWithoutCtrl = it.range(); + rangeWithoutCtrl = rx::AddressRange::fromBeginEnd( + rangeWithoutCtrl.beginAddress(), + rangeWithoutCtrl.endAddress() - dmem::kPageSize); + + if (!rangeWithoutCtrl.contains(range)) { + return {{}, ErrorCode::INVAL}; + } + + if (auto errc = blockpool::commit(process, range, type, prot); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + + auto controlAllocationInfo = it.get(); + controlAllocationInfo.flags = BlockFlags::PooledMemory; + controlAllocationInfo.flagsEx = + BlockFlagsEx::Allocated | BlockFlagsEx::PoolControl; + controlAllocationInfo.deviceOffset = range.endAddress() - it.beginAddress(); + controlAllocationInfo.type = MemoryType::WbOnion; + + vmem->map(range.endAddress(), it.endAddress() - range.endAddress(), + controlAllocationInfo, AllocationFlags::Fixed, kPageSize); + } else { + if (auto errc = blockpool::commit(process, range, type, prot); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + } + + vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + + // vmemDump(process, rx::format("commitPooled {:x}-{:x}", + // range.beginAddress(), + // range.endAddress())); + return {range, {}}; +} + +orbis::ErrorCode orbis::vmem::decommitPooled(Process *process, + rx::AddressRange range) { + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + auto it = vmem->query(range.beginAddress()); + + while (it != vmem->end() && it.beginAddress() < range.endAddress()) { + if (!it->isAllocated() || + it->flags != (BlockFlags::PooledMemory | BlockFlags::Commited)) { + if (it->flagsEx & BlockFlagsEx::PoolControl) { + if (it.range().contains(range)) { + // ignore decommits of control block + return {}; + } + + if (it.endAddress() <= range.endAddress()) { + // if requested range overlaps with control block, adjust range to + // ignore control block + range = rx::AddressRange::fromBeginEnd(range.beginAddress(), + it.beginAddress()); + --it; + break; + } + } + + return ErrorCode::INVAL; + } + + ++it; + } + + if (it == vmem->end()) { + return ErrorCode::INVAL; + } + + blockpool::decommit(process, range); + auto allocationInfo = it.get(); + allocationInfo.prot = {}; + allocationInfo.flags = BlockFlags::PooledMemory; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.deviceOffset = 0; + auto unmapped = vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + + auto controlBlockIt = unmapped.it; + ++controlBlockIt; + + if (controlBlockIt != vmem->end() && + controlBlockIt->flags == BlockFlags::PooledMemory && + controlBlockIt->flagsEx & BlockFlagsEx::PoolControl) { + allocationInfo.flagsEx |= BlockFlagsEx::PoolControl; + // compress control block + + if (controlBlockIt.beginAddress() - controlBlockIt->deviceOffset != + unmapped.it.beginAddress()) { + // if not whole reserved space was decommited, preserve offset + allocationInfo.deviceOffset = controlBlockIt->deviceOffset; + } else { + allocationInfo.deviceOffset = 0; + } + + auto controlRange = unmapped.it.range().merge(controlBlockIt.range()); + + vmem->map(controlRange.beginAddress(), controlRange.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + } + + // vmemDump(process, rx::format("decommitPooled {:x}-{:x}", + // range.beginAddress(), + // range.endAddress())); + + return {}; +} + +orbis::ErrorCode orbis::vmem::protect(Process *process, rx::AddressRange range, + rx::EnumBitSet prot) { + auto vmem = process->get(g_vmInstance); + + range = rx::AddressRange::fromBeginEnd( + rx::alignDown(range.beginAddress(), kPageSize), + rx::alignUp(range.endAddress(), kPageSize)); + { + std::lock_guard lock(*vmem); + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end()) { + rx::println(stderr, + "vmem: attempt to set protection of invalid address range: " + "{:x}-{:x}", + range.beginAddress(), range.endAddress()); + return orbis::ErrorCode::INVAL; + } + + auto errc = validateRange(vmem, it, range, + [](const VirtualMemoryAllocation &alloc) { + if (alloc.flags == BlockFlags::PooledMemory) { + return ErrorCode::ACCES; + } + + if (alloc.flagsEx & BlockFlagsEx::Reserved) { + return ErrorCode::ACCES; + } + + return ErrorCode{}; + }); + + if (errc != ErrorCode{}) { + return errc; + } + + if (auto errc = + toErrorCode(rx::mem::protect(range, vmem::toCpuProtection(prot))); + errc != ErrorCode{}) { + return errc; + } + + modifyRange(vmem, it, range, + [prot](VirtualMemoryAllocation &alloc, rx::AddressRange) { + if (!alloc.isAllocated()) { + return; + } + + if (alloc.device != nullptr && + alloc.flags & BlockFlags::FlexibleMemory) { + alloc.prot = prot & ~Protection::CpuExec; + } + + alloc.flags = alloc.prot + ? alloc.flags | BlockFlags::Commited + : alloc.flags & ~BlockFlags::Commited; + }); + } + + amdgpu::protectMemory(process->pid, range, prot); + + // vmemDump(process, rx::format("protected {:x}-{:x} {}", + // range.beginAddress(), + // range.endAddress(), prot)); + + return {}; +} + orbis::ErrorCode orbis::vmem::setName(Process *process, rx::AddressRange range, std::string_view name) { auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - auto it = vmem->query(range.beginAddress()); if (it == vmem->end()) { @@ -215,48 +917,162 @@ orbis::ErrorCode orbis::vmem::setName(Process *process, rx::AddressRange range, return orbis::ErrorCode::INVAL; } - if (!it->isAllocated()) { - rx::println(stderr, - "vmem: attempt to set name of unallocated range: request " - "{:x}-{:x}, node: {:x}-{:x}, name {}", - range.beginAddress(), range.endAddress(), it.beginAddress(), - it.endAddress(), name); + modifyRange(vmem, it, range, + [name](VirtualMemoryAllocation &alloc, rx::AddressRange) { + if (alloc.isAllocated()) { + alloc.name = name; + } + }); + + return {}; +} + +orbis::ErrorCode orbis::vmem::setType(Process *process, rx::AddressRange range, + MemoryType type) { + auto vmem = process->get(g_vmInstance); + + std::lock_guard lock(*vmem); + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end()) { return orbis::ErrorCode::INVAL; } - if (it.range() != range) { - rx::println(stderr, - "vmem: set name range mismatch " - "{:x}-{:x}, node: {:x}-{:x}, name {}", - range.beginAddress(), range.endAddress(), it.beginAddress(), - it.endAddress(), name); + modifyRange( + vmem, it, range, + [type](VirtualMemoryAllocation &alloc, rx::AddressRange range) { + if (!alloc.isAllocated()) { + return; + } + + if (alloc.flags & BlockFlags::DirectMemory) { + dmem::setType( + 0, + rx::AddressRange::fromBeginSize(alloc.deviceOffset, range.size()), + type); + alloc.type = type; + return; + } + + if (alloc.flags != (BlockFlags::PooledMemory | BlockFlags::Commited)) { + alloc.type = type; + return; + } + }); + + return {}; +} + +orbis::ErrorCode +orbis::vmem::setTypeAndProtect(Process *process, rx::AddressRange range, + MemoryType type, + rx::EnumBitSet prot) { + auto vmem = process->get(g_vmInstance); + + std::lock_guard lock(*vmem); + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end()) { + return orbis::ErrorCode::INVAL; } - it->name = name; + auto errc = + validateRange(vmem, it, range, [](const VirtualMemoryAllocation &alloc) { + if (alloc.flags == BlockFlags::PooledMemory) { + return ErrorCode::ACCES; + } + + if (alloc.flagsEx & BlockFlagsEx::Reserved) { + return ErrorCode::ACCES; + } + + return ErrorCode{}; + }); + + if (errc != ErrorCode{}) { + return errc; + } + + if (auto errc = + toErrorCode(rx::mem::protect(range, vmem::toCpuProtection(prot))); + errc != ErrorCode{}) { + return errc; + } + + modifyRange( + vmem, it, range, + [type, prot](VirtualMemoryAllocation &alloc, rx::AddressRange range) { + if (alloc.isAllocated()) { + alloc.type = type; + alloc.prot = prot; + + if (alloc.flags & BlockFlags::DirectMemory) { + dmem::setType(0, + rx::AddressRange::fromBeginSize(alloc.deviceOffset, + range.size()), + type); + return; + } + } + }); + + amdgpu::protectMemory(process->pid, range, prot); return {}; } orbis::ErrorCode orbis::vmem::unmap(Process *process, rx::AddressRange range) { auto vmem = process->get(g_vmInstance); - - std::lock_guard lock(*vmem); + auto budget = process->getBudget(); VirtualMemoryAllocation allocationInfo{}; - auto [it, errc, _] = - vmem->map(range.beginAddress(), range.endAddress(), allocationInfo, - AllocationFlags::Fixed, kPageSize); + + range = rx::AddressRange::fromBeginEnd( + rx::alignDown(range.beginAddress(), kPageSize), + rx::alignUp(range.endAddress(), kPageSize)); + + orbis::ErrorCode result; + { + std::lock_guard lock(*vmem); + + if (auto errc = validateOverwrite(vmem, range, true); + errc != orbis::ErrorCode{}) { + return errc; + } + + release(process, vmem, budget, range); + + auto [it, errc, _] = + vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + + result = toErrorCode(errc); + } rx::mem::release(range, kPageSize); - return toErrorCode(errc); + amdgpu::unmapMemory(process->pid, range); + + // vmemDump(process, rx::format("unmap {:x}-{:x}", range.beginAddress(), + // range.endAddress())); + return result; } std::optional -orbis::vmem::query(Process *process, std::uint64_t address) { +orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) { auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - auto it = vmem->query(address); - if (it == vmem->end()) { + auto it = vmem->lowerBound(address); + + if (lowerBound) { + while (it != vmem->end() && !it->isAllocated()) { + ++it; + } + + if (it == vmem->end()) { + return {}; + } + } else if (it == vmem->end() || !it.range().contains(address) || + !it->isAllocated()) { return {}; } @@ -264,30 +1080,54 @@ orbis::vmem::query(Process *process, std::uint64_t address) { result.start = it.beginAddress(); result.end = it.endAddress(); - if (it->isAllocated()) { - if (it->flags == BlockFlags::DirectMemory) { - result.offset = it->deviceOffset; - } - - result.protection = it->prot.toUnderlying(); - result.flags = it->flags.toUnderlying(); - result.name = it->name; + if (!(it->flags & BlockFlags::FlexibleMemory) || it->device != nullptr) { + result.offset = it->deviceOffset; } + if (it->flags & BlockFlags::DirectMemory) { + // if (auto queryResult = dmem::query(0, it->deviceOffset)) { + result.memoryType = it->type; + // } + } else if (it->flags == (BlockFlags::PooledMemory | BlockFlags::Commited)) { + result.memoryType = it->type; + } + + result.protection = it->prot; + result.flags = it->flags; + result.name = it->name; + + // vmemDump(process, rx::format("query result {:x}-{:x} {:x} {} {} {}", + // result.start, result.end, result.offset, + // result.protection, result.flags, + // result.name)); + return result; } std::optional -orbis::vmem::queryProtection(Process *process, std::uint64_t address) { +orbis::vmem::queryProtection(Process *process, std::uint64_t address, + bool lowerBound) { auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - auto it = vmem->query(address); + auto it = vmem->lowerBound(address); if (it == vmem->end()) { return {}; } + if (lowerBound) { + while (it != vmem->end() && !it->isAllocated()) { + ++it; + } + + if (it == vmem->end()) { + return {}; + } + } else if (!it.range().contains(address) || !it->isAllocated()) { + return {}; + } + orbis::vmem::MemoryProtection result{}; result.startAddress = it.beginAddress(); result.endAddress = it.endAddress(); diff --git a/rpcsx/CMakeLists.txt b/rpcsx/CMakeLists.txt index b6830bc30..96ae16ff9 100644 --- a/rpcsx/CMakeLists.txt +++ b/rpcsx/CMakeLists.txt @@ -72,7 +72,6 @@ if(LINUX AND WITH_PS4) main.cpp AudioOut.cpp backtrace.cpp - vm.cpp ops.cpp linker.cpp io-device.cpp diff --git a/rpcsx/gpu/Device.cpp b/rpcsx/gpu/Device.cpp index 5c2f764e1..858083414 100644 --- a/rpcsx/gpu/Device.cpp +++ b/rpcsx/gpu/Device.cpp @@ -7,6 +7,8 @@ #include "orbis-config.hpp" #include "orbis/KernelContext.hpp" #include "orbis/note.hpp" +#include "orbis/pmem.hpp" +#include "orbis/vmem.hpp" #include "rx/AddressRange.hpp" #include "rx/Config.hpp" #include "rx/bits.hpp" @@ -135,8 +137,8 @@ static vk::Context createVkContext(Device *device) { &device->debugMessenger)); } - glfwCreateWindowSurface(vk::context->instance, device->window, nullptr, - &device->surface); + VK_VERIFY(glfwCreateWindowSurface(vk::context->instance, device->window, + nullptr, &device->surface)); result.createDevice(device->surface, rx::g_config.gpuIndex, { @@ -293,18 +295,6 @@ Device::~Device() { vk::context->allocator); } - for (auto fd : dmemFd) { - if (fd >= 0) { - ::close(fd); - } - } - - for (auto &[pid, info] : processInfo) { - if (info.vmFd >= 0) { - ::close(info.vmFd); - } - } - for (auto &cachePage : cachePages) { orbis::kfree(cachePage, kCachePageSize); } @@ -321,28 +311,6 @@ void Device::start() { }); } - for (std::size_t i = 0; i < std::size(dmemFd); ++i) { - if (dmemFd[i] != -1) { - continue; - } - - auto path = rx::format("{}/dmem-{}", rx::getShmPath(), i); - if (!std::filesystem::exists(path)) { - std::println("Waiting for dmem {}", i); - - while (!std::filesystem::exists(path)) { - std::this_thread::sleep_for(std::chrono::milliseconds(300)); - } - } - - dmemFd[i] = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR); - - if (dmemFd[i] < 0) { - std::println(stderr, "failed to open dmem {}", path); - std::abort(); - } - } - std::jthread vblankThread([](const std::stop_token &stopToken) { orbis::g_context->deviceEventEmitter->emit( orbis::kEvFiltDisplay, @@ -573,45 +541,27 @@ void Device::mapProcess(std::uint32_t pid, int vmId) { auto memory = amdgpu::RemoteMemory{vmId}; - std::string pidVmName = rx::format("{}/memory-{}", rx::getShmPath(), pid); - int memoryFd = ::open(pidVmName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); - process.vmFd = memoryFd; - - if (memoryFd < 0) { - std::println("failed to open shared memory of process {}", (int)pid); - std::abort(); - } - for (auto slot : process.vmTable) { - auto gpuProt = slot->prot >> 4; - if (gpuProt == 0) { + auto gpuProt = orbis::vmem::toGpuProtection(slot->prot); + if (!gpuProt) { continue; } auto devOffset = slot->offset + slot.beginAddress() - slot->baseAddress; - int mapFd = memoryFd; - if (slot->memoryType >= 0) { - mapFd = dmemFd[slot->memoryType]; - } + auto errc = orbis::pmem::map( + memory.getVirtualAddress(slot.beginAddress()), + rx::AddressRange::fromBeginSize(devOffset, slot.size()), gpuProt); + rx::dieIf( + errc != orbis::ErrorCode{}, + "failed to map process {} memory, address {}-{}, type {}, vmId {}", + (int)pid, memory.getPointer(slot.beginAddress()), + memory.getPointer(slot.endAddress()), slot->memoryType, vmId); - auto mmapResult = - ::mmap(memory.getPointer(slot.beginAddress()), slot.size(), gpuProt, - MAP_FIXED | MAP_SHARED, mapFd, devOffset); - - if (mmapResult == MAP_FAILED) { - std::println( - stderr, - "failed to map process {} memory, address {}-{}, type {:x}, vmId {}", - (int)pid, memory.getPointer(slot.beginAddress()), - memory.getPointer(slot.endAddress()), slot->memoryType, vmId); - std::abort(); - } - - // std::println(stderr, - // "map process {} memory, address {}-{}, type {:x}, vmId {}", - // (int)pid, memory.getPointer(startAddress), - // memory.getPointer(endAddress), slot.memoryType, vmId); + std::println(stderr, "map process {} memory, address {}-{}, vmId {}", + (int)pid, memory.getVirtualAddress(slot.beginAddress()), + memory.getVirtualAddress(slot.beginAddress()) + slot.size(), + vmId); } } @@ -629,13 +579,12 @@ void Device::unmapProcess(std::uint32_t pid) { rx::die("failed to release userspace memory: {}", (int)errc); } - ::close(process.vmFd); - process.vmFd = -1; process.vmId = -1; } void Device::protectMemory(std::uint32_t pid, std::uint64_t address, - std::uint64_t size, int prot) { + std::uint64_t size, + rx::EnumBitSet prot) { auto &process = processInfo[pid]; auto vmSlotIt = process.vmTable.queryArea(address); @@ -648,22 +597,20 @@ void Device::protectMemory(std::uint32_t pid, std::uint64_t address, process.vmTable.map(rx::AddressRange::fromBeginSize(address, size), VmMapSlot{ .memoryType = vmSlot.memoryType, - .prot = static_cast(prot), + .prot = prot, .offset = vmSlot.offset, .baseAddress = vmSlot.baseAddress, }); if (process.vmId >= 0) { auto memory = amdgpu::RemoteMemory{process.vmId}; - rx::mem::protect( - rx::AddressRange::fromBeginSize(memory.getVirtualAddress(address), - size), - rx::EnumBitSet::fromUnderlying(prot >> 4)); + rx::mem::protect(rx::AddressRange::fromBeginSize( + memory.getVirtualAddress(address), size), + orbis::vmem::toGpuProtection(prot)); - // std::println(stderr, "protect process {} memory, address {}-{}, prot - // {:x}", - // (int)pid, memory.getPointer(address), - // memory.getPointer(address + size), prot); + std::println(stderr, "protect process {} memory, address {}-{}, prot {}", + (int)pid, memory.getPointer(address), + memory.getPointer(address + size), prot); } } @@ -1006,17 +953,18 @@ void Device::waitForIdle() { } } -void Device::mapMemory(std::uint32_t pid, std::uint64_t address, - std::uint64_t size, int memoryType, int dmemIndex, - int prot, std::int64_t offset) { +void Device::mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t physicalOffset) { auto &process = processInfo[pid]; - process.vmTable.map(rx::AddressRange::fromBeginSize(address, size), + process.vmTable.map(virtualRange, VmMapSlot{ - .memoryType = memoryType >= 0 ? dmemIndex : -1, + .memoryType = memoryType, .prot = prot, - .offset = offset, - .baseAddress = address, + .offset = physicalOffset, + .baseAddress = virtualRange.beginAddress(), }); if (process.vmId < 0) { @@ -1025,33 +973,29 @@ void Device::mapMemory(std::uint32_t pid, std::uint64_t address, auto memory = amdgpu::RemoteMemory{process.vmId}; - int mapFd = process.vmFd; - - if (memoryType >= 0) { - mapFd = dmemFd[dmemIndex]; + auto vmemAddress = memory.getVirtualAddress(virtualRange.beginAddress()); + auto errc = orbis::pmem::map(vmemAddress, + rx::AddressRange::fromBeginSize( + physicalOffset, virtualRange.beginAddress()), + orbis::vmem::toGpuProtection(prot)); + if (errc != orbis::ErrorCode{}) { + rx::die("failed to map process {} memory, address {:x}-{:x}, type {}, " + "offset {:x}, prot {}, error {}", + pid, vmemAddress, vmemAddress + virtualRange.size(), memoryType, + physicalOffset, prot, errc); } - auto mmapResult = ::mmap(memory.getPointer(address), size, prot >> 4, - MAP_FIXED | MAP_SHARED, mapFd, offset); - - if (mmapResult == MAP_FAILED) { - perror("::mmap"); - - rx::die("failed to map process {} memory, address {}-{}, type {:x}, offset " - "{:x}, prot {:x}", - pid, memory.getPointer(address), memory.getPointer(address + size), - memoryType, offset, prot); - } - - // std::println(stderr, "map memory of process {}, address {}-{}, prot {:x}", - // (int)pid, memory.getPointer(address), - // memory.getPointer(address + size), prot); + std::println( + stderr, + "map memory of process {}, address {:x}-{:x}, prot {}, phy memory {:x}", + (int)pid, vmemAddress, vmemAddress + virtualRange.size(), prot, + physicalOffset); } void Device::unmapMemory(std::uint32_t pid, std::uint64_t address, std::uint64_t size) { // TODO - protectMemory(pid, address, size, 0); + protectMemory(pid, address, size, {}); } static void notifyPageChanges(Device *device, int vmId, std::uint32_t firstPage, diff --git a/rpcsx/gpu/Device.hpp b/rpcsx/gpu/Device.hpp index 5a17f169b..200e5d990 100644 --- a/rpcsx/gpu/Device.hpp +++ b/rpcsx/gpu/Device.hpp @@ -5,6 +5,11 @@ #include "Pipe.hpp" #include "amdgpu/tiler_vulkan.hpp" #include "orbis/KernelAllocator.hpp" +#include "orbis/dmem.hpp" +#include "orbis/thread/Process.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/MemoryTable.hpp" #include "rx/Rc.hpp" #include "rx/SharedMutex.hpp" @@ -37,9 +42,9 @@ std::array createPm4Packet(std::uint32_t op, } struct VmMapSlot { - int memoryType; - int prot; - std::int64_t offset; + orbis::MemoryType memoryType; + rx::EnumBitSet prot; + std::uint64_t offset; std::uint64_t baseAddress; auto operator<=>(const VmMapSlot &) const = default; @@ -47,7 +52,6 @@ struct VmMapSlot { struct ProcessInfo { int vmId = -1; - int vmFd = -1; BufferAttribute bufferAttributes[10]; Buffer buffers[10]; rx::MemoryTableWithPayload vmTable; @@ -90,7 +94,6 @@ struct Device : rx::RcBase, DeviceContext { std::jthread cacheUpdateThread; - int dmemFd[3] = {-1, -1, -1}; orbis::kmap processInfo; Cache caches[kMaxProcessCount]{ @@ -123,7 +126,8 @@ struct Device : rx::RcBase, DeviceContext { void mapProcess(std::uint32_t pid, int vmId); void unmapProcess(std::uint32_t pid); void protectMemory(std::uint32_t pid, std::uint64_t address, - std::uint64_t size, int prot); + std::uint64_t size, + rx::EnumBitSet prot); void onCommandBuffer(std::uint32_t pid, int cmdHeader, std::uint64_t address, std::uint64_t size); bool processPipes(); @@ -131,8 +135,10 @@ struct Device : rx::RcBase, DeviceContext { VkImage swapchainImage, VkImageView swapchainImageView); void flip(std::uint32_t pid, int bufferIndex, std::uint64_t arg); void waitForIdle(); - void mapMemory(std::uint32_t pid, std::uint64_t address, std::uint64_t size, - int memoryType, int dmemIndex, int prot, std::int64_t offset); + void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t physicalOffset); void unmapMemory(std::uint32_t pid, std::uint64_t address, std::uint64_t size); void watchWrites(int vmId, std::uint64_t address, std::uint64_t size); diff --git a/rpcsx/gpu/DeviceCtl.cpp b/rpcsx/gpu/DeviceCtl.cpp index a9232e4a1..4750fc600 100644 --- a/rpcsx/gpu/DeviceCtl.cpp +++ b/rpcsx/gpu/DeviceCtl.cpp @@ -1,6 +1,7 @@ #include "DeviceCtl.hpp" #include "Device.hpp" #include "gnm/pm4.hpp" +#include "orbis/KernelContext.hpp" #include "orbis/error/ErrorCode.hpp" #include "rx/bits.hpp" #include "rx/die.hpp" @@ -230,3 +231,26 @@ void DeviceCtl::submitComputeQueue(std::uint32_t meId, std::uint32_t pipeId, void DeviceCtl::start() { mDevice->start(); } void DeviceCtl::waitForIdle() { mDevice->waitForIdle(); } + +void amdgpu::mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset) { + if (auto gpu = DeviceCtl{orbis::g_context->gpuDevice.rawCast()}) { + gpu.submitMapMemory(pid, virtualRange.beginAddress(), virtualRange.size(), + (int)memoryType, 0, prot.toUnderlying(), offset); + } +} +void amdgpu::unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange) { + if (auto gpu = DeviceCtl{orbis::g_context->gpuDevice.rawCast()}) { + gpu.submitUnmapMemory(pid, virtualRange.beginAddress(), + virtualRange.size()); + } +} +void amdgpu::protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot) { + if (auto gpu = DeviceCtl{orbis::g_context->gpuDevice.rawCast()}) { + gpu.submitProtectMemory(pid, virtualRange.beginAddress(), + virtualRange.size(), prot.toUnderlying()); + } +} diff --git a/rpcsx/gpu/DeviceCtl.hpp b/rpcsx/gpu/DeviceCtl.hpp index bdd4e1c7f..e3b41ea88 100644 --- a/rpcsx/gpu/DeviceCtl.hpp +++ b/rpcsx/gpu/DeviceCtl.hpp @@ -2,6 +2,10 @@ #include "DeviceContext.hpp" #include "orbis-config.hpp" +#include "orbis/dmem.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/Rc.hpp" #include #include @@ -58,4 +62,12 @@ public: explicit operator bool() const { return mDevice != nullptr; } }; + +void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset); +void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange); +void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot); } // namespace amdgpu diff --git a/rpcsx/gpu/Pipe.cpp b/rpcsx/gpu/Pipe.cpp index 741cb6376..9158e9198 100644 --- a/rpcsx/gpu/Pipe.cpp +++ b/rpcsx/gpu/Pipe.cpp @@ -5,6 +5,9 @@ #include "gnm/mmio.hpp" #include "gnm/pm4.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/print.hpp" #include "vk.hpp" #include @@ -2221,9 +2224,10 @@ void CommandPipe::mapMemory(Ring &ring) { auto addressHi = ring.rptr[3]; auto sizeLo = ring.rptr[4]; auto sizeHi = ring.rptr[5]; - auto memoryType = ring.rptr[6]; - auto dmemIndex = ring.rptr[7]; - auto prot = ring.rptr[8]; + auto memoryType = orbis::MemoryType(ring.rptr[6]); + // auto dmemIndex = ring.rptr[7]; + auto prot = + rx::EnumBitSet::fromUnderlying(ring.rptr[8]); auto offsetLo = ring.rptr[9]; auto offsetHi = ring.rptr[10]; @@ -2231,7 +2235,8 @@ void CommandPipe::mapMemory(Ring &ring) { auto size = sizeLo | (static_cast(sizeHi) << 32); auto offset = offsetLo | (static_cast(offsetHi) << 32); - device->mapMemory(pid, address, size, memoryType, dmemIndex, prot, offset); + device->mapMemory(pid, rx::AddressRange::fromBeginSize(address, size), + memoryType, prot, offset); } void CommandPipe::unmapMemory(Ring &ring) { auto pid = ring.rptr[1]; @@ -2250,7 +2255,8 @@ void CommandPipe::protectMemory(Ring &ring) { auto addressHi = ring.rptr[3]; auto sizeLo = ring.rptr[4]; auto sizeHi = ring.rptr[5]; - auto prot = ring.rptr[6]; + auto prot = + rx::EnumBitSet::fromUnderlying(ring.rptr[6]); auto address = addressLo | (static_cast(addressHi) << 32); auto size = sizeLo | (static_cast(sizeHi) << 32); diff --git a/rpcsx/io-device.cpp b/rpcsx/io-device.cpp index b42134158..cea81ce8c 100644 --- a/rpcsx/io-device.cpp +++ b/rpcsx/io-device.cpp @@ -1,14 +1,16 @@ #include "io-device.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/SocketAddress.hpp" +#include "orbis/error.hpp" #include "orbis/file.hpp" #include "orbis/stat.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/uio.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/Mappable.hpp" #include "vfs.hpp" -#include "vm.hpp" #include #include #include @@ -33,8 +35,8 @@ struct HostFile : orbis::File { bool alignTruncate = false; ~HostFile() { - if (hostFd > 0 && closeOnExit) { - ::close(hostFd); + if (!closeOnExit && hostFd) { + static_cast(hostFd.release()); } } }; @@ -46,12 +48,6 @@ struct SocketFile : orbis::File { int prot = -1; orbis::kmap> options; - - ~SocketFile() { - if (hostFd > 0) { - ::close(hostFd); - } - } }; static orbis::ErrorCode convertErrc(std::errc errc) { @@ -319,7 +315,7 @@ static orbis::ErrorCode host_read(orbis::File *file, orbis::Uio *uio, if (!hostFile->dirEntries.empty()) return orbis::ErrorCode::ISDIR; - return host_fd_read(hostFile->hostFd, uio); + return host_fd_read(hostFile->hostFd.native_handle(), uio); } static orbis::ErrorCode host_write(orbis::File *file, orbis::Uio *uio, @@ -328,73 +324,14 @@ static orbis::ErrorCode host_write(orbis::File *file, orbis::Uio *uio, if (!hostFile->dirEntries.empty()) return orbis::ErrorCode::ISDIR; - return host_fd_write(hostFile->hostFd, uio); -} - -static orbis::ErrorCode host_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - auto hostFile = static_cast(file); - if (!hostFile->dirEntries.empty()) - return orbis::ErrorCode::ISDIR; - - auto result = vm::map(*address, size, prot, flags, - vm::kMapInternalReserveOnly, hostFile->device.get(), offset); - - if (result == (void *)-1) { - return orbis::ErrorCode::NOMEM; - } - - size = rx::alignUp(size, vm::kPageSize); - - result = - ::mmap(result, size, prot & vm::kMapProtCpuAll, - ((flags & vm::kMapFlagPrivate) != 0 ? MAP_PRIVATE : MAP_SHARED) | - MAP_FIXED, - hostFile->hostFd, offset); - if (result == (void *)-1) { - auto errc = convertErrno(); - std::printf("Failed to map file at %p-%p\n", *address, - (char *)*address + size); - return errc; - } - - std::printf("file mapped at %p-%p:%lx\n", result, (char *)result + size, - offset); - - struct stat stat; - fstat(hostFile->hostFd, &stat); - if (stat.st_size < offset + size) { - std::size_t rest = std::min(offset + size - stat.st_size, vm::kPageSize); - - if (rest > rx::mem::pageSize) { - auto fillSize = rx::alignUp(rest, rx::mem::pageSize) - rx::mem::pageSize; - - std::printf("adding dummy mapping %p-%p, file ends at %p\n", - (char *)result + size - fillSize, (char *)result + size, - (char *)result + (stat.st_size - offset)); - - auto ptr = ::mmap((char *)result + size - fillSize, fillSize, - prot & vm::kMapProtCpuAll, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); - - if (ptr == (void *)-1) { - std::printf("failed to add dummy mapping %p-%p\n", result, - (char *)result + size); - } - } - } - - *address = result; - return {}; + return host_fd_write(hostFile->hostFd.native_handle(), uio); } static orbis::ErrorCode host_stat(orbis::File *file, orbis::Stat *sb, orbis::Thread *thread) { auto hostFile = static_cast(file); struct stat hostStat; - ::fstat(hostFile->hostFd, &hostStat); + ::fstat(hostFile->hostFd.native_handle(), &hostStat); sb->dev = hostStat.st_dev; // TODO sb->ino = hostStat.st_ino; sb->mode = hostStat.st_mode; @@ -437,10 +374,10 @@ static orbis::ErrorCode host_truncate(orbis::File *file, std::uint64_t len, } if (hostFile->alignTruncate) { - len = rx::alignUp(len, vm::kPageSize); + len = rx::alignUp(len, orbis::vmem::kPageSize); } - if (::ftruncate(hostFile->hostFd, len)) { + if (::ftruncate(hostFile->hostFd.native_handle(), len)) { return convertErrno(); } @@ -458,7 +395,7 @@ static orbis::ErrorCode socket_read(orbis::File *file, orbis::Uio *uio, orbis::Thread *) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { while (true) { std::this_thread::sleep_for(std::chrono::days(1)); } @@ -468,14 +405,14 @@ static orbis::ErrorCode socket_read(orbis::File *file, orbis::Uio *uio, if (uio->iov->len) { ORBIS_LOG_FATAL(__FUNCTION__, file, uio->iov->len); } - return host_fd_read(socket->hostFd, uio); + return host_fd_read(socket->hostFd.native_handle(), uio); } static orbis::ErrorCode socket_write(orbis::File *file, orbis::Uio *uio, orbis::Thread *) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { for (auto io : std::span(uio->iov, uio->iovcnt)) { uio->offset += io.len; } @@ -483,7 +420,7 @@ static orbis::ErrorCode socket_write(orbis::File *file, orbis::Uio *uio, } ORBIS_LOG_FATAL(__FUNCTION__, file, uio->iov->len); - return host_fd_write(socket->hostFd, uio); + return host_fd_write(socket->hostFd.native_handle(), uio); } static orbis::ErrorCode socket_bind(orbis::File *file, @@ -492,7 +429,7 @@ static orbis::ErrorCode socket_bind(orbis::File *file, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { return {}; } @@ -512,8 +449,8 @@ static orbis::ErrorCode socket_bind(orbis::File *file, sockaddr_un un{.sun_family = AF_UNIX}; std::strncpy(un.sun_path, socketPath.c_str(), sizeof(un.sun_path)); - if (::bind(socket->hostFd, reinterpret_cast<::sockaddr *>(&un), - sizeof(un)) < 0) { + if (::bind(socket->hostFd.native_handle(), + reinterpret_cast<::sockaddr *>(&un), sizeof(un)) < 0) { return convertErrno(); } @@ -528,11 +465,11 @@ static orbis::ErrorCode socket_listen(orbis::File *file, int backlog, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { return {}; } - if (::listen(socket->hostFd, backlog) < 0) { + if (::listen(socket->hostFd.native_handle(), backlog) < 0) { return convertErrno(); } @@ -545,7 +482,7 @@ static orbis::ErrorCode socket_accept(orbis::File *file, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { ORBIS_LOG_ERROR(__FUNCTION__, socket->name, "wait forever"); while (true) { @@ -558,8 +495,8 @@ static orbis::ErrorCode socket_accept(orbis::File *file, if (socket->dom == 1 && socket->type == 1 && socket->prot == 0) { sockaddr_un un{.sun_family = AF_UNIX}; socklen_t len = sizeof(un); - int result = - ::accept(socket->hostFd, reinterpret_cast(&un), &len); + int result = ::accept(socket->hostFd.native_handle(), + reinterpret_cast(&un), &len); if (result < 0) { return convertErrno(); @@ -585,7 +522,7 @@ static orbis::ErrorCode socket_connect(orbis::File *file, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (socket->hostFd.native_handle()) { return orbis::ErrorCode::CONNREFUSED; } @@ -604,8 +541,8 @@ static orbis::ErrorCode socket_connect(orbis::File *file, sockaddr_un un{.sun_family = AF_UNIX}; std::strncpy(un.sun_path, socketPath.c_str(), sizeof(un.sun_path)); - if (::connect(socket->hostFd, reinterpret_cast<::sockaddr *>(&un), - sizeof(un)) < 0) { + if (::connect(socket->hostFd.native_handle(), + reinterpret_cast<::sockaddr *>(&un), sizeof(un)) < 0) { return convertErrno(); } @@ -650,12 +587,13 @@ orbis::ErrorCode socket_recvfrom(orbis::File *file, void *buf, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { return orbis::ErrorCode::CONNREFUSED; } if (socket->dom == 1 && socket->type == 1 && socket->prot == 0) { - auto count = ::recvfrom(socket->hostFd, buf, len, flags, nullptr, nullptr); + auto count = ::recvfrom(socket->hostFd.native_handle(), buf, len, flags, + nullptr, nullptr); if (count < 0) { return convertErrno(); } @@ -678,7 +616,6 @@ static const orbis::FileOps hostOps = { .write = host_write, .truncate = host_truncate, .stat = host_stat, - .mmap = host_mmap, }; static const orbis::FileOps socketOps = { @@ -710,7 +647,7 @@ rx::Ref wrapSocket(int hostFd, orbis::kstring name, int dom, s->dom = dom; s->type = type; s->prot = prot; - s->hostFd = hostFd; + s->hostFd = rx::Mappable::CreateFromNativeHandle(hostFd); s->ops = &socketOps; return s; } @@ -736,7 +673,7 @@ orbis::ErrorCode createSocket(rx::Ref *file, orbis::kstring name, static std::optional findFileInDir(const std::filesystem::path &dir, const char *name) { - for (auto entry : std::filesystem::directory_iterator(dir)) { + for (auto &entry : std::filesystem::directory_iterator(dir)) { auto entryName = entry.path().filename(); if (strcasecmp(entryName.c_str(), name) == 0) { return entryName; @@ -752,7 +689,7 @@ toRealPath(const std::filesystem::path &inp) { } std::filesystem::path result; - for (auto elem : inp) { + for (auto &elem : inp) { if (result.empty() || std::filesystem::exists(result / elem)) { result /= elem; continue; @@ -887,7 +824,7 @@ orbis::ErrorCode HostFsDevice::open(rx::Ref *file, } auto newFile = orbis::knew(); - newFile->hostFd = hostFd; + newFile->hostFd = rx::Mappable::CreateFromNativeHandle(hostFd); newFile->dirEntries = std::move(dirEntries); newFile->ops = &hostOps; newFile->device = this; @@ -936,10 +873,10 @@ orbis::ErrorCode HostFsDevice::rename(const char *from, const char *to, return convertErrorCode(ec); } -orbis::File *createHostFile(int hostFd, rx::Ref device, +orbis::File *createHostFile(int hostFd, orbis::IoDevice *device, bool alignTruncate) { auto newFile = orbis::knew(); - newFile->hostFd = hostFd; + newFile->hostFd = rx::Mappable::CreateFromNativeHandle(hostFd); newFile->ops = &hostOps; newFile->device = device; newFile->alignTruncate = alignTruncate; diff --git a/rpcsx/io-device.hpp b/rpcsx/io-device.hpp index bf0394251..b4bd66cfc 100644 --- a/rpcsx/io-device.hpp +++ b/rpcsx/io-device.hpp @@ -35,6 +35,6 @@ rx::Ref wrapSocket(int hostFd, orbis::kstring name, int dom, int type, int prot); orbis::ErrorCode createSocket(rx::Ref *file, orbis::kstring name, int dom, int type, int prot); -orbis::File *createHostFile(int hostFd, rx::Ref device, +orbis::File *createHostFile(int hostFd, orbis::IoDevice *device, bool alignTruncate = false); orbis::IoDevice *createFdWrapDevice(int fd); diff --git a/rpcsx/io-devices.hpp b/rpcsx/io-devices.hpp index 6c6e260cb..05e55d31f 100644 --- a/rpcsx/io-devices.hpp +++ b/rpcsx/io-devices.hpp @@ -5,9 +5,10 @@ namespace orbis { struct IoDevice; +struct Process; } -orbis::IoDevice *createDceCharacterDevice(); +orbis::IoDevice *createDceCharacterDevice(orbis::Process *process); orbis::IoDevice *createDipswCharacterDevice(); orbis::IoDevice *createDmemCharacterDevice(int index); orbis::IoDevice *createGcCharacterDevice(); diff --git a/rpcsx/iodev/ajm.cpp b/rpcsx/iodev/ajm.cpp index 858a1e7a4..9c2820b5c 100644 --- a/rpcsx/iodev/ajm.cpp +++ b/rpcsx/iodev/ajm.cpp @@ -68,7 +68,7 @@ struct AjmDevice AjmDevice(); }; -AVSampleFormat ajmToAvFormat(AJMFormat ajmFormat) { +static AVSampleFormat ajmToAvFormat(AJMFormat ajmFormat) { switch (ajmFormat) { case AJM_FORMAT_S16: return AV_SAMPLE_FMT_S16; @@ -81,7 +81,7 @@ AVSampleFormat ajmToAvFormat(AJMFormat ajmFormat) { } } -void reset(Instance *instance) { +static void reset(Instance *instance) { instance->gapless.skipSamples = 0; instance->gapless.totalSamples = 0; instance->gapless.totalSkippedSamples = 0; @@ -90,7 +90,7 @@ void reset(Instance *instance) { instance->aac.framesSkipped = 0; } -void resetAt9(Instance *instance) { +static void resetAt9(Instance *instance) { if (instance->at9.configData) { Atrac9ReleaseHandle(instance->at9.handle); instance->at9.estimatedSizeUsed = 0; @@ -316,7 +316,8 @@ ajm_ioctl_start_batch_buffer(orbis::Thread *, AjmDevice *device, args.result = 0; args.batchId = device->batchId++; // ORBIS_LOG_ERROR(__FUNCTION__, args.result, args.unk0, args.pBatch, - // args.batchSize, args.priority, args.batchError, args.batchId); + // args.batchSize, args.priority, args.batchError, + // args.batchId); // thread->where(); auto ptr = args.pBatch; diff --git a/rpcsx/iodev/blockpool.cpp b/rpcsx/iodev/blockpool.cpp index c9f1f1a80..974ff463f 100644 --- a/rpcsx/iodev/blockpool.cpp +++ b/rpcsx/iodev/blockpool.cpp @@ -1,107 +1,114 @@ -#include "blockpool.hpp" -#include "dmem.hpp" +#include "orbis/blockpool.hpp" #include "orbis/IoDevice.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/dmem.hpp" #include "orbis/file.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" -#include "vm.hpp" -#include -#include -#include +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" + +enum { + BLOCKPOOL_IOCTL_EXPAND = 0xc020a801, + BLOCKPOOL_IOCTL_GET_BLOCK_STATS = 0x4010a802, +}; + +struct BlockPoolAllocation { + bool allocated = false; + + [[nodiscard]] bool isAllocated() const { return allocated; } + [[nodiscard]] bool isRelated(const BlockPoolAllocation &, rx::AddressRange, + rx::AddressRange) const { + return true; + } + + [[nodiscard]] BlockPoolAllocation merge(const BlockPoolAllocation &other, + rx::AddressRange, + rx::AddressRange) const { + return other; + } + + bool operator==(const BlockPoolAllocation &) const = default; +}; + +using DirectMemoryResource = + kernel::AllocableResource; struct BlockPoolFile : public orbis::File {}; +struct BlockPoolDevice + : orbis::IoDeviceWithIoctl { + rx::shared_mutex mtx; + rx::MemoryAreaTable<> pool; + orbis::sint availBlocks{}; + orbis::sint commitBlocks{}; -static orbis::ErrorCode blockpool_ioctl(orbis::File *file, - std::uint64_t request, void *argp, - orbis::Thread *thread) { - auto blockPool = static_cast(file->device.get()); - std::lock_guard lock(blockPool->mtx); + BlockPoolDevice(); - switch (request) { - case 0xc020a801: { - struct Args { - std::uint64_t len; - std::uint64_t searchStart; - std::uint64_t searchEnd; - std::uint32_t flags; - }; - auto args = reinterpret_cast(argp); - ORBIS_LOG_TODO("blockpool expand", args->len, args->searchStart, - args->searchEnd, args->flags); - - auto dmem = orbis::g_context->dmemDevice.rawStaticCast(); - std::lock_guard lock(dmem->mtx); - std::uint64_t start = args->searchStart; - ORBIS_RET_ON_ERROR( - dmem->allocate(&start, args->searchEnd, args->len, 1, args->flags)); - - blockPool->pool.map(rx::AddressRange::fromBeginSize(start, args->len)); - return {}; - } - } - - ORBIS_LOG_FATAL("Unhandled blockpool ioctl", request); - thread->where(); - return {}; -} - -static orbis::ErrorCode blockpool_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - auto blockPool = static_cast(file->device.get()); - std::lock_guard lock(blockPool->mtx); - ORBIS_LOG_FATAL("blockpool mmap", *address, size, offset); - - std::size_t totalBlockPoolSize = 0; - for (auto entry : blockPool->pool) { - totalBlockPoolSize += entry.endAddress - entry.beginAddress; - } - - if (totalBlockPoolSize < size) { - return orbis::ErrorCode::NOMEM; - } - - auto dmem = orbis::g_context->dmemDevice.rawStaticCast(); - auto mapped = reinterpret_cast(vm::map( - *address, size, prot, flags, vm::kMapInternalReserveOnly, blockPool)); - - if (mapped == MAP_FAILED) { - return orbis::ErrorCode::NOMEM; - } - - auto result = mapped; - - flags |= vm::kMapFlagFixed; - flags &= ~vm::kMapFlagNoOverwrite; - while (true) { - auto entry = *blockPool->pool.begin(); - auto blockSize = std::min(entry.endAddress - entry.beginAddress, size); - void *mapAddress = mapped; - ORBIS_LOG_FATAL("blockpool mmap", mapAddress, blockSize, entry.beginAddress, - blockSize); - ORBIS_RET_ON_ERROR( - dmem->mmap(&mapAddress, blockSize, prot, flags, entry.beginAddress)); - - mapped += blockSize; - size -= blockSize; - blockPool->pool.unmap(entry.beginAddress, entry.beginAddress + blockSize); - if (size == 0) { - break; - } - } - - *address = result; - return {}; -} - -static const orbis::FileOps ops = { - .ioctl = blockpool_ioctl, - .mmap = blockpool_mmap, + orbis::ErrorCode open(rx::Ref *file, const char *path, + std::uint32_t flags, std::uint32_t mode, + orbis::Thread *thread) override; + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override; }; +#pragma pack(push, 1) + +struct BlockPoolIoctlExpand { + orbis::uint64_t len; + orbis::uint64_t searchStart; + orbis::uint64_t searchEnd; + orbis::uint32_t flags; + orbis::uint32_t _padding; +}; +#pragma pack(pop) + +static orbis::ErrorCode blockpool_ioctl_expand(orbis::Thread *thread, + BlockPoolDevice *device, + BlockPoolIoctlExpand &args) { + ORBIS_LOG_TODO(__FUNCTION__, args.len, args.searchStart, args.searchEnd, + args.flags); + + if (args.len % orbis::dmem::kPageSize || args.len == 0) { + return orbis::ErrorCode::INVAL; + } + + auto alignment = args.flags == 0 ? 0 : 1ull << ((args.flags >> 24) & 0x1f); + + auto [dmemOffset, dmemErrc] = orbis::dmem::allocate( + 0, rx::AddressRange::fromBeginEnd(args.searchStart, args.searchEnd), + args.len, orbis::MemoryType::WbOnion, alignment, true); + + if (dmemErrc != orbis::ErrorCode{}) { + return dmemErrc; + } + + auto [pmemOffset, pmemErrc] = orbis::dmem::getPmemOffset(0, dmemOffset); + + if (pmemErrc != orbis::ErrorCode{}) { + return pmemErrc; + } + + args.searchStart = dmemOffset; + return orbis::blockpool::expand( + rx::AddressRange::fromBeginSize(pmemOffset, args.len)); +} + +static std::pair +blockpool_ioctl_get_block_stats(orbis::Thread *, BlockPoolDevice *) { + return {{}, orbis::blockpool::stats()}; +} + +static const orbis::FileOps ops = {}; + +BlockPoolDevice::BlockPoolDevice() { + blockFlags = orbis::vmem::BlockFlags::PooledMemory; + + addIoctl(blockpool_ioctl_expand); + addIoctl(blockpool_ioctl_get_block_stats); +} + orbis::ErrorCode BlockPoolDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, @@ -113,32 +120,17 @@ orbis::ErrorCode BlockPoolDevice::open(rx::Ref *file, return {}; } -orbis::ErrorCode BlockPoolDevice::map(void **address, std::uint64_t len, - std::int32_t prot, std::int32_t flags, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("blockpool device map", *address, len); - if (prot == 0) { - // FIXME: investigate it - prot = 0x33; +orbis::ErrorCode +BlockPoolDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) { + if (protection || offset != 0) { + return orbis::ErrorCode::INVAL; } - auto result = vm::map(*address, len, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::NOMEM; - } - - *address = result; return {}; } -orbis::ErrorCode BlockPoolDevice::unmap(void *address, std::uint64_t len, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("blockpool device unmap", address, len); - if (vm::unmap(address, len)) { - return {}; - } - return orbis::ErrorCode::INVAL; -} + orbis::IoDevice *createBlockPoolDevice() { return orbis::knew(); } diff --git a/rpcsx/iodev/blockpool.hpp b/rpcsx/iodev/blockpool.hpp deleted file mode 100644 index ec4ab6256..000000000 --- a/rpcsx/iodev/blockpool.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "orbis/IoDevice.hpp" -#include "orbis/error/ErrorCode.hpp" -#include "orbis/file.hpp" -#include "rx/MemoryTable.hpp" -#include "rx/Rc.hpp" -#include "rx/SharedMutex.hpp" -#include - -struct BlockPoolDevice : public orbis::IoDevice { - rx::shared_mutex mtx; - rx::MemoryAreaTable<> pool; - - orbis::ErrorCode open(rx::Ref *file, const char *path, - std::uint32_t flags, std::uint32_t mode, - orbis::Thread *thread) override; - orbis::ErrorCode map(void **address, std::uint64_t len, std::int32_t prot, - std::int32_t flags, orbis::Thread *thread); - orbis::ErrorCode unmap(void *address, std::uint64_t len, - orbis::Thread *thread); -}; diff --git a/rpcsx/iodev/dce.cpp b/rpcsx/iodev/dce.cpp index dcafe6b9e..da2f1c01d 100644 --- a/rpcsx/iodev/dce.cpp +++ b/rpcsx/iodev/dce.cpp @@ -1,18 +1,22 @@ #include "dce.hpp" #include "gpu/DeviceCtl.hpp" -#include "io-device.hpp" -#include "iodev/dmem.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/dmem.hpp" #include "orbis/error/ErrorCode.hpp" #include "orbis/file.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/mem.hpp" +#include "rx/print.hpp" #include "rx/watchdog.hpp" -#include "vm.hpp" #include #include #include @@ -144,7 +148,7 @@ struct ResolutionStatus { // refreshRate = 0x23, result.refreshHz = 0x42b3d1ec( 89.91) REFRESH_RATE_89_91HZ // clang-format on -static void runBridge(int vmId) { +static void runBridge(int vmId, orbis::Process *process) { std::thread{[=] { pthread_setname_np(pthread_self(), "Bridge"); @@ -186,29 +190,23 @@ static void runBridge(int vmId) { gpuCtx.cachePages[vmId][page].load(std::memory_order::relaxed); auto address = static_cast(page) * rx::mem::pageSize; - auto origVmProt = vm::getPageProtection(address); - int prot = 0; + auto range = + rx::AddressRange::fromBeginSize(address, rx::mem::pageSize * count); + auto origVmProt = orbis::vmem::queryProtection(process, address); + if (origVmProt && origVmProt->prot) { + auto prot = orbis::vmem::toCpuProtection(origVmProt->prot); - if (origVmProt & vm::kMapProtCpuRead) { - prot |= PROT_READ; - } - if (origVmProt & vm::kMapProtCpuWrite) { - prot |= PROT_WRITE; - } - if (origVmProt & vm::kMapProtCpuExec) { - prot |= PROT_EXEC; - } + if (pageFlags & amdgpu::kPageReadWriteLock) { + prot = prot & ~(rx::mem::Protection::R | rx::mem::Protection::W); + } else if (pageFlags & amdgpu::kPageWriteWatch) { + prot = prot & ~(rx::mem::Protection::W); + } - if (pageFlags & amdgpu::kPageReadWriteLock) { - prot &= ~(PROT_READ | PROT_WRITE); - } else if (pageFlags & amdgpu::kPageWriteWatch) { - prot &= ~PROT_WRITE; - } - - if (::mprotect(reinterpret_cast(address), - rx::mem::pageSize * count, prot)) { - perror("protection failed"); - std::abort(); + if (auto errc = rx::mem::protect(range, prot); errc != std::errc{}) { + rx::die("gpu cache: failed to protect memory {:x}-{:x}, error {}", + range.beginAddress(), range.endAddress(), + static_cast(errc)); + } } } @@ -237,43 +235,6 @@ int DceDevice::allocateVmId() { void DceDevice::deallocateVmId(int vmId) { freeVmIds |= (1 << vmId); } -static void initDceMemory(DceDevice *device) { - if (device->dmemOffset + 1) { - return; - } - - std::lock_guard lock(device->mtx); - if (device->dmemOffset + 1) { - return; - } - - auto dmem = orbis::g_context->dmemDevice.cast(); - std::uint64_t start = 0; - if (dmem->allocate(&start, ~0ull, kDceControlMemorySize, 0x100000, 1) != - orbis::ErrorCode{}) { - std::abort(); - } - - void *address = nullptr; - if (dmem->mmap(&address, kDceControlMemorySize, vm::kMapProtCpuWrite, 0, - start) != orbis::ErrorCode{}) { - std::abort(); - } - - auto dceControl = reinterpret_cast(address); - *reinterpret_cast(dceControl + 0x130) = 0; - *reinterpret_cast(dceControl + 0x138) = 1; - *reinterpret_cast(dceControl + 0x140) = - orbis::kEvFiltDisplay; - vm::unmap(address, kDceControlMemorySize); - device->dmemOffset = start; -} - -static orbis::ErrorCode dce_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread); - static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, void *argp, orbis::Thread *thread) { auto device = static_cast(file->device.get()); @@ -312,21 +273,30 @@ static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, } if (args->id == 9) { - ORBIS_LOG_NOTICE("dce: FlipControl allocate", args->id, args->padding, + ORBIS_LOG_NOTICE("dce: FlipControl map", args->id, args->padding, args->arg2, args->ptr, args->size, args->arg5, args->arg6); - void *address; - ORBIS_RET_ON_ERROR( - dce_mmap(file, &address, vm::kPageSize, - vm::kMapProtCpuReadWrite | vm::kMapProtGpuAll, - vm::kMapFlagShared, 0, thread)); + auto [range, errc] = orbis::vmem::mapDirect( + thread->tproc, 0, + rx::AddressRange::fromBeginSize(device->dmemRange.beginAddress(), + orbis::vmem::kPageSize), + orbis::vmem::Protection::CpuRead | + orbis::vmem::Protection::CpuWrite | + orbis::vmem::Protection::GpuRead | + orbis::vmem::Protection::GpuWrite, + {}, "DCE"); - *(void **)args->ptr = address; - *(std::uint64_t *)args->arg5 = vm::kPageSize; + if (errc != orbis::ErrorCode{}) { + return errc; + } + + *(std::uint64_t *)args->ptr = range.beginAddress(); + *(std::uint64_t *)args->arg5 = range.size(); return {}; } + if (args->id == 0x38) { auto attrs = (RegisterBufferAttributeArgs *)args->ptr; @@ -527,11 +497,11 @@ static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, if (request == 0xc0308207) { // SCE_SYS_DCE_IOCTL_REGISTER_BUFFER_ATTRIBUTE auto args = reinterpret_cast(argp); - ORBIS_LOG_ERROR("dce: RegisterBufferAttributes", args->canary, args->attrid, - args->submit, args->unk3, args->pixelFormat, - args->tilingMode, args->pitch, args->width, args->height, - args->unk4_zero, args->unk5_zero, args->options, - args->reserved1, args->reserved2); + ORBIS_LOG_ERROR( + "dce: RegisterBufferAttributes", args->canary, (int)args->attrid, + (int)args->submit, args->unk3, args->pixelFormat, args->tilingMode, + args->pitch, args->width, args->height, (int)args->unk4_zero, + (int)args->unk5_zero, args->options, args->reserved1, args->reserved2); gpu.registerBufferAttribute(thread->tproc->pid, { @@ -596,20 +566,30 @@ static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode dce_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("dce mmap", address, size, offset); - auto dce = file->device.cast(); - initDceMemory(dce.get()); - auto dmem = orbis::g_context->dmemDevice.cast(); - return dmem->mmap(address, size, prot, flags, dce->dmemOffset + offset); +orbis::ErrorCode +DceDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *, orbis::Process *process) { + if (offset + range.size() > dmemRange.size()) { + return orbis::ErrorCode::INVAL; + } + + rx::println(stderr, "map dce {:x}-{:x} {:04x} {}", range.beginAddress(), + range.endAddress(), offset, protection); + + auto result = + orbis::dmem::map(0, range, dmemRange.beginAddress() + offset, protection); + + if (result == orbis::ErrorCode{}) { + amdgpu::mapMemory(process->pid, range, orbis::MemoryType::WbGarlic, + protection, dmemRange.beginAddress() + offset); + } + + return result; } static const orbis::FileOps ops = { .ioctl = dce_ioctl, - .mmap = dce_mmap, }; static void createGpu() { @@ -626,6 +606,8 @@ static void createGpu() { } } +DceDevice::~DceDevice() { orbis::dmem::release(0, dmemRange); } + orbis::ErrorCode DceDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) { @@ -649,8 +631,32 @@ void DceDevice::initializeProcess(orbis::Process *process) { process->vmId = vmId; } - runBridge(vmId); + runBridge(vmId, process); } } -orbis::IoDevice *createDceCharacterDevice() { return orbis::knew(); } +orbis::IoDevice *createDceCharacterDevice(orbis::Process *process) { + auto result = orbis::knew(); + auto dmemSize = orbis::dmem::getSize(0); + auto [dmemOffset, errc] = orbis::dmem::allocate( + 0, + rx::AddressRange::fromBeginEnd(dmemSize - orbis::dmem::kPageSize * 2, + dmemSize), + orbis::dmem::kPageSize, orbis::MemoryType::WbGarlic); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to allocate DCE memory, error {}", errc); + result->dmemRange = + rx::AddressRange::fromBeginSize(dmemOffset, orbis::dmem::kPageSize); + + auto [vmem, mapErrc] = orbis::vmem::mapDirect( + process, 0, result->dmemRange, orbis::vmem::Protection::CpuWrite, {}); + auto dceControl = reinterpret_cast(vmem.beginAddress()); + *reinterpret_cast(dceControl + 0x130) = 0; + *reinterpret_cast(dceControl + 0x138) = 1; + *reinterpret_cast(dceControl + 0x140) = + orbis::kEvFiltDisplay; + + orbis::vmem::unmap(process, vmem); + return result; +} diff --git a/rpcsx/iodev/dce.hpp b/rpcsx/iodev/dce.hpp index 38dd03741..1af40a03a 100644 --- a/rpcsx/iodev/dce.hpp +++ b/rpcsx/iodev/dce.hpp @@ -1,7 +1,7 @@ #pragma once -#include "io-device.hpp" #include "orbis-config.hpp" +#include "orbis/IoDevice.hpp" #include "orbis/error/ErrorCode.hpp" #include "orbis/file.hpp" #include "orbis/thread/Process.hpp" @@ -12,14 +12,19 @@ static constexpr auto kVmIdCount = 6; struct DceDevice : orbis::IoDevice { rx::shared_mutex mtx; + rx::AddressRange dmemRange; std::uint32_t eopCount = 0; std::uint32_t freeVmIds = (1 << (kVmIdCount + 1)) - 1; - orbis::uint64_t dmemOffset = ~static_cast(0); + + DceDevice() { blockFlags = orbis::vmem::BlockFlags::DirectMemory; } + ~DceDevice(); orbis::ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override; - + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override; int allocateVmId(); void deallocateVmId(int vmId); void initializeProcess(orbis::Process *process); diff --git a/rpcsx/iodev/dmem.cpp b/rpcsx/iodev/dmem.cpp index 1a3120c0a..d5eb755a5 100644 --- a/rpcsx/iodev/dmem.cpp +++ b/rpcsx/iodev/dmem.cpp @@ -1,390 +1,353 @@ -#include "dmem.hpp" +#include "orbis/dmem.hpp" #include "gpu/DeviceCtl.hpp" -#include "io-device.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/error.hpp" #include "orbis/file.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" -#include "rx/align.hpp" -#include "rx/format.hpp" -#include "rx/watchdog.hpp" -#include "vm.hpp" -#include -#include -#include -#include -#include +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" + +enum { + DMEM_IOCTL_ALLOCATE = 0xc0288001, + DMEM_IOCTL_RELEASE = 0x80108002, + DMEM_IOCTL_SET_TYPE = 0x80188003, + DMEM_IOCTL_GET_TYPE = 0xc0208004, + DMEM_IOCTL_GET_TOTAL_SIZE = 0x4008800a, + DMEM_IOCTL_CLEAR = 0x2000800b, + DMEM_IOCTL_TRANSFER_BUDGET = 0xc018800d, + DMEM_IOCTL_CONTROL_RELEASE = 0xc018800e, + DMEM_IOCTL_SET_PID_AND_PROTECT = 0xc018800f, + DMEM_IOCTL_ALLOCATE_FOR_MINI_APP = 0xc0288010, + DMEM_IOCTL_ALLOCATE_MAIN = 0xc0288011, + DMEM_IOCTL_QUERY = 0x80288012, + DMEM_IOCTL_CHECKED_RELEASE = 0x80108015, + DMEM_IOCTL_GET_AVAIL_SIZE = 0xc0208016, + DMEM_IOCTL_RESERVE = 0xc010801a, +}; +struct DmemDevice + : orbis::IoDeviceWithIoctl { + int index; + DmemDevice(int index); + + orbis::ErrorCode open(rx::Ref *file, const char *path, + std::uint32_t flags, std::uint32_t mode, + orbis::Thread *thread) override; + + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override; +}; struct DmemFile : public orbis::File {}; -struct AllocateDirectMemoryArgs { - std::uint64_t searchStart; - std::uint64_t searchEnd; - std::uint64_t len; - std::uint64_t alignment; - std::uint32_t memoryType; +#pragma pack(push, 1) +struct DmemIoctlAllocate { + orbis::uintptr_t searchStart; + orbis::uintptr_t searchEnd; + orbis::size_t len; + orbis::size_t alignment; + orbis::MemoryType memoryType; + orbis::uint32_t padding; }; -static constexpr auto dmemSize = 0x5000000000; -// static const std::uint64_t nextOffset = 0; -// static const std::uint64_t memBeginAddress = 0xfe0000000; +struct DmemIoctlRelease { + orbis::uintptr_t address; + orbis::size_t size; +}; +struct DmemIoctlSetType { + orbis::uintptr_t start; + orbis::uintptr_t end; + orbis::MemoryType memoryType; + orbis::uint32_t padding; +}; +struct DmemIoctlGetType { + orbis::uintptr_t start; + orbis::uintptr_t regionStart; + orbis::uintptr_t regionEnd; + orbis::MemoryType memoryType; + orbis::uint32_t padding; +}; +struct DmemIoctlTransferBudget { + orbis::uint64_t unk0; + orbis::uint64_t unk1; + orbis::uint64_t unk2; +}; +struct DmemIoctlControlRelease { + orbis::uint64_t unk0; + orbis::uint64_t unk1; + orbis::uint64_t unk2; +}; +struct DmemIoctlSetPidAndProtect { + orbis::uintptr_t address; + orbis::size_t size; + orbis::pid_t pid; // 0 if all + rx::EnumBitSet prot; +}; -DmemDevice::~DmemDevice() { - if (shmFd > 0) { - close(shmFd); +struct DirectMemoryQueryInfo { + orbis::uintptr_t start; + orbis::uintptr_t end; + orbis::MemoryType memoryType; + orbis::uint32_t padding; +}; +struct DmemIoctlQuery { + orbis::uint32_t devIndex; + orbis::uint32_t flags; + orbis::uint32_t unk; + orbis::uint32_t _padding; + orbis::uint64_t offset; + orbis::ptr info; + orbis::uint64_t infoSize; +}; +struct DmemIoctlGetAvailSize { + orbis::uintptr_t searchStart; + orbis::uintptr_t searchEnd; + orbis::size_t alignment; + orbis::size_t size; +}; + +struct DmemIoctlReserve { + orbis::size_t size; + orbis::uint32_t flags; + orbis::uint32_t padding; +}; +#pragma pack(pop) + +static orbis::ErrorCode dmem_ioctl_allocate(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlAllocate &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.searchStart, args.searchEnd, + args.alignment, args.len, (int)args.memoryType); + auto [offset, errc] = orbis::dmem::allocate( + device->index, + rx::AddressRange::fromBeginEnd(args.searchStart, args.searchEnd), + args.len, args.memoryType); + + if (errc != orbis::ErrorCode{}) { + return errc; } - std::filesystem::remove(rx::format("{}/dmem-{}", rx::getShmPath(), index)); + args.searchStart = offset; + return {}; } -orbis::ErrorCode DmemDevice::mmap(void **address, std::uint64_t len, - std::int32_t prot, std::int32_t flags, - std::int64_t directMemoryStart) { - if (prot == 0) { - // hack - // fixme: implement protect for pid - prot = vm::kMapProtCpuReadWrite | vm::kMapProtGpuAll; +static orbis::ErrorCode dmem_ioctl_release(orbis::Thread *thread, + DmemDevice *device, + const DmemIoctlRelease &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.address, args.size); + + return orbis::dmem::release( + device->index, rx::AddressRange::fromBeginSize(args.address, args.size)); +} + +static orbis::ErrorCode dmem_ioctl_set_type(orbis::Thread *thread, + DmemDevice *device, + const DmemIoctlSetType &args) { + // removed ioctl + return orbis::ErrorCode::INVAL; +} + +static orbis::ErrorCode dmem_ioctl_get_type(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlGetType &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.start); + + auto result = orbis::dmem::query(device->index, args.start); + + if (!result) { + return orbis::ErrorCode::NOENT; } - if (*address == nullptr) { - *address = std::bit_cast(0x80000000ull); - flags &= ~vm::kMapFlagFixed; + args.regionStart = result->range.beginAddress(); + args.regionEnd = result->range.endAddress(); + args.memoryType = result->memoryType; + return {}; +} + +static std::pair +dmem_ioctl_get_total_size(orbis::Thread *thread, DmemDevice *device) { + auto result = orbis::dmem::getSize(device->index); + ORBIS_LOG_WARNING(__FUNCTION__, result); + + auto limit = thread->tproc->getBudget()->get(orbis::BudgetResource::Dmem); + return {{}, orbis::uint64_t(std::min(result, limit.total))}; +} + +static orbis::ErrorCode dmem_ioctl_clear(orbis::Thread *thread, + DmemDevice *device) { + ORBIS_LOG_WARNING(__FUNCTION__); + auto result = orbis::dmem::clear(device->index); + if (result == orbis::ErrorCode{}) { + thread->tproc->getBudget()->release(orbis::BudgetResource::Dmem, -1); + } + return result; +} + +static orbis::ErrorCode +dmem_ioctl_transfer_budget(orbis::Thread *thread, DmemDevice *device, + DmemIoctlTransferBudget &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.unk0, args.unk1, args.unk2); + return {}; +} + +static orbis::ErrorCode +dmem_ioctl_control_release(orbis::Thread *thread, DmemDevice *device, + DmemIoctlControlRelease &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.unk0, args.unk1, args.unk2); + return {}; +} + +static orbis::ErrorCode +dmem_ioctl_set_pid_and_protect(orbis::Thread *thread, DmemDevice *device, + DmemIoctlSetPidAndProtect &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.pid, args.address, args.size, + args.prot.toUnderlying()); + return {}; +} + +static orbis::ErrorCode +dmem_ioctl_allocate_for_mini_app(orbis::Thread *thread, DmemDevice *device, + DmemIoctlAllocate &args) { + // FIXME: implement + return dmem_ioctl_allocate(thread, device, args); +} + +static orbis::ErrorCode dmem_ioctl_allocate_main(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlAllocate &args) { + // FIXME: implement + return dmem_ioctl_allocate(thread, device, args); +} + +static orbis::ErrorCode dmem_ioctl_query(orbis::Thread *thread, + DmemDevice *device, + const DmemIoctlQuery &args) { + ORBIS_LOG_WARNING(__FUNCTION__, device->index, args.devIndex, args.unk, + args.flags, args.offset, args.info, args.infoSize); + + if (args.devIndex != device->index) { + ORBIS_LOG_WARNING(__FUNCTION__, "device mismatch", device->index, + args.devIndex, args.unk, args.flags, args.offset, + args.info, args.infoSize); } - int memoryType = 0; - if (auto allocationInfoIt = allocations.queryArea(directMemoryStart); - allocationInfoIt != allocations.end()) { - memoryType = allocationInfoIt->memoryType; - } - - auto result = vm::map(*address, len, prot, flags, vm::kMapInternalReserveOnly, - this, directMemoryStart); - - ORBIS_LOG_WARNING("dmem mmap", index, directMemoryStart, len, prot, flags, - result, *address); - if (result == (void *)-1) { - return orbis::ErrorCode::NOMEM; // TODO - } - - if (::mmap(result, len, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, shmFd, - directMemoryStart) == (void *)-1) { + if (args.infoSize != sizeof(DirectMemoryQueryInfo) || args.devIndex >= 3) { return orbis::ErrorCode::INVAL; } - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitMapMemory(orbis::g_currentThread->tproc->pid, - reinterpret_cast(result), len, - memoryType, index, prot, directMemoryStart); + rx::EnumBitSet queryFlags = {}; + + if (args.flags & 1) { + queryFlags |= orbis::dmem::QueryFlags::LowerBound; } - *address = result; + auto result = orbis::dmem::query(args.devIndex, args.offset, queryFlags); + if (!result) { + return orbis::ErrorCode::ACCES; + } + + DirectMemoryQueryInfo info{ + .start = result->range.beginAddress(), + .end = result->range.endAddress(), + .memoryType = result->memoryType, + }; + + ORBIS_LOG_WARNING(__FUNCTION__, device->index, args.devIndex, args.unk, + args.flags, args.offset, args.info, args.infoSize, + info.start, info.end, (int)info.memoryType); + + return orbis::uwrite(args.info, info); +} + +static orbis::ErrorCode +dmem_ioctl_checked_release(orbis::Thread *thread, DmemDevice *device, + const DmemIoctlRelease &args) { + return dmem_ioctl_release(thread, device, args); +} + +static orbis::ErrorCode dmem_ioctl_get_avail_size(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlGetAvailSize &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.searchStart, args.searchEnd, + args.alignment, args.size); + + auto [range, errc] = orbis::dmem::getAvailSize( + device->index, + rx::AddressRange::fromBeginEnd(args.searchStart, args.searchEnd), + args.alignment); + + if (errc != orbis::ErrorCode{}) { + return errc; + } + + ORBIS_LOG_WARNING(__FUNCTION__, args.searchStart, args.searchEnd, + args.alignment, args.size, range.beginAddress(), + range.size()); + + args.searchStart = range.beginAddress(); + args.size = range.size(); return {}; } -static orbis::ErrorCode dmem_ioctl(orbis::File *file, std::uint64_t request, - void *argp, orbis::Thread *thread) { - auto device = file->device.rawStaticCast(); +static orbis::ErrorCode dmem_ioctl_reserve(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlReserve &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.size, (int)args.flags); + auto [offset, errc] = orbis::dmem::reserveSystem(device->index, args.size); - std::lock_guard lock(device->mtx); - switch (request) { - case 0x4008800a: // get size - ORBIS_LOG_WARNING("dmem getTotalSize", device->index, argp); - *(std::uint64_t *)argp = device->dmemTotalSize / 0x10; - return {}; - - case 0xc0208016: { // get available size - struct Args { - std::uint64_t searchStart; - std::uint64_t searchEnd; - std::uint64_t alignment; - std::uint64_t size; - }; - - auto args = reinterpret_cast(argp); - - return device->queryMaxFreeChunkSize(&args->searchStart, args->searchEnd, - args->alignment, &args->size); + if (errc == orbis::ErrorCode{}) { + args.size = offset | 0x4000000000; } - - case 0xc0288010: // sceKernelAllocateDirectMemoryForMiniApp - case 0xc0288011: - case 0xc0288001: { // sceKernelAllocateDirectMemory - auto args = reinterpret_cast(argp); - - return device->allocate(&args->searchStart, args->searchEnd, args->len, - args->alignment, args->memoryType); - } - - case 0xc018800d: { // transfer budget - return {}; - } - - case 0xc018800f: { // protect memory for pid - struct Args { - std::uint64_t address; - std::uint64_t size; - std::uint32_t pid; // 0 if all - std::uint32_t prot; - }; - return {}; - } - - case 0x80108015: // sceKernelCheckedReleaseDirectMemory - case 0x80108002: { // sceKernelReleaseDirectMemory - struct Args { - std::uint64_t address; - std::uint64_t size; - }; - - auto args = reinterpret_cast(argp); - - ORBIS_LOG_WARNING("dmem releaseDirectMemory", device->index, args->address, - args->size); - - device->allocations.map( - rx::AddressRange::fromBeginSize(args->address, args->size), - {.memoryType = -1u}); - return {}; - } - - case 0xc0208004: { // get direct memory type - struct Args { - std::uint64_t start; - std::uint64_t regionStart; - std::uint64_t regionEnd; - std::uint32_t memoryType; - }; - - auto args = reinterpret_cast(argp); - - auto it = device->allocations.lowerBound(args->start); - - if (it == device->allocations.end() || it->memoryType == -1u) { - return orbis::ErrorCode::SRCH; - } - - args->regionStart = it.beginAddress(); - args->regionEnd = it.endAddress(); - args->memoryType = it->memoryType; - return {}; - } - - case 0x80288012: { // direct memory query - struct DirectMemoryQueryInfo { - std::uint64_t start; - std::uint64_t end; - std::uint32_t memoryType; - }; - - struct Args { - std::uint32_t devIndex; - std::uint32_t flags; - std::uint32_t unk; - std::uint64_t offset; - orbis::ptr info; - std::uint64_t infoSize; - }; - - auto args = reinterpret_cast(argp); - - ORBIS_LOG_WARNING("dmem directMemoryQuery", device->index, args->devIndex, - args->unk, args->flags, args->offset, args->info, - args->infoSize); - - if (args->devIndex != device->index) { - // TODO - ORBIS_LOG_ERROR("dmem directMemoryQuery: device mismatch", device->index, - args->devIndex, args->unk, args->flags, args->offset, - args->info, args->infoSize); - - return orbis::ErrorCode::INVAL; - } - - if (args->infoSize != sizeof(DirectMemoryQueryInfo)) { - return orbis::ErrorCode::INVAL; - } - - auto it = device->allocations.lowerBound(args->offset); - - if (it == device->allocations.end()) { - return orbis::ErrorCode::ACCES; - } - - auto queryInfo = *it; - - if (it->memoryType == -1u) { - return orbis::ErrorCode::ACCES; - } - - if ((args->flags & 1) == 0) { - if (it.endAddress() <= args->offset) { - return orbis::ErrorCode::ACCES; - } - } else { - if (it.beginAddress() > args->offset || it.endAddress() <= args->offset) { - return orbis::ErrorCode::ACCES; - } - } - - DirectMemoryQueryInfo info{ - .start = it.beginAddress(), - .end = it.endAddress(), - .memoryType = it->memoryType, - }; - - ORBIS_LOG_WARNING("dmem directMemoryQuery", device->index, args->devIndex, - args->unk, args->flags, args->offset, args->info, - args->infoSize, info.start, info.end, info.memoryType); - return orbis::uwrite(args->info, info); - } - } - - ORBIS_LOG_FATAL("Unhandled dmem ioctl", device->index, request); - thread->where(); - return {}; + return errc; } -static orbis::ErrorCode dmem_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - auto device = file->device.rawStaticCast(); - return device->mmap(address, size, prot, flags, offset); +DmemDevice::DmemDevice(int index) : index(index) { + blockFlags = orbis::vmem::BlockFlags::DirectMemory; + + addIoctl(dmem_ioctl_allocate); + addIoctl(dmem_ioctl_release); + addIoctl(dmem_ioctl_set_type); + addIoctl(dmem_ioctl_get_type); + addIoctl(dmem_ioctl_get_total_size); + addIoctl(dmem_ioctl_clear); + addIoctl(dmem_ioctl_transfer_budget); + addIoctl(dmem_ioctl_control_release); + addIoctl(dmem_ioctl_set_pid_and_protect); + addIoctl(dmem_ioctl_allocate_for_mini_app); + addIoctl(dmem_ioctl_allocate_main); + addIoctl(dmem_ioctl_query); + addIoctl(dmem_ioctl_checked_release); + addIoctl(dmem_ioctl_get_avail_size); + addIoctl(dmem_ioctl_reserve); } -static const orbis::FileOps ops = { - .ioctl = dmem_ioctl, - .mmap = dmem_mmap, -}; +orbis::ErrorCode +DmemDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *, orbis::Process *process) { + auto result = orbis::dmem::map(index, range, offset, protection); -orbis::ErrorCode DmemDevice::allocate(std::uint64_t *start, - std::uint64_t searchEnd, - std::uint64_t len, - std::uint64_t alignment, - std::uint32_t memoryType) { - std::size_t offset = *start; - if (alignment == 0) { - alignment = 1; - } - if (searchEnd == 0) { - searchEnd = dmemTotalSize; - } + if (result == orbis::ErrorCode{}) { + if (auto dmemType = orbis::dmem::query(0, offset)) { + auto [pmemOffset, errc] = orbis::dmem::getPmemOffset(0, offset); + rx::dieIf(errc != orbis::ErrorCode{}, "failed to query dmem type {}", + errc); - while (offset < searchEnd) { - offset += alignment - 1; - offset &= ~(alignment - 1); - - if (offset + len > dmemTotalSize) { - ORBIS_LOG_ERROR("dmem: failed to allocate direct memory: out of memory", - *start, searchEnd, len, alignment, memoryType, offset); - return orbis::ErrorCode::AGAIN; + amdgpu::mapMemory(process->pid, range, dmemType->memoryType, protection, + pmemOffset); } - - auto it = allocations.lowerBound(offset); - - if (it != allocations.end()) { - if (it->memoryType == -1u) { - if (offset < it.beginAddress()) { - offset = it.beginAddress() + alignment - 1; - offset &= ~(alignment - 1); - } - - if (offset + len >= it.endAddress()) { - offset = it.endAddress(); - continue; - } - } else { - if (offset + len > it.beginAddress()) { - offset = it.endAddress(); - continue; - } - } - } - - allocations.map(rx::AddressRange::fromBeginSize(offset, len), - { - .memoryType = memoryType, - }); - ORBIS_LOG_WARNING("dmem: allocated direct memory", *start, searchEnd, len, - alignment, memoryType, offset); - *start = offset; - return {}; } - ORBIS_LOG_ERROR("dmem: failed to allocate direct memory", *start, searchEnd, - len, alignment, memoryType, offset); - return orbis::ErrorCode::AGAIN; + return result; } -orbis::ErrorCode DmemDevice::release(std::uint64_t start, std::uint64_t size) { - allocations.unmap(rx::AddressRange::fromBeginSize(start, size)); - return {}; -} - -orbis::ErrorCode DmemDevice::queryMaxFreeChunkSize(std::uint64_t *start, - std::uint64_t searchEnd, - std::uint64_t alignment, - std::uint64_t *size) { - std::size_t offset = *start; - std::size_t resultSize = 0; - std::size_t resultOffset = 0; - - alignment = std::max(alignment, vm::kPageSize); - alignment = rx::alignUp(alignment, vm::kPageSize); - - while (offset < searchEnd) { - offset += alignment - 1; - offset &= ~(alignment - 1); - - if (offset >= dmemTotalSize) { - break; - } - - auto it = allocations.lowerBound(offset); - - if (it == allocations.end()) { - if (resultSize < dmemTotalSize - offset) { - resultSize = dmemTotalSize - offset; - resultOffset = offset; - } - - break; - } - - if (it->memoryType == -1u) { - if (offset < it.beginAddress()) { - offset = it.beginAddress() + alignment - 1; - offset &= ~(alignment - 1); - } - - if (it.endAddress() > offset && resultSize < it.endAddress() - offset) { - resultSize = it.endAddress() - offset; - resultOffset = offset; - } - } else if (offset > it.beginAddress() && - resultSize < offset - it.beginAddress()) { - resultSize = offset - it.beginAddress(); - resultOffset = offset; - } - - offset = it.endAddress(); - } - - resultSize /= 0x20; - - *start = resultOffset; - *size = resultSize; - - ORBIS_LOG_WARNING("dmem queryMaxFreeChunkSize", resultOffset, resultSize); - - if (resultSize == 0) { - return orbis::ErrorCode::NOMEM; - } - return {}; -} +static const orbis::FileOps ops = {}; orbis::ErrorCode DmemDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, @@ -397,18 +360,6 @@ orbis::ErrorCode DmemDevice::open(rx::Ref *file, const char *path, } orbis::IoDevice *createDmemCharacterDevice(int index) { - auto *newDevice = orbis::knew(); - newDevice->index = index; - newDevice->dmemTotalSize = dmemSize; - - auto path = rx::format("{}/dmem-{}", rx::getShmPath(), index); - auto shmFd = ::open(path.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); - - if (ftruncate(shmFd, dmemSize) < 0) { - ::close(shmFd); - std::abort(); - } - - newDevice->shmFd = shmFd; + auto *newDevice = orbis::knew(index); return newDevice; } diff --git a/rpcsx/iodev/dmem.hpp b/rpcsx/iodev/dmem.hpp deleted file mode 100644 index 34bc997f0..000000000 --- a/rpcsx/iodev/dmem.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include "orbis/IoDevice.hpp" -#include "orbis/KernelAllocator.hpp" -#include "orbis/error/ErrorCode.hpp" -#include "orbis/file.hpp" -#include "rx/Rc.hpp" -#include "rx/SharedMutex.hpp" -#include -#include -#include - -struct DmemDevice : public orbis::IoDevice { - rx::shared_mutex mtx; - int index; - int shmFd = -1; - std::size_t dmemTotalSize; - - ~DmemDevice(); - - struct AllocationInfo { - std::uint32_t memoryType; - - bool operator==(const AllocationInfo &) const = default; - auto operator<=>(const AllocationInfo &) const = default; - }; - - rx::MemoryTableWithPayload allocations; - - orbis::ErrorCode allocate(std::uint64_t *start, std::uint64_t searchEnd, - std::uint64_t len, std::uint64_t alignment, - std::uint32_t memoryType); - - orbis::ErrorCode queryMaxFreeChunkSize(std::uint64_t *start, - std::uint64_t searchEnd, - std::uint64_t alignment, - std::uint64_t *size); - - orbis::ErrorCode release(std::uint64_t start, std::uint64_t size); - - orbis::ErrorCode open(rx::Ref *file, const char *path, - std::uint32_t flags, std::uint32_t mode, - orbis::Thread *thread) override; - - orbis::ErrorCode mmap(void **address, std::uint64_t len, std::int32_t prot, - std::int32_t flags, std::int64_t directMemoryStart); -}; diff --git a/rpcsx/iodev/gc.cpp b/rpcsx/iodev/gc.cpp index 6647f25e5..b3caf1b49 100644 --- a/rpcsx/iodev/gc.cpp +++ b/rpcsx/iodev/gc.cpp @@ -1,17 +1,20 @@ +#include "dce.hpp" #include "gpu/DeviceCtl.hpp" -#include "iodev/dce.hpp" -#include "iodev/dmem.hpp" #include "orbis/IoDevice.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/dmem.hpp" #include "orbis/file.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" #include "rx/SharedMutex.hpp" #include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/print.hpp" -#include "vm.hpp" #include #include #include @@ -26,15 +29,44 @@ struct ComputeQueue { struct GcDevice : public orbis::IoDevice { rx::shared_mutex mtx; + rx::AddressRange dmemRange; orbis::kmap clients; orbis::kmap computeQueues; - void *submitArea = nullptr; + orbis::uintptr_t submitArea = 0; orbis::ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override; void addClient(orbis::Process *process); void removeClient(orbis::Process *process); + + GcDevice() { blockFlags = orbis::vmem::BlockFlags::DirectMemory; } + + ~GcDevice() { orbis::pmem::deallocate(dmemRange); } + + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override { + if (offset + range.size() > dmemRange.size()) { + return orbis::ErrorCode::INVAL; + } + + rx::println(stderr, "map gc {:x}-{:x} {:04x} {}", range.beginAddress(), + range.endAddress(), offset, protection); + + auto result = + orbis::pmem::map(range.beginAddress(), + rx::AddressRange::fromBeginSize( + dmemRange.beginAddress() + offset, range.size()), + orbis::vmem::toCpuProtection(protection)); + + if (result == orbis::ErrorCode{}) { + amdgpu::mapMemory(process->pid, range, orbis::MemoryType::WbOnion, + protection, dmemRange.beginAddress() + offset); + } + + return result; + } }; struct GcFile : public orbis::File { @@ -55,26 +87,35 @@ static orbis::ErrorCode gc_ioctl(orbis::File *file, std::uint64_t request, switch (request) { case 0xc008811b: // get submit done flag ptr? - if (device->submitArea == nullptr) { - auto dmem = orbis::g_context->dmemDevice.staticCast(); - std::uint64_t start = 0; - auto err = dmem->allocate(&start, ~0, vm::kPageSize, 0, 0); - if (err != orbis::ErrorCode{}) { - return err; + if (device->submitArea == 0) { + auto [dmemOffset, dmemErrc] = orbis::dmem::allocate( + 0, rx::AddressRange::fromBeginEnd(0, 0), orbis::dmem::kPageSize, + orbis::MemoryType::WbGarlic); + + if (dmemErrc != orbis::ErrorCode{}) { + return dmemErrc; } - auto address = reinterpret_cast(0xfe0100000); - err = dmem->mmap(&address, vm::kPageSize, - vm::kMapProtCpuReadWrite | vm::kMapProtGpuAll, - vm::kMapFlagShared, start); - if (err != orbis::ErrorCode{}) { - dmem->release(start, vm::kPageSize); - return err; + + auto directRange = + rx::AddressRange::fromBeginSize(dmemOffset, orbis::dmem::kPageSize); + + auto [vmemRange, vmemErrc] = orbis::vmem::mapDirect( + thread->tproc, 0xfe0100000, directRange, + orbis::vmem::Protection::CpuRead | orbis::vmem::Protection::CpuWrite | + orbis::vmem::Protection::GpuRead | + orbis::vmem::Protection::GpuWrite, + {}); + + if (vmemErrc != orbis::ErrorCode{}) { + orbis::dmem::release(0, directRange); + return dmemErrc; } - device->submitArea = address; + + device->submitArea = vmemRange.beginAddress(); } ORBIS_LOG_ERROR("gc ioctl 0xc008811b", *(std::uint64_t *)argp); - *reinterpret_cast(argp) = device->submitArea; + *reinterpret_cast(argp) = device->submitArea; break; case 0xc004812e: { @@ -438,25 +479,7 @@ static orbis::ErrorCode gc_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode gc_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("gc mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - -static const orbis::FileOps ops = { - .ioctl = gc_ioctl, - .mmap = gc_mmap, -}; +static const orbis::FileOps ops = {.ioctl = gc_ioctl}; orbis::ErrorCode GcDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, @@ -490,4 +513,18 @@ void GcDevice::removeClient(orbis::Process *process) { } } -orbis::IoDevice *createGcCharacterDevice() { return orbis::knew(); } +orbis::IoDevice *createGcCharacterDevice() { + auto result = orbis::knew(); + auto dmemSize = orbis::dmem::getSize(0); + auto [dmemOffset, errc] = orbis::dmem::allocate( + 0, + rx::AddressRange::fromBeginEnd(dmemSize - orbis::dmem::kPageSize * 2, + dmemSize), + orbis::dmem::kPageSize, orbis::MemoryType::WbGarlic); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to allocate GC memory, error {}", errc); + result->dmemRange = + rx::AddressRange::fromBeginSize(dmemOffset, orbis::dmem::kPageSize); + return result; +} diff --git a/rpcsx/iodev/hmd_mmap.cpp b/rpcsx/iodev/hmd_mmap.cpp index dc9909e95..c9d6e0157 100644 --- a/rpcsx/iodev/hmd_mmap.cpp +++ b/rpcsx/iodev/hmd_mmap.cpp @@ -2,7 +2,6 @@ #include "orbis/KernelAllocator.hpp" #include "orbis/file.hpp" #include "orbis/utils/Logs.hpp" -#include "vm.hpp" struct HmdMmapDevice : public orbis::IoDevice { orbis::ErrorCode open(rx::Ref *file, const char *path, @@ -19,24 +18,8 @@ static orbis::ErrorCode hmd_mmap_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode hmd_mmap_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("hmd_mmap mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - static const orbis::FileOps ops = { .ioctl = hmd_mmap_ioctl, - .mmap = hmd_mmap_mmap, }; orbis::ErrorCode HmdMmapDevice::open(rx::Ref *file, diff --git a/rpcsx/iodev/rng.cpp b/rpcsx/iodev/rng.cpp index e089f8a7b..8f3041440 100644 --- a/rpcsx/iodev/rng.cpp +++ b/rpcsx/iodev/rng.cpp @@ -2,7 +2,6 @@ #include "orbis/KernelAllocator.hpp" #include "orbis/file.hpp" #include "orbis/utils/Logs.hpp" -#include "vm.hpp" struct RngDevice : public orbis::IoDevice { orbis::ErrorCode open(rx::Ref *file, const char *path, @@ -19,24 +18,8 @@ static orbis::ErrorCode rng_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode rng_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("rng mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - static const orbis::FileOps ops = { - .ioctl = rng_ioctl, - .mmap = rng_mmap, + .ioctl = rng_ioctl }; orbis::ErrorCode RngDevice::open(rx::Ref *file, const char *path, diff --git a/rpcsx/iodev/sbl_srv.cpp b/rpcsx/iodev/sbl_srv.cpp index e08269a57..cb6b565aa 100644 --- a/rpcsx/iodev/sbl_srv.cpp +++ b/rpcsx/iodev/sbl_srv.cpp @@ -5,7 +5,6 @@ #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" #include "rx/SharedMutex.hpp" -#include "vm.hpp" struct SblSrvFile : public orbis::File {}; @@ -23,24 +22,8 @@ static orbis::ErrorCode sbl_srv_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode sbl_srv_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("sbl_srv mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - static const orbis::FileOps ops = { .ioctl = sbl_srv_ioctl, - .mmap = sbl_srv_mmap, }; orbis::ErrorCode SblSrvDevice::open(rx::Ref *file, diff --git a/rpcsx/ipmi.cpp b/rpcsx/ipmi.cpp index 0f981df68..3b109e14f 100644 --- a/rpcsx/ipmi.cpp +++ b/rpcsx/ipmi.cpp @@ -4,15 +4,20 @@ #include "io-device.hpp" #include "orbis/KernelContext.hpp" #include "orbis/osem.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" +#include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/align.hpp" +#include "rx/die.hpp" #include "rx/format.hpp" #include "rx/hexdump.hpp" -#include "rx/mem.hpp" #include "rx/print.hpp" #include "rx/watchdog.hpp" #include "vfs.hpp" -#include "vm.hpp" +#include #include #include #include @@ -21,45 +26,65 @@ ipmi::IpmiClient ipmi::audioIpmiClient; +orbis::Process *g_workerProcess; + template struct GuestAlloc { orbis::ptr guestAddress; + std::size_t size; - GuestAlloc(std::size_t size) { + GuestAlloc(std::size_t size) : size(size) { if (size == 0) { guestAddress = nullptr; } else { - guestAddress = orbis::ptr( - vm::map(nullptr, size, vm::kMapProtCpuRead | vm::kMapProtCpuWrite, - vm::kMapFlagPrivate | vm::kMapFlagAnonymous)); + size = rx::alignUp(size, orbis::vmem::kPageSize); + + auto [range, vmemErrc] = orbis::vmem::mapFlex( + g_workerProcess ? g_workerProcess : orbis::g_currentThread->tproc, + size, + orbis::vmem::Protection::CpuRead | orbis::vmem::Protection::CpuWrite); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "impi: failed to map memory, error {}", + static_cast(vmemErrc)); + + guestAddress = std::bit_cast>(range.beginAddress()); } } GuestAlloc() : GuestAlloc(sizeof(T)) {} GuestAlloc(const T &data) : GuestAlloc() { - if (orbis::uwrite(guestAddress, data) != orbis::ErrorCode{}) { - std::abort(); + if (auto errc = orbis::uwrite(guestAddress, data); + errc != orbis::ErrorCode{}) { + rx::die("ipmi: failed to write data to allocated page {}, error {}", + (void *)guestAddress, errc); } } GuestAlloc(const void *data, std::size_t size) : GuestAlloc(size) { - if (orbis::uwriteRaw(guestAddress, data, size) != orbis::ErrorCode{}) { - std::abort(); + if (auto errc = orbis::uwriteRaw(guestAddress, data, size); + errc != orbis::ErrorCode{}) { + rx::die("ipmi: failed to write data to allocated page {}, error {}, data " + "{}, size {}", + (void *)guestAddress, errc, data, size); } } GuestAlloc(const GuestAlloc &) = delete; GuestAlloc(GuestAlloc &&other) noexcept : guestAddress(other.guestAddress) { - other.guestAddress = 0; + other.guestAddress = nullptr; } GuestAlloc &operator=(GuestAlloc &&other) noexcept { std::swap(guestAddress, other.guestAddress); } ~GuestAlloc() { - if (guestAddress != 0) { - vm::unmap(guestAddress, sizeof(T)); + if (guestAddress != nullptr) { + orbis::vmem::unmap( + g_workerProcess ? g_workerProcess : orbis::g_currentThread->tproc, + rx::AddressRange::fromBeginSize( + std::bit_cast(guestAddress), size)); } } @@ -117,6 +142,10 @@ orbis::sint ipmi::IpmiClient::sendSyncMessageRaw( return serverResult; } +void ipmi::setWorkerProcess(orbis::Process *process) { + g_workerProcess = process; +} + ipmi::IpmiClient ipmi::createIpmiClient(orbis::Thread *thread, const char *name) { rx::Ref client; diff --git a/rpcsx/ipmi.hpp b/rpcsx/ipmi.hpp index c66b5e8b3..85afe2ea5 100644 --- a/rpcsx/ipmi.hpp +++ b/rpcsx/ipmi.hpp @@ -258,6 +258,8 @@ struct IpmiServer { extern ipmi::IpmiClient audioIpmiClient; +void setWorkerProcess(orbis::Process *process); + IpmiClient createIpmiClient(orbis::Thread *thread, const char *name); IpmiServer &createIpmiServer(orbis::Process *process, const char *name); orbis::EventFlag *createEventFlag(std::string_view name, uint32_t attrs, diff --git a/rpcsx/linker.cpp b/rpcsx/linker.cpp index fec2d8bd6..0808afd48 100644 --- a/rpcsx/linker.cpp +++ b/rpcsx/linker.cpp @@ -1,11 +1,23 @@ #include "linker.hpp" -#include "io-device.hpp" +#include "kernel/KernelObject.hpp" +#include "orbis/IoDevice.hpp" #include "orbis/KernelAllocator.hpp" +#include "orbis/KernelObject.hpp" +#include "orbis/fmem.hpp" #include "orbis/module/Module.hpp" +#include "orbis/pmem.hpp" #include "orbis/stat.hpp" #include "orbis/uio.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/SharedMutex.hpp" +#include "rx/StrUtil.hpp" +#include "rx/debug.hpp" +#include "rx/die.hpp" +#include "rx/format.hpp" +#include "rx/mem.hpp" +#include "rx/print.hpp" #include "vfs.hpp" -#include "vm.hpp" #include #include #include @@ -16,8 +28,6 @@ #include #include -std::uint64_t monoPimpAddress; - static std::vector unself(const std::byte *image, std::size_t size) { struct [[gnu::packed]] Header { std::uint32_t magic; @@ -70,8 +80,7 @@ static std::vector unself(const std::byte *image, std::size_t size) { auto &segment = segments[i]; if ((segment.flags & 0x7fb) != 0 || segment.decryptedSize != segment.encryptedSize) { - std::fprintf(stderr, "Unsupported self segment (%lx)\n", segment.flags); - std::abort(); + rx::die("Unsupported self segment ({:x})", segment.flags); } if (~segment.flags & 0x800) { @@ -92,6 +101,8 @@ static std::vector unself(const std::byte *image, std::size_t size) { return result; } +std::uint64_t monoPimpAddress; + std::uint64_t rx::linker::encodeFid(std::string_view fid) { static const char suffix[] = "\x51\x8D\x64\xA6\x35\xDE\xD8\xC1\xE6\xB0\x39\xB1\xC3\xE5\x52\x30"; @@ -359,12 +370,159 @@ void rx::linker::override(std::string originalModuleName, std::move(replacedModulePath); } -rx::Ref rx::linker::loadModule(std::span image, - orbis::Process *process) { +struct ElfFile : orbis::File { + rx::AddressRange physicalMemory; + orbis::kvector image; + bool initialized = false; +}; + +struct ElfDevice : orbis::IoDevice { + rx::shared_mutex mtx; + orbis::kmap, rx::StringLess> files; + + orbis::ErrorCode open(rx::Ref *file, const char *path, + std::uint32_t flags, std::uint32_t mode, + orbis::Thread *thread) override { + std::lock_guard lock(mtx); + auto it = files.lower_bound(path); + + if (it != files.end() && it->first == path) { + *file = it->second; + return {}; + } + + rx::Ref rawFile; + if (auto result = + vfs::open(path, orbis::kOpenFlagReadOnly, 0, &rawFile, thread)) { + return result.errc(); + } + + orbis::Stat fileStat; + if (auto error = rawFile->ops->stat(rawFile.get(), &fileStat, nullptr); + error != orbis::ErrorCode{}) { + return error; + } + + auto len = fileStat.size; + + std::vector image(len); + auto ptr = image.data(); + orbis::IoVec ioVec{ + .base = ptr, + .len = static_cast(len), + }; + + orbis::Uio io{ + .offset = 0, + .iov = &ioVec, + .iovcnt = 1, + .resid = 0, + .segflg = orbis::UioSeg::SysSpace, + .rw = orbis::UioRw::Read, + .td = thread, + }; + + while (io.offset < image.size()) { + ioVec = { + .base = ptr + io.offset, + .len = image.size() - io.offset, + }; + + auto result = rawFile->ops->read(rawFile.get(), &io, thread); + rx::dieIf(result != orbis::ErrorCode{}, "elf: '{}' read failed: {}", path, + static_cast(result)); + } + + rawFile = {}; + + if (image[0] != std::byte{'\x7f'} || image[1] != std::byte{'E'} || + image[2] != std::byte{'L'} || image[3] != std::byte{'F'}) { + image = unself(image.data(), image.size()); + } + + Elf64_Ehdr header; + std::memcpy(&header, image.data(), sizeof(Elf64_Ehdr)); + + Elf64_Phdr phdrsStorage[16]; + if (header.e_phnum > std::size(phdrsStorage)) { + rx::die("elf: unexpected count of segments {}, {}", header.e_phnum, path); + } + + std::memcpy(phdrsStorage, image.data() + header.e_phoff, + header.e_phnum * sizeof(Elf64_Phdr)); + auto phdrs = std::span(phdrsStorage, header.e_phnum); + + std::uint64_t endAddress = 0; + std::uint64_t baseAddress = ~static_cast(0); + + for (auto &phdr : phdrs) { + switch (phdr.p_type) { + case kElfProgramTypeLoad: + baseAddress = + std::min(baseAddress, rx::alignDown(phdr.p_vaddr, phdr.p_align)); + + if ((phdr.p_flags & PF_W) == 0) { + endAddress = + std::max(endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, + orbis::vmem::kPageSize)); + } + break; + } + } + + auto imageSize = endAddress - baseAddress; + auto alignedImageSize = rx::alignUp(imageSize, orbis::vmem::kPageSize); + + auto [allocationRange, errc] = orbis::pmem::allocate( + orbis::pmem::getSize() - 1, alignedImageSize, + orbis::AllocationFlags::Stack, orbis::vmem::kPageSize); + + if (errc != orbis::ErrorCode{}) { + return errc; + } + + auto newFile = orbis::knew(); + newFile->physicalMemory = allocationRange; + newFile->image = orbis::kvector(image.begin(), image.end()); + newFile->device = this; + *file = newFile; + + files.emplace_hint(it, orbis::kstring(path), newFile); + return {}; + } + + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *) override { + auto elf = static_cast(file); + auto physicalRange = rx::AddressRange::fromBeginSize( + elf->physicalMemory.beginAddress() + offset, range.size()); + if (!physicalRange.isValid() || + physicalRange.endAddress() > elf->physicalMemory.endAddress()) { + return orbis::ErrorCode::INVAL; + } + + return orbis::pmem::map(range.beginAddress(), physicalRange, + orbis::vmem::toCpuProtection(protection)); + } + + void serialize(rx::Serializer &s) const { + // FIXME: implement + } + + void deserialize(rx::Deserializer &d) { + // FIXME: implement + } +}; + +static auto g_elfDevice = orbis::createGlobalObject(); + +static rx::Ref loadModule(ElfFile *elf, orbis::Process *process, + std::string_view name) { rx::Ref result{orbis::knew()}; Elf64_Ehdr header; - std::memcpy(&header, image.data(), sizeof(Elf64_Ehdr)); + std::memcpy(&header, elf->image.data(), sizeof(Elf64_Ehdr)); result->type = header.e_type; Elf64_Phdr phdrsStorage[16]; @@ -372,7 +530,7 @@ rx::Ref rx::linker::loadModule(std::span image, std::abort(); } - std::memcpy(phdrsStorage, image.data() + header.e_phoff, + std::memcpy(phdrsStorage, elf->image.data() + header.e_phoff, header.e_phnum * sizeof(Elf64_Phdr)); auto phdrs = std::span(phdrsStorage, header.e_phnum); @@ -402,8 +560,8 @@ rx::Ref rx::linker::loadModule(std::span image, case kElfProgramTypeLoad: baseAddress = std::min(baseAddress, rx::alignDown(phdr.p_vaddr, phdr.p_align)); - endAddress = std::max( - endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, vm::kPageSize)); + endAddress = std::max(endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, + orbis::vmem::kPageSize)); break; case kElfProgramTypeDynamic: dynamicPhdrIndex = index; @@ -436,8 +594,8 @@ rx::Ref rx::linker::loadModule(std::span image, sceRelRoPhdrIndex = index; baseAddress = std::min(baseAddress, rx::alignDown(phdr.p_vaddr, phdr.p_align)); - endAddress = std::max( - endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, vm::kPageSize)); + endAddress = std::max(endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, + orbis::vmem::kPageSize)); break; case kElfProgramTypeGnuEhFrame: gnuEhFramePhdrIndex = index; @@ -455,17 +613,27 @@ rx::Ref rx::linker::loadModule(std::span image, } auto imageSize = endAddress - baseAddress; + auto alignedImageSize = rx::alignUp(imageSize, orbis::vmem::kPageSize); - auto imageBase = reinterpret_cast( - vm::map(reinterpret_cast(baseAddress), - rx::alignUp(imageSize, vm::kPageSize), 0, - vm::kMapFlagPrivate | vm::kMapFlagAnonymous | - (baseAddress ? vm::kMapFlagFixed : 0))); + rx::EnumBitSet allocationFlags = {}; - if (imageBase == MAP_FAILED) { - std::abort(); + auto mapHitAddress = baseAddress; + if (baseAddress != 0) { + allocationFlags |= orbis::AllocationFlags::Fixed; + } else if (header.e_type == rx::linker::kElfTypeDyn || + header.e_type == rx::linker::kElfTypeSceDynamic) { + mapHitAddress = 0x800000000; } + auto [imageRange, errc] = orbis::vmem::reserve( + process, mapHitAddress, alignedImageSize, allocationFlags); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to map image memory {}, errno {}", imageSize, + static_cast(errc)); + + auto imageBase = reinterpret_cast(imageRange.beginAddress()); + result->entryPoint = header.e_entry ? reinterpret_cast( imageBase - baseAddress + header.e_entry) @@ -473,7 +641,7 @@ rx::Ref rx::linker::loadModule(std::span image, if (interpPhdrIndex >= 0) { result->interp = reinterpret_cast( - image.data() + phdrs[interpPhdrIndex].p_offset); + elf->image.data() + phdrs[interpPhdrIndex].p_offset); } if (sceProcParamIndex >= 0) { @@ -518,19 +686,12 @@ rx::Ref rx::linker::loadModule(std::span image, }; auto *exinfo = reinterpret_cast( - image.data() + phdrs[gnuEhFramePhdrIndex].p_offset); + elf->image.data() + phdrs[gnuEhFramePhdrIndex].p_offset); - if (exinfo->version != 1) { - std::abort(); - } - - if (exinfo->fdeCount != 0x03) { - std::abort(); - } - - if (exinfo->encodingTable != 0x3b) { - std::abort(); - } + rx::dieIf(exinfo->version != 1, "Unexpected gnu ehframe version"); + rx::dieIf(exinfo->fdeCount != 0x03, "Unexpected gnu ehframe fde count"); + rx::dieIf(exinfo->encodingTable != 0x3b, + "Unexpected gnu ehframe encoding table"); std::byte *dataBuffer = nullptr; @@ -541,7 +702,7 @@ rx::Ref rx::linker::loadModule(std::span image, auto offset = *reinterpret_cast(&exinfo->first); dataBuffer = &exinfo->first + sizeof(std::int32_t) + offset; } else { - std::abort(); + rx::die("unexpected gnu ehframe encoding {:x}", exinfo->encoding); } auto *dataBufferIt = dataBuffer; @@ -563,7 +724,7 @@ rx::Ref rx::linker::loadModule(std::span image, result->ehFrame = imageBase - baseAddress + phdrs[gnuEhFramePhdrIndex].p_vaddr + - (dataBuffer - image.data() - phdrs[gnuEhFramePhdrIndex].p_offset); + (dataBuffer - elf->image.data() - phdrs[gnuEhFramePhdrIndex].p_offset); result->ehFrameSize = dataBufferIt - dataBuffer; } @@ -575,7 +736,7 @@ rx::Ref rx::linker::loadModule(std::span image, if (dynamicPhdrIndex >= 0 && phdrs[dynamicPhdrIndex].p_filesz > 0) { auto &dynPhdr = phdrs[dynamicPhdrIndex]; std::vector dyns(dynPhdr.p_filesz / sizeof(Elf64_Dyn)); - std::memcpy(dyns.data(), image.data() + dynPhdr.p_offset, + std::memcpy(dyns.data(), elf->image.data() + dynPhdr.p_offset, dyns.size() * sizeof(Elf64_Dyn)); int sceStrtabIndex = -1; @@ -616,7 +777,7 @@ rx::Ref rx::linker::loadModule(std::span image, auto sceStrtab = sceStrtabIndex >= 0 && sceDynlibDataPhdrIndex >= 0 ? reinterpret_cast( - image.data() + + elf->image.data() + dynTabOffsetGet(dyns[sceStrtabIndex].d_un.d_val) + phdrs[sceDynlibDataPhdrIndex].p_offset) : nullptr; @@ -625,12 +786,12 @@ rx::Ref rx::linker::loadModule(std::span image, if (strtab == nullptr && strtabIndex >= 0) { strtab = reinterpret_cast( - image.data() + dynTabOffsetGet(dyns[strtabIndex].d_un.d_val)); + elf->image.data() + dynTabOffsetGet(dyns[strtabIndex].d_un.d_val)); } auto sceDynlibData = sceDynlibDataPhdrIndex >= 0 - ? image.data() + phdrs[sceDynlibDataPhdrIndex].p_offset + ? elf->image.data() + phdrs[sceDynlibDataPhdrIndex].p_offset : nullptr; auto sceSymtabData = @@ -646,15 +807,13 @@ rx::Ref rx::linker::loadModule(std::span image, : 0; if (symtab == nullptr && symtabIndex >= 0) { - if (hashIndex < 0) { - std::fprintf(stderr, "SYMTAB without HASH!\n"); - std::abort(); - } + rx::dieIf(hashIndex < 0, "elf: SYMTAB without HASH! {}", + result->moduleName); symtab = reinterpret_cast( - image.data() + dynTabOffsetGet(dyns[symtabIndex].d_un.d_val)); + elf->image.data() + dynTabOffsetGet(dyns[symtabIndex].d_un.d_val)); symtabSize = *reinterpret_cast( - image.data() + dynTabOffsetGet(dyns[hashIndex].d_un.d_val) + + elf->image.data() + dynTabOffsetGet(dyns[hashIndex].d_un.d_val) + sizeof(std::uint32_t)); } @@ -760,7 +919,7 @@ rx::Ref rx::linker::loadModule(std::span image, sceDynlibData + dyn.d_un.d_ptr); } else { pltRelocations = reinterpret_cast( - image.data() + dyn.d_un.d_ptr); + elf->image.data() + dyn.d_un.d_ptr); } break; case kElfDynamicTypeScePltRel: @@ -781,7 +940,7 @@ rx::Ref rx::linker::loadModule(std::span image, sceDynlibData + dyn.d_un.d_ptr); } else { nonPltRelocations = reinterpret_cast( - image.data() + dyn.d_un.d_ptr); + elf->image.data() + dyn.d_un.d_ptr); } break; case kElfDynamicTypeRelaSize: @@ -800,10 +959,7 @@ rx::Ref rx::linker::loadModule(std::span image, } } - if (hasPs4Dyn && hasPs5Dyn) { - std::fprintf(stderr, "unexpected import type\n"); - std::abort(); - } + rx::dieIf(hasPs4Dyn && hasPs5Dyn, "unexpected import type"); if (!hasPs4Dyn && !hasPs5Dyn && interpPhdrIndex >= 0) { result->dynType = orbis::DynType::FreeBsd; @@ -854,24 +1010,23 @@ rx::Ref rx::linker::loadModule(std::span image, library = moduleLibary.substr(0, hashPos); module = moduleLibary.substr(hashPos + 1); - auto libaryNid = *decodeNid(library); - auto moduleNid = *decodeNid(module); + auto libaryNid = *rx::linker::decodeNid(library); + auto moduleNid = *rx::linker::decodeNid(module); symbol.libraryIndex = idToLibraryIndex.at(libaryNid); symbol.moduleIndex = idToModuleIndex.at(moduleNid); - symbol.id = *decodeNid(name); + symbol.id = *rx::linker::decodeNid(name); if (name == "5JrIq4tzVIo") { monoPimpAddress = symbol.address + (std::uint64_t)imageBase; - std::fprintf(stderr, "mono_pimp address = %lx\n", - monoPimpAddress); + rx::println(stderr, "mono_pimp address = {:x}", monoPimpAddress); } - } else if (auto nid = decodeNid(fullName)) { + } else if (auto nid = rx::linker::decodeNid(fullName)) { symbol.id = *nid; symbol.libraryIndex = -1; symbol.moduleIndex = -1; } else { - symbol.id = - encodeFid(strtab + static_cast(sym.st_name)); + symbol.id = rx::linker::encodeFid( + strtab + static_cast(sym.st_name)); symbol.libraryIndex = -1; symbol.moduleIndex = -1; } @@ -882,50 +1037,126 @@ rx::Ref rx::linker::loadModule(std::span image, } } + std::string_view mapName = result->moduleName; + + if (header.e_type == rx::linker::kElfTypeExec || + header.e_type == rx::linker::kElfTypeSceExec || + header.e_type == rx::linker::kElfTypeSceDynExec) { + mapName = "executable"; + } + for (auto phdr : phdrs) { if (phdr.p_type == kElfProgramTypeLoad || phdr.p_type == kElfProgramTypeSceRelRo || phdr.p_type == kElfProgramTypeGnuRelRo) { - auto segmentEnd = rx::alignUp(phdr.p_vaddr + phdr.p_memsz, vm::kPageSize); - auto segmentBegin = - rx::alignDown(phdr.p_vaddr - baseAddress, phdr.p_align); - auto segmentSize = segmentEnd - segmentBegin; - ::mprotect(imageBase + segmentBegin, segmentSize, PROT_WRITE); - std::memcpy(imageBase + phdr.p_vaddr - baseAddress, - image.data() + phdr.p_offset, phdr.p_filesz); - - if (phdr.p_type == kElfProgramTypeSceRelRo || - phdr.p_type == kElfProgramTypeGnuRelRo) { - phdr.p_flags |= vm::kMapProtCpuWrite; // TODO: reprotect on relocations - } - - int mapFlags = 0; + rx::EnumBitSet protFlags = {}; if (phdr.p_flags & PF_X) { - mapFlags |= vm::kMapProtCpuExec; + protFlags |= orbis::vmem::Protection::CpuExec; } if (phdr.p_flags & PF_W) { - mapFlags |= vm::kMapProtCpuWrite; + protFlags |= orbis::vmem::Protection::CpuWrite; } if (phdr.p_flags & PF_R) { - mapFlags |= vm::kMapProtCpuRead; + protFlags |= orbis::vmem::Protection::CpuRead; } - if (mapFlags == 0) { - mapFlags = vm::kMapProtCpuWrite; + if (!protFlags) { + protFlags = orbis::vmem::Protection::CpuRead | + orbis::vmem::Protection::CpuWrite; } - vm::protect(imageBase + segmentBegin, segmentSize, mapFlags); + phdr.p_memsz = rx::alignUp(phdr.p_memsz, orbis::vmem::kPageSize); - if (phdr.p_type == kElfProgramTypeLoad) { - if (result->segmentCount >= std::size(result->segments)) { - std::abort(); + auto segmentEnd = + rx::alignUp(phdr.p_vaddr + phdr.p_memsz, orbis::vmem::kPageSize); + auto segmentBegin = rx::alignDown(phdr.p_vaddr, phdr.p_align); + + auto segmentRange = + rx::AddressRange::fromBeginEnd(segmentBegin, segmentEnd); + + if ((phdr.p_flags & PF_W) || phdr.p_type == kElfProgramTypeSceRelRo || + phdr.p_type == kElfProgramTypeGnuRelRo) { + // map anonymous memory, copy segment data + + auto [vmem, vmemErrc] = + orbis::vmem::mapFlex(process, segmentRange.size(), protFlags, + imageRange.beginAddress() + segmentBegin, + orbis::AllocationFlags::Fixed, {}, mapName); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "elf: failed to map flexible to virtual memory {}", + (int)vmemErrc); + + if ((phdr.p_flags & PF_W) == 0) { + rx::mem::protect(vmem, + rx::mem::Protection::R | rx::mem::Protection::W); } - auto &segment = result->segments[result->segmentCount++]; + std::memcpy(imageBase + phdr.p_vaddr, elf->image.data() + phdr.p_offset, + phdr.p_filesz); + + rx::println(stderr, "{}: RW segment {:x}-{:x}, {}", result->moduleName, + segmentRange.beginAddress(), segmentRange.endAddress(), + protFlags.raw()); + } else { + // map elf device directly + rx::dieIf(rx::alignUp(phdr.p_filesz, orbis::vmem::kPageSize) != + segmentRange.size(), + "unexpected read only segment size, {:x} vs {:x}", + phdr.p_filesz, segmentRange.size()); + + rx::println(stderr, "{}: RX segment {:x}-{:x}, {}", result->moduleName, + segmentRange.beginAddress(), segmentRange.endAddress(), + protFlags.raw()); + + { + std::lock_guard lock(elf->mtx); + if (!elf->initialized) { + auto [vmem, vmemErrc] = orbis::vmem::mapFile( + process, imageRange.beginAddress() + segmentBegin, + segmentRange.size(), orbis::AllocationFlags::Fixed, + protFlags | orbis::vmem::Protection::CpuWrite, {}, {}, elf, + segmentBegin, mapName); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "elf: failed to map elf to virtual memory {}", vmemErrc); + + std::memset(imageBase + phdr.p_vaddr + phdr.p_filesz, 0, + phdr.p_memsz - phdr.p_filesz); + std::memcpy(imageBase + phdr.p_vaddr, + elf->image.data() + phdr.p_offset, phdr.p_filesz); + elf->initialized = true; + } + } + + auto [vmem, vmemErrc] = orbis::vmem::mapFile( + process, imageRange.beginAddress() + segmentBegin, + segmentRange.size(), orbis::AllocationFlags::Fixed, protFlags, + orbis::vmem::BlockFlags::FlexibleMemory | + orbis::vmem::BlockFlags::Commited, + orbis::vmem::BlockFlagsEx::Shared, elf, segmentBegin, mapName); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "elf: failed to map elf to virtual memory {}", (int)vmemErrc); + } + + std::uint32_t segmentIndex; + if (phdr.p_type == kElfProgramTypeLoad) { + segmentIndex = protFlags & orbis::vmem::Protection::CpuExec ? 0 : 1; + } else { + segmentIndex = 2; + } + + auto &segment = result->segments[segmentIndex]; + if (segment.addr != nullptr) { + rx::println(stderr, "elf: corrupted, segment {} overriding. {}", + segmentIndex, name); + } else { segment.addr = imageBase + segmentBegin; segment.size = phdr.p_memsz; - segment.prot = phdr.p_flags; + segment.prot = protFlags.toUnderlying(); + result->segmentCount = std::max(segmentIndex + 1, result->segmentCount); } } } @@ -951,16 +1182,19 @@ rx::Ref rx::linker::loadModule(std::span image, result->phNum = header.e_phnum; result->proc = process; - std::printf("Loaded module '%s' (%lx) from object '%s', address: %p - %p\n", + std::strncpy(result->soName, name.data(), sizeof(result->soName)); + rx::println("Loaded module '{}' ({:x}) from object '{}', " + "address: {} - {}", result->moduleName, (unsigned long)result->attributes, - result->soName, imageBase, (char *)imageBase + result->size); + result->soName, (void *)imageBase, + (void *)((char *)imageBase + result->size)); for (const auto &mod : result->neededModules) { - std::printf(" needed module '%s' (%lx)\n", mod.name.c_str(), + rx::println(" needed module '{}' ({:x})", mod.name.c_str(), (unsigned long)mod.attr); } for (const auto &lib : result->neededLibraries) { - std::printf(" needed library '%s' (%lx), kind %s\n", lib.name.c_str(), + rx::println(" needed library '{}' ({:x}), kind {}", lib.name.c_str(), (unsigned long)lib.attr, lib.isExport ? "export" : "import"); } @@ -973,59 +1207,17 @@ rx::Ref rx::linker::loadModule(std::span image, static rx::Ref loadModuleFileImpl(std::string_view path, orbis::Thread *thread) { - rx::Ref instance; - if (vfs::open(path, orbis::kOpenFlagReadOnly, 0, &instance, thread) - .isError()) { + rx::Ref elf; + if (auto errc = g_elfDevice->open(&elf, path.data(), 0, 0, thread); + errc != orbis::ErrorCode{}) { return {}; } - orbis::Stat fileStat; - if (instance->ops->stat(instance.get(), &fileStat, nullptr) != - orbis::ErrorCode{}) { - return {}; + if (auto sepPos = path.rfind('/'); sepPos != std::string_view::npos) { + path.remove_prefix(sepPos + 1); } - auto len = fileStat.size; - - std::vector image(len); - auto ptr = image.data(); - orbis::IoVec ioVec{ - .base = ptr, - .len = static_cast(len), - }; - orbis::Uio io{ - .offset = 0, - .iov = &ioVec, - .iovcnt = 1, - .resid = 0, - .segflg = orbis::UioSeg::SysSpace, - .rw = orbis::UioRw::Read, - .td = thread, - }; - - while (io.offset < image.size()) { - ioVec = { - .base = ptr + io.offset, - .len = image.size() - io.offset, - }; - auto result = instance->ops->read(instance.get(), &io, thread); - if (result != orbis::ErrorCode{}) { - std::fprintf(stderr, "Module file reading error\n"); - std::abort(); - } - } - - if (image[0] != std::byte{'\x7f'} || image[1] != std::byte{'E'} || - image[2] != std::byte{'L'} || image[3] != std::byte{'F'}) { - image = unself(image.data(), image.size()); - - // std::ofstream( - // std::filesystem::path(path).filename().replace_extension("elf"), - // std::ios::binary) - // .write((const char *)image.data(), image.size()); - } - - return rx::linker::loadModule(image, thread->tproc); + return loadModule(elf.rawStaticCast(), thread->tproc, path); } rx::Ref rx::linker::loadModuleFile(std::string_view path, diff --git a/rpcsx/linker.hpp b/rpcsx/linker.hpp index 22d48cfe4..9f8e1b53a 100644 --- a/rpcsx/linker.hpp +++ b/rpcsx/linker.hpp @@ -75,8 +75,6 @@ enum OrbisElfType_t { void override(std::string originalModuleName, std::filesystem::path replacedModulePath); -rx::Ref loadModule(std::span image, - orbis::Process *process); rx::Ref loadModuleFile(std::string_view path, orbis::Thread *thread); } // namespace rx::linker diff --git a/rpcsx/main.cpp b/rpcsx/main.cpp index a999183ab..cbe11b8a0 100644 --- a/rpcsx/main.cpp +++ b/rpcsx/main.cpp @@ -8,16 +8,22 @@ #include "ipmi.hpp" #include "linker.hpp" #include "ops.hpp" +#include "orbis/dmem.hpp" +#include "orbis/fmem.hpp" +#include "orbis/pmem.hpp" #include "orbis/ucontext.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" #include "rx/Config.hpp" +#include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/mem.hpp" #include "rx/print.hpp" #include "rx/watchdog.hpp" #include "thread.hpp" #include "vfs.hpp" -#include "vm.hpp" #include "xbyak/xbyak.h" +#include #include #include #include @@ -42,6 +48,7 @@ #include #include #include +#include #include #include @@ -65,26 +72,19 @@ handle_signal(int sig, siginfo_t *info, void *ucontext) { orbis::g_currentThread->tproc->vmId >= 0 && sig == SIGSEGV && signalAddress >= orbis::kMinAddress && signalAddress < orbis::kMaxAddress) { - auto vmid = orbis::g_currentThread->tproc->vmId; + auto process = orbis::g_currentThread->tproc; + auto vmid = process->vmId; auto ctx = reinterpret_cast(ucontext); bool isWrite = (ctx->uc_mcontext.gregs[REG_ERR] & 0x2) != 0; - auto origVmProt = vm::getPageProtection(signalAddress); - int prot = 0; + auto origVmProt = orbis::vmem::queryProtection(process, signalAddress); + auto page = signalAddress / rx::mem::pageSize; - - if (origVmProt & vm::kMapProtCpuRead) { - prot |= PROT_READ; - } - if (origVmProt & vm::kMapProtCpuWrite) { - prot |= PROT_WRITE; - } - if (origVmProt & vm::kMapProtCpuExec) { - prot |= PROT_EXEC; - } - auto gpuDevice = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}; - if (gpuDevice && (prot & (isWrite ? PROT_WRITE : PROT_READ)) != 0) { + if (gpuDevice && origVmProt && + (origVmProt->prot & (isWrite ? orbis::vmem::Protection::CpuWrite + : orbis::vmem::Protection::CpuRead))) { + auto prot = toCpuProtection(origVmProt->prot); auto &gpuContext = gpuDevice.getContext(); while (true) { auto flags = @@ -115,7 +115,7 @@ handle_signal(int sig, siginfo_t *info, void *ucontext) { } if (!isWrite) { - prot &= ~PROT_WRITE; + prot = prot & ~rx::mem::Protection::W; break; } @@ -125,19 +125,20 @@ handle_signal(int sig, siginfo_t *info, void *ucontext) { } } - if (::mprotect((void *)(page * rx::mem::pageSize), rx::mem::pageSize, - prot)) { - std::perror("cache reprotection error"); - std::abort(); - } + auto range = rx::AddressRange::fromBeginSize(page * rx::mem::pageSize, + rx::mem::pageSize); + auto errc = rx::mem::protect(range, prot); + rx::dieIf(errc != std::errc{}, + "cache: virtual memory protection failed, address {}, error {}", + range.beginAddress(), static_cast(errc)); _writefsbase_u64(orbis::g_currentThread->fsBase); return; } - std::fprintf(stderr, "SIGSEGV, address %lx, access %s, prot %s\n", + std::fprintf(stderr, "SIGSEGV, address %lx, access %s, prot %u\n", signalAddress, isWrite ? "write" : "read", - vm::mapProtToString(origVmProt).c_str()); + origVmProt ? origVmProt->prot.toUnderlying() : -1); } if (orbis::g_currentThread != nullptr) { @@ -290,14 +291,14 @@ struct StackWriter { }; static bool g_traceSyscalls = false; -static const char *getSyscallName(orbis::Thread *thread, int sysno) { +static const orbis::sysent *getSyscallEnt(orbis::Thread *thread, int sysno) { auto sysvec = thread->tproc->sysent; if (sysno >= sysvec->size) { return nullptr; } - return orbis::getSysentName(sysvec->table[sysno].call); + return sysvec->table + sysno; } static void onSysEnter(orbis::Thread *thread, int id, uint64_t *args, int argsCount) { @@ -307,12 +308,13 @@ static void onSysEnter(orbis::Thread *thread, int id, uint64_t *args, flockfile(stderr); std::fprintf(stderr, " [%u] ", thread->tid); - if (auto name = getSyscallName(thread, id)) { - std::fprintf(stderr, "%s(", name); - } else { - std::fprintf(stderr, "sys_%u(", id); + if (auto ent = getSyscallEnt(thread, id)) { + std::fprintf(stderr, "%s\n", ent->format(args).c_str()); + return; } + std::fprintf(stderr, "sys_%u(", id); + for (int i = 0; i < argsCount; ++i) { if (i != 0) { std::fprintf(stderr, ", "); @@ -332,24 +334,26 @@ static void onSysExit(orbis::Thread *thread, int id, uint64_t *args, } flockfile(stderr); - std::fprintf(stderr, "%c: [%u] ", result.isError() ? 'E' : 'S', thread->tid); + rx::print(stderr, "{}: [{}] ", result.isError() ? 'E' : 'S', thread->tid); - if (auto name = getSyscallName(thread, id)) { - std::fprintf(stderr, "%s(", name); + if (auto ent = getSyscallEnt(thread, id)) { + rx::print(stderr, "{}", ent->format(args)); } else { - std::fprintf(stderr, "sys_%u(", id); - } + rx::print(stderr, "sys_{}(", id); - for (int i = 0; i < argsCount; ++i) { - if (i != 0) { - std::fprintf(stderr, ", "); + for (int i = 0; i < argsCount; ++i) { + if (i != 0) { + rx::print(stderr, ", "); + } + + rx::print(stderr, "{:#x}", args[i]); } - std::fprintf(stderr, "%#lx", args[i]); + rx::print(stderr, ")"); } - std::fprintf(stderr, ") -> Status %d, Value %lx:%lx\n", result.value(), - thread->retval[0], thread->retval[1]); + rx::println(stderr, " -> {}, Value {:x}:{:x}", result.errc(), + thread->retval[0], thread->retval[1]); if (result.isError()) { thread->where(); @@ -357,11 +361,11 @@ static void onSysExit(orbis::Thread *thread, int id, uint64_t *args, funlockfile(stderr); } -static void guestInitDev() { - auto dmem1 = createDmemCharacterDevice(1); - orbis::g_context->dmemDevice = dmem1; +static void guestInitDev(orbis::Thread *thread) { + auto dmem0 = createDmemCharacterDevice(0); + dmem0->open(&orbis::g_context->dmem, "", 0, 0, thread); - auto dce = createDceCharacterDevice(); + auto dce = createDceCharacterDevice(thread->tproc); orbis::g_context->dceDevice = dce; auto ttyFd = ::open("tty.txt", O_CREAT | O_TRUNC | O_WRONLY, 0666); @@ -377,12 +381,12 @@ static void guestInitDev() { auto analogAudioDevice = nullAudioDevice; auto spdifAudioDevice = nullAudioDevice; - vfs::addDevice("dmem0", createDmemCharacterDevice(0)); vfs::addDevice("npdrm", createNpdrmCharacterDevice()); vfs::addDevice("icc_configuration", createIccConfigurationCharacterDevice()); vfs::addDevice("console", consoleDev); vfs::addDevice("camera", createCameraCharacterDevice()); - vfs::addDevice("dmem1", dmem1); + vfs::addDevice("dmem0", dmem0); + vfs::addDevice("dmem1", createDmemCharacterDevice(1)); vfs::addDevice("dmem2", createDmemCharacterDevice(2)); vfs::addDevice("stdout", consoleDev); vfs::addDevice("stderr", consoleDev); @@ -518,7 +522,7 @@ static void guestInitDev() { auto shm = createShmDevice(); orbis::g_context->shmDevice = shm; - orbis::g_context->blockpoolDevice = createBlockPoolDevice(); + createBlockPoolDevice()->open(&orbis::g_context->blockpool, "", 0, 0, thread); } static void guestInitFd(orbis::Thread *mainThread) { @@ -542,22 +546,29 @@ struct ExecEnv { int guestExec(orbis::Thread *mainThread, ExecEnv execEnv, rx::Ref executableModule, std::span argv, std::span envp) { - const auto stackEndAddress = 0x7'ffff'c000ull; - const auto stackSize = 0x40000 * 32; - auto stackStartAddress = stackEndAddress - stackSize; - mainThread->stackStart = - vm::map(reinterpret_cast(stackStartAddress), stackSize, - vm::kMapProtCpuWrite | vm::kMapProtCpuRead, - vm::kMapFlagAnonymous | vm::kMapFlagFixed | vm::kMapFlagPrivate | - vm::kMapFlagStack); + const auto stackEndAddress = 0x7'eeff'c000ull; + const auto stackSize = 0x200000; - mainThread->stackEnd = - reinterpret_cast(mainThread->stackStart) + stackSize; + auto stackStartAddress = stackEndAddress - stackSize; + + auto [stackVmRange, vmErrc] = orbis::vmem::mapFlex( + mainThread->tproc, stackSize, + orbis::vmem::Protection::CpuRead | orbis::vmem::Protection::CpuWrite, + stackStartAddress, + orbis::AllocationFlags::Stack | orbis::AllocationFlags::Fixed, + orbis::vmem::BlockFlags::Stack, "main stack"); + + rx::dieIf(vmErrc != orbis::ErrorCode{}, + "failed to map main thread stack, error {}", + static_cast(vmErrc)); + + mainThread->stackStart = stackVmRange.beginAddress(); + mainThread->stackEnd = stackVmRange.endAddress(); std::vector argvOffsets; std::vector envpOffsets; - StackWriter stack{reinterpret_cast(mainThread->stackEnd)}; + StackWriter stack{mainThread->stackEnd}; for (auto &elem : argv) { argvOffsets.push_back(stack.pushString(elem.data())); @@ -680,26 +691,24 @@ ExecEnv guestCreateExecEnv(orbis::Thread *mainThread, return {.entryPoint = entryPoint, .interpBase = interpBase}; } - auto libSceLibcInternal = rx::linker::loadModuleFile( - "/system/common/lib/libSceLibcInternal.sprx", mainThread); - - if (libSceLibcInternal == nullptr) { - std::println(stderr, "libSceLibcInternal not found"); - std::abort(); - } - - libSceLibcInternal->id = - mainThread->tproc->modulesMap.insert(libSceLibcInternal); - auto libkernel = rx::linker::loadModuleFile( (isSystem ? "/system/common/lib/libkernel_sys.sprx" : "/system/common/lib/libkernel.sprx"), mainThread); - if (libkernel == nullptr) { - rx::println(stderr, "libkernel not found"); - std::abort(); - } + rx::dieIf(libkernel == nullptr, "libkernel not found"); + libkernel->id = mainThread->tproc->modulesMap.insert(libkernel); + + mainThread->tproc->libkernelRange = rx::AddressRange::fromBeginSize( + std::bit_cast(libkernel->base), libkernel->size); + + auto libSceLibcInternal = rx::linker::loadModuleFile( + "/system/common/lib/libSceLibcInternal.sprx", mainThread); + + rx::dieIf(libSceLibcInternal == nullptr, "libSceLibcInternal not found"); + + libSceLibcInternal->id = + mainThread->tproc->modulesMap.insert(libSceLibcInternal); if (orbis::g_context->fwType == orbis::FwType::Ps4) { for (auto sym : libkernel->symbols) { @@ -707,7 +716,7 @@ ExecEnv guestCreateExecEnv(orbis::Thread *mainThread, auto address = (uint64_t)libkernel->base + sym.address; ::mprotect((void *)rx::alignDown(address, 0x1000), rx::alignUp(sym.size + sym.address, 0x1000), PROT_WRITE); - std::println("patching sceKernelGetMainSocId"); + rx::println("patching sceKernelGetMainSocId"); struct GetMainSocId : Xbyak::CodeGenerator { GetMainSocId(std::uint64_t address, std::uint64_t size) : Xbyak::CodeGenerator(size, (void *)address) { @@ -741,7 +750,6 @@ ExecEnv guestCreateExecEnv(orbis::Thread *mainThread, } } - libkernel->id = mainThread->tproc->modulesMap.insert(libkernel); interpBase = reinterpret_cast(libkernel->base); entryPoint = libkernel->entryPoint; @@ -757,18 +765,18 @@ int guestExec(orbis::Thread *mainThread, } static void usage(const char *argv0) { - std::println("{} [...] [args...]", argv0); - std::println(" options:"); - std::println(" --version, -v - print version"); - std::println(" -m, --mount "); - std::println(" -o, --override "); - std::println(" --fw "); - std::println( + rx::println("{} [...] [args...]", argv0); + rx::println(" options:"); + rx::println(" --version, -v - print version"); + rx::println(" -m, --mount "); + rx::println(" -o, --override "); + rx::println(" --fw "); + rx::println( " --gpu - specify physical gpu index to use, default is 0"); - std::println(" --disable-cache - disable cache of gpu resources"); - // std::println(" --presenter "); - std::println(" --trace"); + rx::println(" --disable-cache - disable cache of gpu resources"); + // rx::println(" --presenter "); + rx::println(" --trace"); } static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, @@ -792,6 +800,7 @@ static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, } auto process = orbis::createProcess(thread->tproc, childPid); + orbis::vmem::initialize(process, true); // override init process mappings auto logFd = ::open(("log-" + std::to_string(childPid) + ".txt").c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666); @@ -822,10 +831,9 @@ static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, 0xF0000000FFFF4000, }, }; - process->budgetId = 0; + process->budgetId = thread->tproc->budgetId; process->isInSandbox = false; - vm::fork(childPid); vfs::fork(); *flag = true; @@ -859,8 +867,6 @@ static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, } } - vm::reset(); - thread->tproc->nextTlsSlot = 1; auto executableModule = rx::linker::loadModuleFile(path, thread); @@ -920,10 +926,10 @@ int main(int argc, const char *argv[]) { return 1; } - std::println("mounting '{}' to virtual '{}'", argv[argIndex + 1], - argv[argIndex + 2]); + rx::println("mounting '{}' to virtual '{}'", argv[argIndex + 1], + argv[argIndex + 2]); if (!std::filesystem::is_directory(argv[argIndex + 1])) { - std::println(stderr, "Directory '{}' not exists", argv[argIndex + 1]); + rx::println(stderr, "Directory '{}' not exists", argv[argIndex + 1]); return 1; } @@ -939,7 +945,7 @@ int main(int argc, const char *argv[]) { return 1; } - std::println("mounting firmware '{}'", argv[argIndex + 1]); + rx::println("mounting firmware '{}'", argv[argIndex + 1]); vfs::mount("/", createHostIoDevice(argv[argIndex + 1], "/")); @@ -1030,10 +1036,19 @@ int main(int argc, const char *argv[]) { orbis::constructAllGlobals(); orbis::g_context->deviceEventEmitter = orbis::knew(); + // FIXME: determine mode by reading elf file + orbis::pmem::initialize(10ull * 1024 * 1024 * 1024); + orbis::dmem::initialize(); + orbis::fmem::initialize(2ull * 1024 * 1024 * 1024); + rx::startWatchdog(); rx::createGpuDevice(); vfs::initialize(); + while (orbis::g_context->gpuDevice == nullptr) { + std::this_thread::yield(); + } + std::vector guestArgv(argv + argIndex, argv + argc); if (guestArgv.empty()) { guestArgv.emplace_back("/mini-syscore.elf"); @@ -1046,6 +1061,8 @@ int main(int argc, const char *argv[]) { // vm::printHostStats(); orbis::allocatePid(); auto initProcess = orbis::createProcess(nullptr, asRoot ? 1 : 10); + orbis::vmem::initialize(initProcess); + // pthread_setname_np(pthread_self(), "10.MAINTHREAD"); int status = 0; @@ -1080,8 +1097,7 @@ int main(int argc, const char *argv[]) { .flags = 0, .item = { - // vmem - reserved space for stack - .total = 2ul * 1024 * 1024 * 1024 - (64 * 1024 * 1024), + .total = 0x1C000000, }, }, { @@ -1134,8 +1150,6 @@ int main(int argc, const char *argv[]) { }, }; - vm::initialize(initProcess->pid); - auto bigAppBudget = orbis::g_context->createProcessTypeBudget( orbis::Budget::ProcessType::BigApp, "big app budget", bigAppBudgetInfo); @@ -1219,10 +1233,7 @@ int main(int argc, const char *argv[]) { auto executableModule = rx::linker::loadModuleFile(guestArgv[0], mainThread); - if (executableModule == nullptr) { - std::println(stderr, "Failed to open '{}'", guestArgv[0]); - std::abort(); - } + rx::dieIf(executableModule == nullptr, "Failed to open '{}'", guestArgv[0]); executableModule->id = initProcess->modulesMap.insert(executableModule); initProcess->processParam = executableModule->processParam; @@ -1249,7 +1260,7 @@ int main(int argc, const char *argv[]) { executableModule->dynType = orbis::DynType::Ps5; } - guestInitDev(); + guestInitDev(mainThread); guestInitFd(mainThread); // data transfer mode @@ -1294,6 +1305,9 @@ int main(int argc, const char *argv[]) { orbis::g_context->regMgrInt[0x9010000] = 0; // video out color effect if (!isSystem) { + // do IPMI allocations in init process + ipmi::setWorkerProcess(initProcess); + ipmi::createMiniSysCoreObjects(initProcess); ipmi::createSysAvControlObjects(initProcess); ipmi::createSysCoreObjects(initProcess); @@ -1380,7 +1394,6 @@ int main(int argc, const char *argv[]) { status = guestExec(mainThread, execEnv, std::move(executableModule), guestArgv, {}); - vm::deinitialize(); rx::thread::deinitialize(); return status; diff --git a/rpcsx/ops.cpp b/rpcsx/ops.cpp index 29b29c965..b9a1d4a7e 100644 --- a/rpcsx/ops.cpp +++ b/rpcsx/ops.cpp @@ -2,8 +2,6 @@ #include "backtrace.hpp" #include "io-device.hpp" #include "io-devices.hpp" -#include "iodev/blockpool.hpp" -#include "iodev/dmem.hpp" #include "linker.hpp" #include "orbis-config.hpp" #include "orbis/KernelContext.hpp" @@ -14,14 +12,12 @@ #include "orbis/uio.hpp" #include "orbis/umtx.hpp" #include "orbis/utils/Logs.hpp" -#include "orbis/vm.hpp" +#include "orbis/vmem.hpp" #include "rx/Rc.hpp" #include "rx/watchdog.hpp" #include "thread.hpp" #include "vfs.hpp" -#include "vm.hpp" #include -#include #include #include #include @@ -31,10 +27,15 @@ #include #include #include + +#ifdef __linux #include #include -#include #include +#include +#endif + +#include using namespace orbis; @@ -89,7 +90,7 @@ loadPrx(orbis::Thread *thread, std::string_view name, bool relocate, loadedObjects[module->soName] = module.get(); if (loadedModules.try_emplace(module->moduleName, module.get()).second) { - std::printf("Setting '%s' as '%s' module\n", module->soName, + std::printf("Registering '%s' as '%s' module\n", module->soName, module->moduleName); } @@ -142,17 +143,7 @@ loadPrx(orbis::Thread *thread, std::string_view path, bool relocate) { std::string expectedName; if (auto sep = path.rfind('/'); sep != std::string_view::npos) { - auto tmpExpectedName = path.substr(sep + 1); - - if (tmpExpectedName.ends_with(".sprx")) { - tmpExpectedName.remove_suffix(5); - } - - expectedName += tmpExpectedName; - - if (!expectedName.ends_with(".prx")) { - expectedName += ".prx"; - } + expectedName = path.substr(sep + 1); } return loadPrx(thread, path, relocate, loadedObjects, loadedModules, @@ -174,126 +165,6 @@ std::string getAbsolutePath(std::string path, Thread *thread) { return std::filesystem::path(path).lexically_normal().string(); } -orbis::SysResult mmap(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len, orbis::sint prot, orbis::sint flags, - orbis::sint fd, orbis::off_t pos) { - if (fd == -1) { - auto result = vm::map(addr, len, prot, flags); - if (result == (void *)-1) { - return ErrorCode::NOMEM; - } - - thread->retval[0] = reinterpret_cast(result); - return {}; - } - - auto file = thread->tproc->fileDescriptors.get(fd); - if (file == nullptr) { - return ErrorCode::BADF; - } - - if (file->ops->mmap == nullptr) { - ORBIS_LOG_FATAL("unimplemented mmap", fd, (void *)addr, len, prot, flags, - pos); - return mmap(thread, addr, len, prot, flags, -1, 0); - } - - void *maddr = addr; - auto result = - file->ops->mmap(file.get(), &maddr, len, prot, flags, pos, thread); - - if (result != ErrorCode{}) { - return result; - } - - thread->retval[0] = reinterpret_cast(maddr); - return {}; -} - -orbis::SysResult dmem_mmap(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len, orbis::sint memoryType, - orbis::sint prot, sint flags, - orbis::off_t directMemoryStart) { - auto dmem = static_cast(orbis::g_context->dmemDevice.get()); - void *address = addr; - auto result = dmem->mmap(&address, len, prot, flags, directMemoryStart); - if (result != ErrorCode{}) { - return result; - } - - thread->retval[0] = reinterpret_cast(address); - return {}; -} - -orbis::SysResult munmap(orbis::Thread *, orbis::ptr addr, - orbis::size_t len) { - if (vm::unmap(addr, len)) { - return {}; - } - return ErrorCode::INVAL; -} - -orbis::SysResult msync(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint flags) { - return {}; -} - -orbis::SysResult mprotect(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint prot) { - if (!vm::protect((void *)addr, len, prot)) { - return ErrorCode::INVAL; - } - return {}; -} - -orbis::SysResult minherit(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint inherit) { - return ErrorCode::INVAL; -} - -orbis::SysResult madvise(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint behav) { - return {}; -} - -orbis::SysResult mincore(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::ptr vec) { - return ErrorCode::INVAL; -} - -orbis::SysResult mlock(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len) { - return {}; -} -orbis::SysResult mlockall(orbis::Thread *thread, orbis::sint how) { return {}; } -orbis::SysResult munlockall(orbis::Thread *thread) { return {}; } -orbis::SysResult munlock(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len) { - return {}; -} -orbis::SysResult virtual_query(orbis::Thread *thread, - orbis::ptr addr, orbis::sint flags, - orbis::ptr info, orbis::ulong infoSize) { - if (infoSize != sizeof(vm::VirtualQueryInfo)) { - return ErrorCode::INVAL; - } - - if (!vm::virtualQuery(addr, flags, (vm::VirtualQueryInfo *)info)) { - return ErrorCode::ACCES; - } - return {}; -} - -orbis::SysResult -query_memory_protection(orbis::Thread *thread, orbis::ptr address, - orbis::ptr protection) { - if (vm::queryProtection(address, &protection->startAddress, - &protection->endAddress, &protection->prot)) { - return {}; - } - return ErrorCode::INVAL; -} - orbis::SysResult open(orbis::Thread *thread, orbis::ptr path, orbis::sint flags, orbis::sint mode, rx::Ref *file) { @@ -324,33 +195,6 @@ orbis::SysResult rename(Thread *thread, ptr from, thread); } -orbis::SysResult blockpool_open(orbis::Thread *thread, - rx::Ref *file) { - auto dev = static_cast(orbis::g_context->blockpoolDevice.get()); - return dev->open(file, nullptr, 0, 0, thread); -} - -orbis::SysResult blockpool_map(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len, orbis::sint prot, - orbis::sint flags) { - auto blockpool = - static_cast(orbis::g_context->blockpoolDevice.get()); - void *address = addr; - auto result = blockpool->map(&address, len, prot, flags, thread); - if (result != ErrorCode{}) { - return result; - } - - thread->retval[0] = reinterpret_cast(address); - return {}; -} -orbis::SysResult blockpool_unmap(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len) { - auto blockpool = - static_cast(orbis::g_context->blockpoolDevice.get()); - return blockpool->unmap(addr, len, thread); -} - orbis::SysResult socket(orbis::Thread *thread, orbis::ptr name, orbis::sint domain, orbis::sint type, orbis::sint protocol, rx::Ref *file) { @@ -780,6 +624,7 @@ SysResult fork(Thread *thread, slong flags) { } auto process = orbis::createProcess(thread->tproc, childPid); + orbis::vmem::fork(process, thread->tproc); process->hostPid = ::getpid(); process->sysent = thread->tproc->sysent; process->onSysEnter = thread->tproc->onSysEnter; @@ -803,7 +648,6 @@ SysResult fork(Thread *thread, slong flags) { } } - vm::fork(childPid); vfs::fork(); *flag = true; @@ -892,7 +736,7 @@ SysResult execve(Thread *thread, ptr fname, ptr> argv, } } - vm::reset(); + orbis::vmem::initialize(thread->tproc, true); thread->tproc->nextTlsSlot = 1; for (auto [id, mod] : thread->tproc->modulesMap) { @@ -940,6 +784,7 @@ void block(Thread *thread) { if (--thread->unblocked != 0) { return; } +#ifdef __linux sigset_t set; sigemptyset(&set); sigaddset(&set, SIGUSR1); @@ -954,6 +799,7 @@ void block(Thread *thread) { std::free(thread->altStack.back()); thread->altStack.pop_back(); thread->sigReturns.pop_back(); +#endif } void unblock(Thread *thread) { @@ -982,29 +828,12 @@ void unblock(Thread *thread) { } // namespace ProcessOps rx::procOpsTable = { - .mmap = mmap, - .dmem_mmap = dmem_mmap, - .munmap = munmap, - .msync = msync, - .mprotect = mprotect, - .minherit = minherit, - .madvise = madvise, - .mincore = mincore, - .mlock = mlock, - .mlockall = mlockall, - .munlockall = munlockall, - .munlock = munlock, - .virtual_query = virtual_query, - .query_memory_protection = query_memory_protection, .open = open, .shm_open = shm_open, .unlink = unlink, .mkdir = mkdir, .rmdir = rmdir, .rename = rename, - .blockpool_open = blockpool_open, - .blockpool_map = blockpool_map, - .blockpool_unmap = blockpool_unmap, .socket = socket, .socketpair = socketPair, .shm_unlink = shm_unlink, diff --git a/rpcsx/thread.cpp b/rpcsx/thread.cpp index efea0816d..9ed38b556 100644 --- a/rpcsx/thread.cpp +++ b/rpcsx/thread.cpp @@ -276,7 +276,7 @@ void rx::thread::copyContext(orbis::Thread *thread, orbis::UContext &dst, void rx::thread::setContext(orbis::Thread *thread, const orbis::UContext &src) { auto &context = *std::bit_cast(thread->context); thread->stackStart = src.stack.sp; - thread->stackEnd = (char *)thread->stackStart + src.stack.size; + thread->stackEnd = thread->stackStart + src.stack.size; thread->setSigMask(src.sigmask); // dst.onstack = src.gregs[REG_ONSTACK]; diff --git a/rpcsx/vm.cpp b/rpcsx/vm.cpp deleted file mode 100644 index 2f6d8f28d..000000000 --- a/rpcsx/vm.cpp +++ /dev/null @@ -1,1221 +0,0 @@ -#include "vm.hpp" -#include "gpu/DeviceCtl.hpp" -#include "io-device.hpp" -#include "iodev/dmem.hpp" -#include "orbis-config.hpp" -#include "orbis/KernelContext.hpp" -#include "orbis/thread/Process.hpp" -#include "orbis/thread/Thread.hpp" -#include "orbis/utils/Logs.hpp" -#include "rx/Rc.hpp" -#include "rx/format.hpp" -#include "rx/print.hpp" -#include "rx/watchdog.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static std::mutex g_mtx; - -std::string vm::mapFlagsToString(std::int32_t flags) { - std::string result; - - if ((flags & kMapFlagShared) == kMapFlagShared) { - if (!result.empty()) { - result += " | "; - } - - result += "Shared"; - flags &= ~kMapFlagShared; - } - if ((flags & kMapFlagPrivate) == kMapFlagPrivate) { - if (!result.empty()) { - result += " | "; - } - - result += "Private"; - flags &= ~kMapFlagPrivate; - } - if ((flags & kMapFlagFixed) == kMapFlagFixed) { - if (!result.empty()) { - result += " | "; - } - - result += "Fixed"; - flags &= ~kMapFlagFixed; - } - if ((flags & kMapFlagRename) == kMapFlagRename) { - if (!result.empty()) { - result += " | "; - } - - result += "Rename"; - flags &= ~kMapFlagRename; - } - if ((flags & kMapFlagNoReserve) == kMapFlagNoReserve) { - if (!result.empty()) { - result += " | "; - } - - result += "NoReserve"; - flags &= ~kMapFlagNoReserve; - } - if ((flags & kMapFlagNoOverwrite) == kMapFlagNoOverwrite) { - if (!result.empty()) { - result += " | "; - } - - result += "NoOverwrite"; - flags &= ~kMapFlagNoOverwrite; - } - if ((flags & kMapFlagVoid) == kMapFlagVoid) { - if (!result.empty()) { - result += " | "; - } - - result += "Void"; - flags &= ~kMapFlagVoid; - } - if ((flags & kMapFlagHasSemaphore) == kMapFlagHasSemaphore) { - if (!result.empty()) { - result += " | "; - } - - result += "HasSemaphore"; - flags &= ~kMapFlagHasSemaphore; - } - if ((flags & kMapFlagStack) == kMapFlagStack) { - if (!result.empty()) { - result += " | "; - } - - result += "Stack"; - flags &= ~kMapFlagStack; - } - if ((flags & kMapFlagNoSync) == kMapFlagNoSync) { - if (!result.empty()) { - result += " | "; - } - - result += "NoSync"; - flags &= ~kMapFlagNoSync; - } - if ((flags & kMapFlagAnonymous) == kMapFlagAnonymous) { - if (!result.empty()) { - result += " | "; - } - - result += "Anonymous"; - flags &= ~kMapFlagAnonymous; - } - if ((flags & kMapFlagSystem) == kMapFlagSystem) { - if (!result.empty()) { - result += " | "; - } - - result += "System"; - flags &= ~kMapFlagSystem; - } - if ((flags & kMapFlagAllAvailable) == kMapFlagAllAvailable) { - if (!result.empty()) { - result += " | "; - } - - result += "AllAvailable"; - flags &= ~kMapFlagAllAvailable; - } - if ((flags & kMapFlagNoCore) == kMapFlagNoCore) { - if (!result.empty()) { - result += " | "; - } - - result += "NoCore"; - flags &= ~kMapFlagNoCore; - } - if ((flags & kMapFlagPrefaultRead) == kMapFlagPrefaultRead) { - if (!result.empty()) { - result += " | "; - } - - result += "PrefaultRead"; - flags &= ~kMapFlagPrefaultRead; - } - if ((flags & kMapFlagSelf) == kMapFlagSelf) { - if (!result.empty()) { - result += " | "; - } - - result += "Self"; - flags &= ~kMapFlagSelf; - } - - auto alignment = (flags & kMapFlagsAlignMask) >> kMapFlagsAlignShift; - flags &= ~kMapFlagsAlignMask; - - if (alignment != 0) { - if (!result.empty()) { - result += " | "; - } - - result += "Alignment(" + std::to_string(alignment) + ")"; - } - - if (flags != 0) { - if (!result.empty()) { - result += " | "; - } - - result += std::to_string(flags); - } - - return result; -} - -std::string vm::mapProtToString(std::int32_t prot) { - std::string result; - - if ((prot & kMapProtCpuRead) == kMapProtCpuRead) { - if (!result.empty()) { - result += " | "; - } - result += "CpuRead"; - prot &= ~kMapProtCpuRead; - } - if ((prot & kMapProtCpuWrite) == kMapProtCpuWrite) { - if (!result.empty()) { - result += " | "; - } - result += "CpuWrite"; - prot &= ~kMapProtCpuWrite; - } - if ((prot & kMapProtCpuExec) == kMapProtCpuExec) { - if (!result.empty()) { - result += " | "; - } - result += "CpuExec"; - prot &= ~kMapProtCpuExec; - } - if ((prot & kMapProtGpuRead) == kMapProtGpuRead) { - if (!result.empty()) { - result += " | "; - } - result += "GpuRead"; - prot &= ~kMapProtGpuRead; - } - if ((prot & kMapProtGpuWrite) == kMapProtGpuWrite) { - if (!result.empty()) { - result += " | "; - } - result += "GpuWrite"; - prot &= ~kMapProtGpuWrite; - } - - if (prot != 0) { - if (!result.empty()) { - result += " | "; - } - - result += std::to_string(prot); - } - - return result; -} - -static constexpr std::uint64_t kPageMask = vm::kPageSize - 1; -static constexpr std::uint64_t kBlockShift = 32; -static constexpr std::uint64_t kBlockSize = static_cast(1) - << kBlockShift; -static constexpr std::uint64_t kBlockMask = kBlockSize - 1; -static constexpr std::uint64_t kPagesInBlock = kBlockSize / vm::kPageSize; -static constexpr std::uint64_t kFirstBlock = 0x00; -static constexpr std::uint64_t kLastBlock = 0xff; -static constexpr std::uint64_t kBlockCount = kLastBlock - kFirstBlock + 1; -static constexpr std::uint64_t kGroupSize = 64; -static constexpr std::uint64_t kGroupMask = kGroupSize - 1; -static constexpr std::uint64_t kGroupsInBlock = kPagesInBlock / kGroupSize; -static constexpr std::uint64_t kMinAddress = orbis::kMinAddress; -static constexpr std::uint64_t kMaxAddress = orbis::kMaxAddress; -static constexpr std::uint64_t kMemorySize = kBlockCount * kBlockSize; - -static_assert((kLastBlock + 1) * kBlockSize == orbis::kMaxAddress); - -static int gMemoryShm = -1; - -struct Group { - std::uint64_t allocated; - std::uint64_t shared; - std::uint64_t readable; - std::uint64_t writable; - std::uint64_t executable; - std::uint64_t gpuReadable; - std::uint64_t gpuWritable; -}; - -enum { - kReadable = vm::kMapProtCpuRead, - kWritable = vm::kMapProtCpuWrite, - kExecutable = vm::kMapProtCpuExec, - kGpuReadable = vm::kMapProtGpuRead, - kGpuWritable = vm::kMapProtGpuWrite, - - kAllocated = 1 << 3, - kShared = 1 << 6, -}; - -inline constexpr std::uint64_t makePagesMask(std::uint64_t page, - std::uint64_t count) { - if (count == 64) { - return ~0ull << page; - } - - return ((1ull << count) - 1ull) << page; -} -struct Block { - Group groups[kGroupsInBlock]; - - void setFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t flags, bool noOverwrite) { - modifyFlags(firstPage, pagesCount, flags, ~static_cast(0), - noOverwrite); - } - - void addFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t flags) { - modifyFlags(firstPage, pagesCount, flags, 0); - } - - void removeFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t flags) { - modifyFlags(firstPage, pagesCount, 0, flags); - } - - unsigned getProtection(std::uint64_t page) const { - std::uint64_t groupIndex = page / kGroupSize; - auto mask = makePagesMask(page & kGroupMask, 1); - auto &group = groups[groupIndex]; - - if ((group.allocated & mask) == 0) { - return 0; - } - - unsigned result = 0; - - result |= (group.readable & mask) == mask ? kReadable : 0; - result |= (group.writable & mask) == mask ? kReadable | kWritable : 0; - - result |= (group.executable & mask) == mask ? kReadable | kExecutable : 0; - - result |= (group.gpuReadable & mask) == mask ? kGpuReadable : 0; - result |= (group.gpuWritable & mask) == mask ? kGpuWritable : 0; - result |= (group.shared & mask) == mask ? kShared : 0; - - return result; - } - - void modifyFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t addFlags, std::uint32_t removeFlags, - bool noOverwrite = false) { - std::uint64_t groupIndex = firstPage / kGroupSize; - - std::uint64_t addAllocatedFlags = - (addFlags & kAllocated) ? ~static_cast(0) : 0; - std::uint64_t addSharedFlags = - (addFlags & kShared) ? ~static_cast(0) : 0; - std::uint64_t addReadableFlags = - (addFlags & kReadable) ? ~static_cast(0) : 0; - std::uint64_t addWritableFlags = - (addFlags & kWritable) ? ~static_cast(0) : 0; - std::uint64_t addExecutableFlags = - (addFlags & kExecutable) ? ~static_cast(0) : 0; - std::uint64_t addGpuReadableFlags = - (addFlags & kGpuReadable) ? ~static_cast(0) : 0; - std::uint64_t addGpuWritableFlags = - (addFlags & kGpuWritable) ? ~static_cast(0) : 0; - - std::uint64_t removeAllocatedFlags = - (removeFlags & kAllocated) ? ~static_cast(0) : 0; - std::uint64_t removeSharedFlags = - (removeFlags & kShared) ? ~static_cast(0) : 0; - std::uint64_t removeReadableFlags = - (removeFlags & kReadable) ? ~static_cast(0) : 0; - std::uint64_t removeWritableFlags = - (removeFlags & kWritable) ? ~static_cast(0) : 0; - std::uint64_t removeExecutableFlags = - (removeFlags & kExecutable) ? ~static_cast(0) : 0; - std::uint64_t removeGpuReadableFlags = - (removeFlags & kGpuReadable) ? ~static_cast(0) : 0; - std::uint64_t removeGpuWritableFlags = - (removeFlags & kGpuWritable) ? ~static_cast(0) : 0; - - if ((firstPage & kGroupMask) != 0) { - auto count = kGroupSize - (firstPage & kGroupMask); - - if (count > pagesCount) { - count = pagesCount; - } - - auto mask = makePagesMask(firstPage & kGroupMask, count); - pagesCount -= count; - - auto &group = groups[groupIndex++]; - - if (noOverwrite) { - mask &= ~group.allocated; - } - - group.allocated = (group.allocated & ~(removeAllocatedFlags & mask)) | - (addAllocatedFlags & mask); - group.shared = (group.shared & ~(removeSharedFlags & mask)) | - (addSharedFlags & mask); - group.readable = (group.readable & ~(removeReadableFlags & mask)) | - (addReadableFlags & mask); - group.writable = (group.writable & ~(removeWritableFlags & mask)) | - (addWritableFlags & mask); - group.executable = (group.executable & ~(removeExecutableFlags & mask)) | - (addExecutableFlags & mask); - group.gpuReadable = - (group.gpuReadable & ~(removeGpuReadableFlags & mask)) | - (addGpuReadableFlags & mask); - group.gpuWritable = - (group.gpuWritable & ~(removeGpuWritableFlags & mask)) | - (addGpuWritableFlags & mask); - } - - if (noOverwrite) { - while (pagesCount >= kGroupSize) { - pagesCount -= kGroupSize; - - auto &group = groups[groupIndex++]; - auto mask = ~group.allocated; - - group.allocated = (group.allocated & ~(removeAllocatedFlags & mask)) | - (addAllocatedFlags & mask); - group.shared = (group.shared & ~(removeSharedFlags & mask)) | - (addSharedFlags & mask); - group.readable = (group.readable & ~(removeReadableFlags & mask)) | - (addReadableFlags & mask); - group.writable = (group.writable & ~(removeWritableFlags & mask)) | - (addWritableFlags & mask); - group.executable = - (group.executable & ~(removeExecutableFlags & mask)) | - (addExecutableFlags & mask); - group.gpuReadable = - (group.gpuReadable & ~(removeGpuReadableFlags & mask)) | - (addGpuReadableFlags & mask); - group.gpuWritable = - (group.gpuWritable & ~(removeGpuWritableFlags & mask)) | - (addGpuWritableFlags & mask); - } - } else { - while (pagesCount >= kGroupSize) { - pagesCount -= kGroupSize; - - auto &group = groups[groupIndex++]; - - group.allocated = - (group.allocated & ~removeAllocatedFlags) | addAllocatedFlags; - group.shared = (group.shared & ~removeSharedFlags) | addSharedFlags; - group.readable = - (group.readable & ~removeReadableFlags) | addReadableFlags; - group.writable = - (group.writable & ~removeWritableFlags) | addWritableFlags; - group.executable = - (group.executable & ~removeExecutableFlags) | addExecutableFlags; - group.gpuReadable = - (group.gpuReadable & ~removeGpuReadableFlags) | addGpuReadableFlags; - group.gpuWritable = - (group.gpuWritable & ~removeGpuWritableFlags) | addGpuWritableFlags; - } - } - - if (pagesCount > 0) { - auto mask = makePagesMask(0, pagesCount); - auto &group = groups[groupIndex++]; - - if (noOverwrite) { - mask &= ~group.allocated; - } - - group.allocated = (group.allocated & ~(removeAllocatedFlags & mask)) | - (addAllocatedFlags & mask); - group.shared = (group.shared & ~(removeSharedFlags & mask)) | - (addSharedFlags & mask); - group.readable = (group.readable & ~(removeReadableFlags & mask)) | - (addReadableFlags & mask); - group.writable = (group.writable & ~(removeWritableFlags & mask)) | - (addWritableFlags & mask); - group.executable = (group.executable & ~(removeExecutableFlags & mask)) | - (addExecutableFlags & mask); - group.gpuReadable = - (group.gpuReadable & ~(removeGpuReadableFlags & mask)) | - (addGpuReadableFlags & mask); - group.gpuWritable = - (group.gpuWritable & ~(removeGpuWritableFlags & mask)) | - (addGpuWritableFlags & mask); - } - } - - bool isFreePages(std::uint64_t page, std::uint64_t count) { - auto groupIndex = page / kGroupSize; - - std::uint64_t foundCount = 0; - - { - auto pageInGroup = page % kGroupSize; - auto allocatedBits = groups[groupIndex].allocated; - auto freePages = std::countr_zero(allocatedBits >> pageInGroup); - - if (freePages < count && freePages + pageInGroup < kGroupSize) { - return false; - } - - foundCount += freePages; - } - - for (++groupIndex; groupIndex < kGroupsInBlock && foundCount < count; - ++groupIndex) { - auto allocatedBits = groups[groupIndex].allocated; - auto freePages = std::countr_zero(allocatedBits); - foundCount += freePages; - - if (freePages != kGroupSize) { - break; - } - } - - return foundCount >= count; - } - - [[nodiscard]] bool isFree() const { - for (auto &group : groups) { - if (group.allocated) { - return false; - } - } - - return true; - } - - std::uint64_t findFreePages(std::uint64_t count, std::uint64_t alignment) { - std::uint64_t foundCount = 0; - std::uint64_t foundPage = 0; - - if (alignment < kGroupSize * vm::kPageSize) { - std::uint64_t groupAlignment = alignment >> vm::kPageShift; - - for (std::uint64_t groupIndex = 0; - groupIndex < kGroupsInBlock && foundCount < count; ++groupIndex) { - auto allocatedBits = groups[groupIndex].allocated; - - if (foundCount != 0) { - // we already found block with free pages at the end - if (count - foundCount >= kGroupSize) { - // we need whole group. if it not empty, we need to try next range - if (allocatedBits != 0) { - foundCount = 0; - } else { - foundCount += kGroupSize; - } - } else { - if (allocatedBits == 0) { - // whole group is clear, fast path - foundCount += kGroupSize; - break; - } else { - // add free pages from beginning of the current group - foundCount += std::countr_zero(allocatedBits); - - if (foundCount >= count) { - break; - } - - // not enough free pages, need to try next range - foundCount = 0; - } - } - } - - if (foundCount == 0) { - if (~allocatedBits == 0) { - continue; - } - - if (count < kGroupSize) { - // For small allocations try to find free room from beggining of - // group - auto tmpAllocatedBits = allocatedBits; - std::uint64_t processedPages = 0; - - while (processedPages < kGroupSize) { - auto freeCount = std::countr_zero(tmpAllocatedBits); - if (freeCount + processedPages > kGroupSize) { - freeCount = kGroupSize - processedPages; - } - - processedPages += freeCount; - if (freeCount >= 64) { - tmpAllocatedBits = 0; - } else { - tmpAllocatedBits >>= freeCount; - } - - if (freeCount >= count || - (freeCount > 0 && processedPages >= kGroupSize)) { - foundPage = - groupIndex * kGroupSize + processedPages - freeCount; - foundCount = freeCount; - break; - } - - while (auto usedCount = std::countr_one(tmpAllocatedBits)) { - auto nextProcessedPages = - rx::alignUp(processedPages + usedCount, groupAlignment); - if (nextProcessedPages - processedPages >= 64) { - tmpAllocatedBits = 0; - } else { - tmpAllocatedBits >>= nextProcessedPages - processedPages; - } - processedPages = nextProcessedPages; - } - } - } else { - // this is big allocation, count free last pages in block, continue - // searching on next iterations - auto freeCount = std::countl_zero(allocatedBits); - auto alignedPageIndex = - rx::alignUp(kGroupSize - freeCount, groupAlignment); - freeCount = - kGroupSize - alignedPageIndex; // calc aligned free pages - - foundCount = freeCount; - foundPage = groupIndex * kGroupSize + alignedPageIndex; - } - } - } - } else { - std::uint64_t blockAlignment = alignment / (kGroupSize * vm::kPageSize); - - for (std::uint64_t groupIndex = 0; - groupIndex < kGroupsInBlock && foundCount < count; ++groupIndex) { - if (foundCount == 0) { - groupIndex = rx::alignUp(groupIndex, blockAlignment); - - if (groupIndex >= kGroupsInBlock) { - break; - } - } - - auto allocatedBits = groups[groupIndex].allocated; - - if (allocatedBits == 0) { - if (foundCount == 0) { - foundPage = groupIndex * kGroupSize; - } - - foundCount += kGroupSize; - } else { - if (foundCount == 0 && count < kGroupSize) { - auto freeCount = std::countr_zero(allocatedBits); - - if (freeCount >= count) { - foundPage = groupIndex * kGroupSize; - foundCount = freeCount; - break; - } - } - - foundCount = 0; - } - } - } - - if (foundCount >= count) { - assert(((foundPage << vm::kPageShift) & (alignment - 1)) == 0); - return foundPage; - } - - return ~static_cast(0); - } -}; - -static Block gBlocks[kBlockCount]; - -struct MapInfo { - rx::Ref device; - std::uint64_t offset; - std::uint32_t flags; - char name[32]; - - bool operator==(const MapInfo &) const = default; -}; - -static rx::MemoryTableWithPayload gMapInfo; - -static void reserve(std::uint64_t startAddress, std::uint64_t endAddress) { - auto blockIndex = startAddress >> kBlockShift; - - assert(endAddress > startAddress); - assert(blockIndex == (endAddress >> kBlockShift)); - - auto firstPage = (startAddress & kBlockMask) >> vm::kPageShift; - auto pagesCount = - (endAddress - startAddress + (vm::kPageSize - 1)) >> vm::kPageShift; - - gBlocks[blockIndex - kFirstBlock].setFlags(firstPage, pagesCount, kAllocated, - false); -} - -void vm::fork(std::uint64_t pid) { - auto shmPath = rx::format("{}/memory-{}", rx::getShmPath(), pid); - gMemoryShm = - ::open(shmPath.c_str(), O_CREAT | O_RDWR | O_TRUNC, S_IRUSR | S_IWUSR); - - (void)g_mtx.try_lock(); - g_mtx.unlock(); // release mutex - - if (gMemoryShm == -1) { - rx::println(stderr, "Memory: failed to open {}", shmPath); - std::abort(); - } - - if (::ftruncate64(gMemoryShm, kMemorySize) < 0) { - std::println(stderr, "Memory: failed to allocate {}", shmPath); - std::abort(); - } - - for (auto address = kMinAddress; address < kMaxAddress; - address += kPageSize) { - auto prot = gBlocks[(address >> kBlockShift) - kFirstBlock].getProtection( - (address & kBlockMask) >> vm::kPageShift); - - if (prot & kShared) { - continue; - } - - if (prot & kMapProtCpuAll) { - auto mapping = ::mmap(nullptr, kPageSize, PROT_WRITE, MAP_SHARED, - gMemoryShm, address - kMinAddress); - assert(mapping != MAP_FAILED); - - rx::mem::protect(rx::AddressRange::fromBeginSize(address, kPageSize), - rx::mem::Protection::R); - std::memcpy(mapping, reinterpret_cast(address), kPageSize); - ::munmap(mapping, kPageSize); - ::munmap(reinterpret_cast(address), kPageSize); - - mapping = ::mmap(reinterpret_cast(address), kPageSize, - prot & kMapProtCpuAll, MAP_FIXED | MAP_SHARED, - gMemoryShm, address - kMinAddress); - assert(mapping != MAP_FAILED); - } - - // TODO: copy gpu memory? - } -} - -void vm::reset() { - std::memset(gBlocks, 0, sizeof(gBlocks)); - - ::munmap(reinterpret_cast(kMinAddress), kMaxAddress - kMinAddress); - if (::ftruncate64(gMemoryShm, 0) < 0) { - std::abort(); - } - if (::ftruncate64(gMemoryShm, kMemorySize) < 0) { - std::abort(); - } - - reserve(0, kMinAddress); - rx::mem::reserve( - rx::AddressRange::fromBeginSize(kMinAddress, kMaxAddress - kMinAddress)); -} - -void vm::initialize(std::uint64_t pid) { - std::println("Memory: initialization"); - auto shmPath = rx::format("{}/memory-{}", rx::getShmPath(), pid); - - gMemoryShm = - ::open(shmPath.c_str(), O_CREAT | O_RDWR | O_TRUNC, S_IRUSR | S_IWUSR); - - if (gMemoryShm == -1) { - std::println(stderr, "Memory: failed to open {}", shmPath); - std::abort(); - } - - if (::ftruncate64(gMemoryShm, kMemorySize) < 0) { - std::println(stderr, "Memory: failed to allocate {}", shmPath); - std::abort(); - } - - reserve(0, kMinAddress); // unmapped area - - rx::mem::reserve(rx::AddressRange::fromBeginEnd(kMinAddress, kMaxAddress)); -} - -void vm::deinitialize() { - std::println("Memory: shutdown"); - ::close(gMemoryShm); - gMemoryShm = -1; - - for (auto &block : gBlocks) { - block = {}; - } -} - -constexpr auto kPhysicalMemorySize = 5568ull * 1024 * 1024; -constexpr auto kFlexibleMemorySize = 448ull * 1024 * 1024; -constexpr auto kMainDirectMemorySize = - kPhysicalMemorySize - kFlexibleMemorySize; - -void *vm::map(void *addr, std::uint64_t len, std::int32_t prot, - std::int32_t flags, std::int32_t internalFlags, - orbis::IoDevice *device, std::uint64_t offset) { - std::println(stderr, "vm::map(addr = {}, len = {}, prot = {}, flags = {})", - addr, len, mapProtToString(prot), mapFlagsToString(flags)); - - len = rx::alignUp(len, kPageSize); - auto pagesCount = (len + (kPageSize - 1)) >> kPageShift; - auto hitAddress = reinterpret_cast(addr); - - std::uint64_t alignment = (flags & kMapFlagsAlignMask) >> kMapFlagsAlignShift; - if (alignment == 0) { - alignment = kPageSize; - } else { - alignment = static_cast(1) << alignment; - } - - if (alignment < kPageSize) { - std::println(stderr, "Memory error: wrong alignment {}", alignment); - alignment = kPageSize; - } - - flags &= ~kMapFlagsAlignMask; - - bool noOverwrite = (flags & (kMapFlagNoOverwrite | kMapFlagFixed)) == - (kMapFlagNoOverwrite | kMapFlagFixed); - - if (hitAddress & (alignment - 1)) { - if (flags & kMapFlagStack) { - hitAddress = rx::alignDown(hitAddress - 1, alignment); - flags |= kMapFlagFixed; - flags &= ~kMapFlagStack; - } else { - hitAddress = rx::alignUp(hitAddress, alignment); - } - } - - std::lock_guard lock(g_mtx); - - std::uint64_t address = 0; - if ((flags & kMapFlagFixed) == kMapFlagFixed) { - address = hitAddress; - - auto blockIndex = address >> kBlockShift; - - if (blockIndex < kFirstBlock || blockIndex > kLastBlock) { - std::println(stderr, - "Memory error: fixed mapping with wrong address {:x} pages", - address); - return MAP_FAILED; - } - } else if (hitAddress != 0) { - auto blockIndex = hitAddress >> kBlockShift; - auto page = (hitAddress & kBlockMask) >> kPageShift; - - if (blockIndex < kFirstBlock || blockIndex > kLastBlock) { - std::println(stderr, "Memory error: wrong hit address {:x} pages", - hitAddress); - hitAddress = 0; - } else { - blockIndex -= kFirstBlock; - - if (gBlocks[blockIndex].isFreePages(page, pagesCount)) { - address = hitAddress; - } - } - } - - static constexpr auto kBadAddress = ~static_cast(0); - - std::size_t mapBlockCount = (len + kBlockSize - 1) / kBlockSize; - - auto isFreeBlockRange = [&](std::uint64_t firstBlock, - std::size_t blockCount) { - for (auto blockIndex = firstBlock; blockIndex < firstBlock + blockCount; - ++blockIndex) { - if (!gBlocks[blockIndex].isFree()) { - return false; - } - } - - return true; - }; - - if (address == 0 && hitAddress != 0) { - auto hitBlockIndex = hitAddress >> kBlockShift; - if (mapBlockCount > 1) { - for (auto blockIndex = hitBlockIndex; - blockIndex <= kLastBlock - mapBlockCount + 1; ++blockIndex) { - if (isFreeBlockRange(blockIndex, mapBlockCount)) { - address = blockIndex * kBlockSize; - break; - } - } - } else { - for (auto blockIndex = hitBlockIndex; blockIndex <= kLastBlock; - ++blockIndex) { - auto pageAddress = gBlocks[blockIndex - kFirstBlock].findFreePages( - pagesCount, alignment); - - if (pageAddress != kBadAddress) { - address = (pageAddress << kPageShift) | (blockIndex * kBlockSize); - break; - } - } - } - } - - if (address == 0) { - if (mapBlockCount > 1) { - for (auto blockIndex = kFirstBlock; - blockIndex <= kLastBlock - mapBlockCount + 1; ++blockIndex) { - if (isFreeBlockRange(blockIndex, mapBlockCount)) { - address = blockIndex * kBlockSize; - break; - } - } - } else { - for (auto blockIndex = kFirstBlock; blockIndex <= kLastBlock; - ++blockIndex) { - auto pageAddress = gBlocks[blockIndex - kFirstBlock].findFreePages( - pagesCount, alignment); - - if (pageAddress != kBadAddress) { - address = (pageAddress << kPageShift) | (blockIndex * kBlockSize); - break; - } - } - } - } - - if (address == 0) { - std::println(stderr, - "Memory error: no free memory left for mapping of {} pages", - pagesCount); - return MAP_FAILED; - } - - if (address & (alignment - 1)) { - std::println(stderr, "Memory error: failed to map aligned address"); - std::abort(); - } - - if (address >= kMaxAddress || address > kMaxAddress - len) { - std::println(stderr, "Memory error: out of memory"); - std::abort(); - } - - int realFlags = MAP_FIXED | MAP_SHARED; - bool isAnon = (flags & kMapFlagAnonymous) == kMapFlagAnonymous; - bool isShared = (flags & kMapFlagShared) == kMapFlagShared; - flags &= ~(kMapFlagFixed | kMapFlagAnonymous | kMapFlagShared); - - auto &block = gBlocks[(address >> kBlockShift) - kFirstBlock]; - if (noOverwrite) { - auto firstPage = (address & kBlockMask) >> kPageShift; - for (std::size_t i = 0; i < pagesCount; ++i) { - if (block.getProtection(firstPage + i)) { - return (void *)-1; - } - } - } - - block.setFlags((address & kBlockMask) >> kPageShift, pagesCount, - (prot & (kMapProtCpuAll | kMapProtGpuAll)) | kAllocated | - (isShared ? kShared : 0), - false); - - /* - if (flags & kMapFlagStack) { - realFlags |= MAP_GROWSDOWN | MAP_STACK | MAP_ANONYMOUS | MAP_PRIVATE; - offset = 0; - fd = -1; - flags &= ~kMapFlagStack; - } else { - realFlags |= MAP_SHARED; - } - */ - if (flags) { - std::println(stderr, " unhandled flags 0x{:x}", flags); - } - - { - MapInfo info; - if (auto it = gMapInfo.queryArea(address); it != gMapInfo.end()) { - info = it.get(); - } - info.device = device; - info.flags = flags; - info.offset = offset; - - gMapInfo.map(rx::AddressRange::fromBeginSize(address, len), info); - } - - if (auto thr = orbis::g_currentThread) { - std::lock_guard lock(orbis::g_context->gpuDeviceMtx); - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitMapMemory(thr->tproc->pid, address, len, -1, -1, prot, - address - kMinAddress); - } - } - - if (internalFlags & kMapInternalReserveOnly) { - return reinterpret_cast(address); - } - - auto result = - ::mmap(reinterpret_cast(address), len, prot & kMapProtCpuAll, - realFlags, gMemoryShm, address - kMinAddress); - - if (result != MAP_FAILED && isAnon) { - bool needReprotect = (prot & PROT_WRITE) == 0; - if (needReprotect) { - ::mprotect(result, len, PROT_WRITE); - } - std::memset(result, 0, len); - if (needReprotect) { - ::mprotect(result, len, prot & kMapProtCpuAll); - } - } - - return result; -} - -bool vm::unmap(void *addr, std::uint64_t size) { - size = rx::alignUp(size, kPageSize); - auto pages = (size + (kPageSize - 1)) >> kPageShift; - auto address = reinterpret_cast(addr); - - if (address < kMinAddress || address >= kMaxAddress || size >= kMaxAddress || - address > kMaxAddress - size) { - std::println(stderr, "Memory error: unmap out of memory"); - return false; - } - - if ((address & kPageMask) != 0) { - std::println(stderr, "Memory error: unmap unaligned address"); - return false; - } - - if ((address >> kBlockShift) != ((address + size - 1) >> kBlockShift)) { - std::println( - stderr, - "Memory error: unmap cross block range. address 0x{:x}, size=0x{:x}", - address, size); - __builtin_trap(); - } - - std::lock_guard lock(g_mtx); - gBlocks[(address >> kBlockShift) - kFirstBlock].removeFlags( - (address & kBlockMask) >> kPageShift, pages, ~0); - if (auto thr = orbis::g_currentThread) { - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitUnmapMemory(thr->tproc->pid, address, size); - } - } else { - std::println(stderr, "ignoring unmapping {:x}-{:x}", address, - address + size); - } - gMapInfo.unmap(rx::AddressRange::fromBeginSize(address, size)); - return ::munmap(addr, size); -} - -bool vm::protect(void *addr, std::uint64_t size, std::int32_t prot) { - std::println("vm::protect(addr = {}, len = {}, prot = {})", addr, size, - mapProtToString(prot)); - - auto address = reinterpret_cast(addr); - auto endAddress = address + size; - address = rx::alignDown(address, kPageSize); - endAddress = rx::alignUp(endAddress, kPageSize); - size = endAddress - address; - auto pages = size >> kPageShift; - if (address < kMinAddress || address >= kMaxAddress || size >= kMaxAddress || - address > kMaxAddress - size) { - std::println(stderr, "Memory error: protect out of memory"); - return false; - } - - if ((address & kPageMask) != 0) { - std::println(stderr, "Memory error: protect unaligned address"); - return false; - } - - if ((address >> kBlockShift) != ((address + size - 1) >> kBlockShift)) { - std::println(stderr, "Memory error: protect cross block range"); - std::abort(); - } - - std::lock_guard lock(g_mtx); - - gBlocks[(address >> kBlockShift) - kFirstBlock].setFlags( - (address & kBlockMask) >> kPageShift, pages, - kAllocated | (prot & (kMapProtCpuAll | kMapProtGpuAll)), false); - - if (auto thr = orbis::g_currentThread) { - std::println("memory prot: {:x}", prot); - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitProtectMemory(thr->tproc->pid, address, size, prot); - } - } else if (prot >> 4) { - std::println(stderr, "ignoring mapping {:x}-{:x}", address, address + size); - } - return ::mprotect(std::bit_cast(address), size, - prot & kMapProtCpuAll) == 0; -} - -static std::int32_t getPageProtectionImpl(std::uint64_t address) { - return gBlocks[(address >> kBlockShift) - kFirstBlock].getProtection( - (address & kBlockMask) >> vm::kPageShift) & - ~kShared; -} - -bool vm::queryProtection(const void *addr, std::uint64_t *startAddress, - std::uint64_t *endAddress, std::int32_t *prot) { - auto address = reinterpret_cast(addr); - if (address < kMinAddress || address >= kMaxAddress) { - return false; - } - - std::uint64_t start = address & ~kPageMask; - std::uint64_t end = start + kPageSize; - - std::lock_guard lock(g_mtx); - - auto resultProt = getPageProtectionImpl(address); - - while (true) { - auto testAddr = start - kPageSize; - if (testAddr < kMinAddress) { - break; - } - - auto testProt = getPageProtectionImpl(testAddr); - - if (resultProt != testProt) { - break; - } - - start = testAddr; - } - - while (true) { - auto testAddr = end; - if (testAddr >= kMaxAddress) { - break; - } - - auto testProt = getPageProtectionImpl(testAddr); - - if (resultProt != testProt) { - break; - } - - end = testAddr + kPageSize; - } - - *startAddress = start; - *endAddress = end; - *prot = resultProt; - - return true; -} - -unsigned vm::getPageProtection(std::uint64_t address) { - if (address < kMinAddress || address >= kMaxAddress) { - std::println(stderr, "Memory error: getPageProtection out of memory"); - return 0; - } - - std::lock_guard lock(g_mtx); - - return getPageProtectionImpl(address); -} - -bool vm::virtualQuery(const void *addr, std::int32_t flags, - VirtualQueryInfo *info) { - std::lock_guard lock(g_mtx); - - auto address = reinterpret_cast(addr); - if (address < kMinAddress || address >= kMaxAddress) { - return false; - } - - auto it = gMapInfo.lowerBound(address); - - if (it == gMapInfo.end()) { - return false; - } - - if ((flags & 1) == 0) { - if (it.endAddress() <= address) { - return false; - } - } else { - if (it.beginAddress() > address || it.endAddress() <= address) { - return false; - } - } - - std::int32_t memoryType = 0; - std::uint32_t blockFlags = 0; - if (it->device != nullptr) { - if (auto dmem = dynamic_cast(it->device.get())) { - auto dmemIt = dmem->allocations.queryArea(it->offset); - if (dmemIt == dmem->allocations.end()) { - return false; - } - memoryType = dmemIt->memoryType; - blockFlags = kBlockFlagDirectMemory; - rx::print(stderr, "virtual query {}", addr); - rx::println(stderr, "memory type: {}", memoryType); - } - // TODO - } - - std::int32_t prot = getPageProtectionImpl(it.beginAddress()); - - *info = { - .start = it.beginAddress(), - .end = it.endAddress(), - .protection = prot, - .memoryType = memoryType, - .flags = blockFlags, - }; - - ORBIS_LOG_ERROR("virtualQuery", addr, flags, info->start, info->end, - info->protection, info->memoryType, info->flags); - - std::memcpy(info->name, it->name, sizeof(info->name)); - return true; -} - -void vm::setName(std::uint64_t start, std::uint64_t size, const char *name) { - std::lock_guard lock(g_mtx); - - MapInfo info; - if (auto it = gMapInfo.queryArea(start); it != gMapInfo.end()) { - info = it.get(); - } - - std::strncpy(info.name, name, sizeof(info.name)); - - gMapInfo.map(rx::AddressRange::fromBeginSize(start, size), info); -} diff --git a/rpcsx/vm.hpp b/rpcsx/vm.hpp deleted file mode 100644 index c2e93cf34..000000000 --- a/rpcsx/vm.hpp +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once -#include "orbis/IoDevice.hpp" -#include -#include - -namespace vm { -static constexpr std::uint64_t kPageShift = 14; -static constexpr std::uint64_t kPageSize = static_cast(1) - << kPageShift; - -enum BlockFlags { - kBlockFlagFlexibleMemory = 1 << 0, - kBlockFlagDirectMemory = 1 << 1, - kBlockFlagStack = 1 << 2, - kBlockFlagPooledMemory = 1 << 3, - kBlockFlagCommited = 1 << 4, -}; - -enum MapFlags { - kMapFlagShared = 0x1, - kMapFlagPrivate = 0x2, - kMapFlagFixed = 0x10, - kMapFlagRename = 0x20, - kMapFlagNoReserve = 0x40, - kMapFlagNoOverwrite = 0x80, - kMapFlagVoid = 0x100, - kMapFlagHasSemaphore = 0x200, - kMapFlagStack = 0x400, - kMapFlagNoSync = 0x800, - kMapFlagAnonymous = 0x1000, - kMapFlagSystem = 0x2000, - kMapFlagAllAvailable = 0x4000, - kMapFlagNoCore = 0x20000, - kMapFlagPrefaultRead = 0x40000, - kMapFlagSelf = 0x80000, -}; - -enum MapProt { - kMapProtCpuRead = 1, - kMapProtCpuWrite = 2, - kMapProtCpuReadWrite = kMapProtCpuRead | kMapProtCpuWrite, - kMapProtCpuExec = 4, - kMapProtCpuAll = 0x7, - kMapProtGpuRead = 0x10, - kMapProtGpuWrite = 0x20, - kMapProtGpuAll = 0x30, -}; - -enum MapInternalFlags { - kMapInternalReserveOnly = 1 << 0, -}; - -struct VirtualQueryInfo { - std::uint64_t start; - std::uint64_t end; - std::uint64_t offset; - std::int32_t protection; - std::int32_t memoryType; - std::uint32_t flags; - char name[32]; -}; - -static constexpr std::uint32_t kMapFlagsAlignShift = 24; -static constexpr std::uint32_t kMapFlagsAlignMask = 0x1f << kMapFlagsAlignShift; - -std::string mapFlagsToString(std::int32_t flags); -std::string mapProtToString(std::int32_t prot); - -void fork(std::uint64_t pid); -void reset(); -void initialize(std::uint64_t pid); -void deinitialize(); -void *map(void *addr, std::uint64_t len, std::int32_t prot, std::int32_t flags, - std::int32_t internalFlags = 0, orbis::IoDevice *device = nullptr, - std::uint64_t offset = 0); -bool unmap(void *addr, std::uint64_t size); -bool protect(void *addr, std::uint64_t size, std::int32_t prot); - -void setName(std::uint64_t start, std::uint64_t size, const char *name); -bool virtualQuery(const void *addr, std::int32_t flags, VirtualQueryInfo *info); -bool queryProtection(const void *addr, std::uint64_t *startAddress, - std::uint64_t *endAddress, std::int32_t *prot); -unsigned getPageProtection(std::uint64_t address); -} // namespace vm diff --git a/rx/include/rx/EnumBitSet.hpp b/rx/include/rx/EnumBitSet.hpp index 6a42eebb2..62bcbe7c5 100644 --- a/rx/include/rx/EnumBitSet.hpp +++ b/rx/include/rx/EnumBitSet.hpp @@ -20,7 +20,9 @@ Examples: Intersection (&) and symmetric difference (^) is also available. */ -#include "types.hpp" +#include "format-base.hpp" +#include "rx/refl.hpp" +#include namespace rx { template @@ -52,8 +54,8 @@ private: : m_data(data) {} public: - static constexpr usz bitmax = sizeof(T) * 8; - static constexpr usz bitsize = + static constexpr std::size_t bitmax = sizeof(T) * 8; + static constexpr std::size_t bitsize = static_cast(T::bitset_last) + 1; static_assert(std::is_enum_v, @@ -69,7 +71,7 @@ public: << static_cast(value); } - EnumBitSet() = default; + constexpr EnumBitSet() = default; // Construct from a single bit constexpr EnumBitSet(T bit) noexcept : m_data(shift(bit)) {} @@ -91,7 +93,9 @@ public: return m_data; } - constexpr detail::InvertedEnumBitSet operator~() const { return {m_data}; } + constexpr detail::InvertedEnumBitSet operator~() const { + return static_cast(~m_data); + } [[deprecated("Use operator|=")]] constexpr EnumBitSet & operator+=(EnumBitSet rhs) { @@ -126,6 +130,11 @@ public: return *this; } + constexpr EnumBitSet &operator&=(detail::InvertedEnumBitSet rhs) { + m_data &= rhs.m_data; + return *this; + } + constexpr EnumBitSet &operator^=(EnumBitSet rhs) { m_data ^= static_cast(rhs); return *this; @@ -191,7 +200,9 @@ public: return (m_data & arg.m_data) == 0; } - underlying_type &raw() { return m_data; } + constexpr underlying_type &raw() { return m_data; } + + constexpr auto operator<=>(const EnumBitSet &) const = default; }; template constexpr EnumBitSet toBitSet(T bit) { @@ -288,4 +299,50 @@ constexpr EnumBitSet operator^(const U &lhs, T rhs) { } // namespace bitset } // namespace rx +template + requires requires(rx::format_parse_context &ctx) { + rx::formatter().parse(ctx); + } +struct rx::formatter> { + constexpr rx::format_parse_context::iterator + parse(rx::format_parse_context &ctx) { + return ctx.begin(); + } + + rx::format_context::iterator format(rx::EnumBitSet bitSet, + rx::format_context &ctx) const { + auto raw = bitSet.toUnderlying(); + if (raw != 0) { + bool first = true; + for (std::size_t i = 0; i <= static_cast(T::bitset_last); + ++i) { + auto mask = 1ull << i; + if (!(raw & mask)) { + continue; + } + + if (first) { + first = false; + } else { + rx::format_to(ctx.out(), " | "); + } + + rx::format_to(ctx.out(), "{}", T(i)); + raw &= ~mask; + } + + if (raw) { + if (!first) { + rx::format_to(ctx.out(), " | "); + } + + rx::format_to(ctx.out(), "{:#x}", raw); + } + } else { + rx::format_to(ctx.out(), "{}::None", rx::getNameOf()); + } + return ctx.out(); + } +}; + using namespace rx::bitset; diff --git a/rx/include/rx/FunctionRef.hpp b/rx/include/rx/FunctionRef.hpp index 630eb1cb1..2f030f243 100644 --- a/rx/include/rx/FunctionRef.hpp +++ b/rx/include/rx/FunctionRef.hpp @@ -20,7 +20,7 @@ public: : context( const_cast> *>(&object)), invoke(+[](void *context, ArgsT... args) -> RT { - return (*reinterpret_cast(context))(std::move(args)...); + return (*reinterpret_cast(context))(args...); }) {} template diff --git a/rx/include/rx/Mappable.hpp b/rx/include/rx/Mappable.hpp index 08d02b17d..61761a723 100644 --- a/rx/include/rx/Mappable.hpp +++ b/rx/include/rx/Mappable.hpp @@ -8,6 +8,7 @@ namespace rx { class Mappable { +public: #ifdef _WIN32 using NativeHandle = void *; static constexpr NativeHandle kInvalidHandle = nullptr; @@ -16,9 +17,6 @@ class Mappable { static constexpr auto kInvalidHandle = NativeHandle(-1); #endif - NativeHandle m_handle = kInvalidHandle; - -public: Mappable() = default; Mappable(Mappable &&other) noexcept { *this = std::move(other); } Mappable(const Mappable &) = delete; @@ -33,6 +31,12 @@ public: } } + static Mappable CreateFromNativeHandle(NativeHandle handle) { + Mappable result; + result.m_handle = handle; + return result; + } + static std::pair CreateMemory(std::size_t size); static std::pair CreateSwap(std::size_t size); std::errc map(rx::AddressRange virtualRange, std::size_t offset, @@ -47,5 +51,7 @@ public: private: void destroy(); + + NativeHandle m_handle = kInvalidHandle; }; } // namespace rx diff --git a/rx/include/rx/MemoryTable.hpp b/rx/include/rx/MemoryTable.hpp index 7fefbf137..7b8a2fcb2 100644 --- a/rx/include/rx/MemoryTable.hpp +++ b/rx/include/rx/MemoryTable.hpp @@ -1,7 +1,8 @@ #pragma once -#include "rx/AddressRange.hpp" -#include "rx/Rc.hpp" +#include "AddressRange.hpp" +#include "Rc.hpp" +#include "Serializer.hpp" #include #include #include @@ -84,7 +85,7 @@ public: return {startAddress, endAddress}; } - void map(rx::AddressRange range) { + void map(AddressRange range) { auto [beginIt, beginInserted] = mAreas.emplace(range.beginAddress(), Kind::O); auto [endIt, endInserted] = mAreas.emplace(range.endAddress(), Kind::X); @@ -311,6 +312,23 @@ public: assert(kind != Kind::X); kind = Kind::O; } + + void serialize(Serializer &s) const + requires Serializable + { + s.serialize(kind); + if (kind != Kind::X) { + s.serialize(storage.data); + } + } + void deserialize(Deserializer &d) + requires Serializable + { + d.deserialize(kind); + if (kind != Kind::X && !d.failure()) { + d.deserialize(storage.data); + } + } }; template class Payload { @@ -365,6 +383,9 @@ public: assert(!isClose()); value &= ~kCloseOpenBit; } + + void serialize(Serializer &s) const { s.serialize(value); } + void deserialize(Deserializer &d) { d.deserialize(value); } }; template class Payload> { @@ -452,6 +473,9 @@ public: assert(!isClose()); value &= ~kCloseOpenBit; } + + void serialize(Serializer &s) const { s.serialize(value); } + void deserialize(Deserializer &d) { d.deserialize(value); } }; template &payload; + template class AreaInfo : public AddressRange { + T &payload; public: - AreaInfo(Payload &payload, rx::AddressRange range) + AreaInfo(T &payload, AddressRange range) : payload(payload), AddressRange(range) {} decltype(auto) operator->() { return &payload.get(); } decltype(auto) get() { return payload.get(); } }; - class iterator { - using map_iterator = - typename std::map::iterator; - map_iterator it; + template class Iterator { + MapIterator it; public: - iterator() = default; - iterator(map_iterator it) : it(it) {} + Iterator() = default; + Iterator(MapIterator it) : it(it) {} AreaInfo operator*() const { return {it->second, range()}; } - rx::AddressRange range() const { - return rx::AddressRange::fromBeginEnd(beginAddress(), endAddress()); + AddressRange range() const { + return AddressRange::fromBeginEnd(beginAddress(), endAddress()); } std::uint64_t beginAddress() const { return it->first; } @@ -495,7 +517,8 @@ public: decltype(auto) get() const { return it->second.get(); } decltype(auto) operator->() const { return &it->second.get(); } - iterator &operator++() { + + Iterator &operator++() { ++it; if (!it->second.isCloseOpen()) { @@ -505,7 +528,7 @@ public: return *this; } - iterator &operator--() { + Iterator &operator--() { --it; if (it->second.isClose()) { @@ -515,18 +538,28 @@ public: return *this; } - bool operator==(iterator other) const { return it == other.it; } - bool operator!=(iterator other) const { return it != other.it; } + bool operator==(Iterator other) const { return it == other.it; } friend MemoryTableWithPayload; }; + using iterator = + Iterator::iterator, + AreaInfo>; + using const_iterator = + Iterator::const_iterator, + AreaInfo>; + MemoryTableWithPayload() = default; MemoryTableWithPayload(MemoryTableWithPayload &&) = default; MemoryTableWithPayload &operator=(MemoryTableWithPayload &&) = default; MemoryTableWithPayload(const MemoryTableWithPayload &) = delete; MemoryTableWithPayload &operator=(const MemoryTableWithPayload &) = delete; + const_iterator cbegin() const { return const_iterator(mAreas.begin()); } + const_iterator cend() const { return const_iterator(mAreas.end()); } + const_iterator begin() const { return const_iterator(mAreas.cbegin()); } + const_iterator end() const { return const_iterator(mAreas.cend()); } iterator begin() { return iterator(mAreas.begin()); } iterator end() { return iterator(mAreas.end()); } @@ -579,9 +612,9 @@ public: return endAddress < address ? mAreas.end() : it; } - iterator map(rx::AddressRange range, PayloadT payload, bool merge = true, + iterator map(AddressRange range, PayloadT payload, bool merge = true, bool noOverride = false) { - assert(range.beginAddress() < range.endAddress()); + assert(range.isValid()); auto [beginIt, beginInserted] = mAreas.emplace(range.beginAddress(), payload_type::createOpen(payload)); auto [endIt, endInserted] = @@ -676,7 +709,7 @@ public: return origBegin; } - void unmap(iterator it) { + iterator unmap(iterator it) { auto openIt = it.it; auto closeIt = openIt; ++closeIt; @@ -690,13 +723,49 @@ public: if (closeIt->second.isCloseOpen()) { closeIt->second.setOpen(); } else { - mAreas.erase(closeIt); + closeIt = mAreas.erase(closeIt); } + + return iterator(closeIt); } - void unmap(rx::AddressRange range) { + iterator unmap(AddressRange range) { // FIXME: can be optimized - unmap(map(range, PayloadT{}, false)); + return unmap(map(range, PayloadT{}, false)); + } + + void serialize(Serializer &s) const + requires Serializable + { + for (auto block : *this) { + s.serialize(block.beginAddress()); + s.serialize(block.endAddress()); + s.serialize(block.get()); + } + + s.serialize(-1); + s.serialize(-1); + } + + void deserialize(Deserializer &d) + requires Serializable + { + clear(); + + while (!d.failure()) { + auto beginAddress = d.deserialize(); + auto endAddress = d.deserialize(); + + if (beginAddress == static_cast(-1) && + endAddress == static_cast(-1)) { + break; + } + + auto value = d.deserialize(); + + map(AddressRange::fromBeginEnd(beginAddress, endAddress), + std::move(value), false); + } } }; } // namespace rx diff --git a/rx/include/rx/Rc.hpp b/rx/include/rx/Rc.hpp index 3c791efd9..a5533469b 100644 --- a/rx/include/rx/Rc.hpp +++ b/rx/include/rx/Rc.hpp @@ -12,7 +12,7 @@ struct RcBase { virtual ~RcBase() = default; void incRef() { - if (references.fetch_add(1, std::memory_order::relaxed) > 4096) { + if (references.fetch_add(1, std::memory_order::relaxed) > 100 * 1024 * 1024) { assert(!"too many references"); } } diff --git a/rx/include/rx/Serializer.hpp b/rx/include/rx/Serializer.hpp index b9742162c..09e419679 100644 --- a/rx/include/rx/Serializer.hpp +++ b/rx/include/rx/Serializer.hpp @@ -169,19 +169,17 @@ struct Deserializer { if constexpr (requires { { type::deserialize(*this) } -> std::convertible_to; }) { - return T::deserialize(*this); + return type::deserialize(*this); } else if constexpr (requires(type &result) { type::deserialize(*this, result); }) { type result; - T::deserialize(*this, result); + type::deserialize(*this, result); return result; } else if constexpr (requires(type &result) { - { - result.deserialize(*this) - } -> std::convertible_to; + result.deserialize(*this); }) { - T result; + type result; result.deserialize(*this); return result; } else if constexpr (requires(type &result) { diff --git a/rx/include/rx/refl.hpp b/rx/include/rx/refl.hpp index 60ff2123a..9662ce79f 100644 --- a/rx/include/rx/refl.hpp +++ b/rx/include/rx/refl.hpp @@ -174,13 +174,29 @@ constexpr auto calcFieldCount() { } else if constexpr (requires { EnumT::count; }) { return static_cast(EnumT::count); } else if constexpr (!requires { getNameOf()[0]; }) { - return N; + if constexpr (requires { getNameOf()[0]; }) { + if constexpr (constexpr auto c = getNameOf()[0]; + c >= '0' && c <= '9') { + return N; + } else { + return calcFieldCount(); + } + } else { + return N; + } } else { - constexpr auto c = getNameOf()[0]; - if constexpr (!requires { getNameOf()[0]; }) { - return N; - } else if constexpr (c >= '0' && c <= '9') { - return N; + if constexpr (constexpr auto c = getNameOf()[0]; + c >= '0' && c <= '9') { + if constexpr (requires { getNameOf()[0]; }) { + if constexpr (constexpr auto c = getNameOf()[0]; + c >= '0' && c <= '9') { + return N; + } else { + return calcFieldCount(); + } + } else { + return N; + } } else { return calcFieldCount(); }