diff --git a/kernel/include/kernel/KernelObject.hpp b/kernel/include/kernel/KernelObject.hpp index 7947b0c2f..d84a02f88 100644 --- a/kernel/include/kernel/KernelObject.hpp +++ b/kernel/include/kernel/KernelObject.hpp @@ -65,6 +65,7 @@ struct LockableKernelObject : KernelObjectBase, StateT { void lock() const { mtx.lock(); } void unlock() const { mtx.unlock(); } + bool try_lock() const { return mtx.try_lock(); } }; namespace detail { @@ -175,6 +176,7 @@ private: template struct StaticObjectRef { + using type = T; std::uint32_t offset; T *get(std::byte *storage) { return reinterpret_cast(storage + offset); } diff --git a/kernel/include/kernel/MemoryResource.hpp b/kernel/include/kernel/MemoryResource.hpp index 31afe59b9..2a78eabdc 100644 --- a/kernel/include/kernel/MemoryResource.hpp +++ b/kernel/include/kernel/MemoryResource.hpp @@ -75,11 +75,13 @@ concept AllocationInfo = requires(T &t, rx::AddressRange range) { requires rx::Serializable; }; -template typename Allocator, rx::Serializable Resource = ExternalResource> struct AllocableResource : Resource { - mutable rx::MemoryTableWithPayload allocations; - using iterator = typename rx::MemoryTableWithPayload::iterator; + mutable rx::MemoryTableWithPayload allocations; + using BaseResource = AllocableResource; + using iterator = + typename rx::MemoryTableWithPayload::iterator; struct AllocationResult { iterator it; @@ -133,6 +135,10 @@ struct AllocableResource : Resource { } } + iterator lowerBound(std::uint64_t address) { + return allocations.lowerBound(address); + } + iterator query(std::uint64_t address) { return allocations.queryArea(address); } @@ -164,7 +170,14 @@ struct AllocableResource : Resource { if (flags & AllocationFlags::Fixed) { it = allocations.queryArea(addressHint); } else { - it = allocations.lowerBound(addressHint); + if (addressHint == 0 && (flags & AllocationFlags::Stack)) { + it = allocations.end(); + if (it != allocations.begin()) { + --it; + } + } else { + it = allocations.lowerBound(addressHint); + } } if (it == allocations.end()) { @@ -245,7 +258,7 @@ struct AllocableResource : Resource { } } else { auto hasEnoughSpace = [=](rx::AddressRange range) { - if (range.contains(addressHint)) { + if (addressHint != 0 && range.contains(addressHint)) { if (flags & AllocationFlags::Stack) { range = rx::AddressRange::fromBeginEnd( rx::alignDown(range.beginAddress(), alignment), addressHint); @@ -253,6 +266,11 @@ struct AllocableResource : Resource { range = rx::AddressRange::fromBeginEnd(addressHint, range.endAddress()); } + } else if (flags & AllocationFlags::Stack) { + auto alignedStackRange = rx::AddressRange::fromBeginEnd( + rx::alignDown(range.endAddress() - size, alignment), + range.endAddress()); + range = range.intersection(alignedStackRange); } auto alignedAddress = rx::AddressRange::fromBeginEnd( @@ -261,7 +279,7 @@ struct AllocableResource : Resource { return alignedAddress.isValid() && alignedAddress.size() >= size; }; - if (addressHint != 0 && (flags & AllocationFlags::Stack)) { + if (flags & AllocationFlags::Stack) { while (it != begin()) { if (!it->isAllocated() && hasEnoughSpace(it.range())) { break; @@ -288,23 +306,28 @@ struct AllocableResource : Resource { } // now `it` points to region that meets requirements, create fixed range - if (it.range().contains(addressHint)) { + if (addressHint != 0 && it.range().contains(addressHint)) { if (flags & AllocationFlags::Stack) { - fixedRange = - rx::AddressRange::fromBeginSize(rx::alignDown(addressHint - size, alignment), size); + fixedRange = rx::AddressRange::fromBeginSize( + rx::alignDown(addressHint - size, alignment), size); } else { fixedRange = rx::AddressRange::fromBeginEnd(addressHint, it.endAddress()); } } else { - fixedRange = rx::AddressRange::fromBeginEnd( - rx::alignUp(it.beginAddress(), alignment), it.endAddress()); + if (flags & AllocationFlags::Stack) { + fixedRange = rx::AddressRange::fromBeginSize( + rx::alignDown(it.endAddress() - size, alignment), + size); + } else { + fixedRange = rx::AddressRange::fromBeginSize( + rx::alignUp(it.beginAddress(), alignment), size); + } } } if (fixedRange.size() > size) { - if ((flags & AllocationFlags::Stack) && - !it.range().contains(addressHint)) { + if (flags & AllocationFlags::Stack) { fixedRange = rx::AddressRange::fromBeginSize( rx::alignDown(fixedRange.endAddress() - size, alignment), size); } else { diff --git a/kernel/orbis/CMakeLists.txt b/kernel/orbis/CMakeLists.txt index bcc80bb9c..98eab3cb4 100644 --- a/kernel/orbis/CMakeLists.txt +++ b/kernel/orbis/CMakeLists.txt @@ -1,6 +1,8 @@ set(CMAKE_POSITION_INDEPENDENT_CODE on) add_library(obj.orbis-kernel OBJECT + src/blockpool.cpp + src/dmem.cpp src/event.cpp src/evf.cpp src/fmem.cpp diff --git a/kernel/orbis/include/orbis/Budget.hpp b/kernel/orbis/include/orbis/Budget.hpp index 0bc352f06..f2a6cb72b 100644 --- a/kernel/orbis/include/orbis/Budget.hpp +++ b/kernel/orbis/include/orbis/Budget.hpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace orbis { enum class BudgetResource : uint32_t { @@ -139,4 +140,44 @@ private: BudgetList mList; char mName[32]{}; }; + +class ScopedBudgetAcquire { + Budget *mBudget = nullptr; + std::uint64_t mSize = 0; + BudgetResource mResourceId = {}; + +public: + ScopedBudgetAcquire() = default; + + ScopedBudgetAcquire(const ScopedBudgetAcquire &) = delete; + ScopedBudgetAcquire &operator=(const ScopedBudgetAcquire &) = delete; + ScopedBudgetAcquire(Budget *budget, BudgetResource resourceId, + std::uint64_t size) + : mBudget(budget), mSize(size), mResourceId(resourceId) { + if (!budget->acquire(resourceId, size)) { + mBudget = nullptr; + } + } + + ScopedBudgetAcquire(ScopedBudgetAcquire &&other) noexcept { + *this = std::move(other); + } + + ScopedBudgetAcquire &operator=(ScopedBudgetAcquire &&other) noexcept { + std::swap(mBudget, other.mBudget); + std::swap(mSize, other.mSize); + std::swap(mResourceId, other.mResourceId); + return *this; + } + + ~ScopedBudgetAcquire() { + if (mBudget) { + mBudget->release(mResourceId, mSize); + } + } + + explicit operator bool() const { return mBudget != nullptr; } + + void commit() { mBudget = nullptr; } +}; } // namespace orbis diff --git a/kernel/orbis/include/orbis/IoDevice.hpp b/kernel/orbis/include/orbis/IoDevice.hpp index 114bdbf6b..b2571f3f9 100644 --- a/kernel/orbis/include/orbis/IoDevice.hpp +++ b/kernel/orbis/include/orbis/IoDevice.hpp @@ -6,6 +6,7 @@ #include "rx/EnumBitSet.hpp" #include "rx/Rc.hpp" #include "rx/mem.hpp" +#include "vmem.hpp" #include #include @@ -34,6 +35,8 @@ struct Process; struct Stat; struct StatFs; struct IoDevice : rx::RcBase { + rx::EnumBitSet blockFlags{}; + virtual ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, Thread *thread) = 0; @@ -67,14 +70,12 @@ struct IoDevice : rx::RcBase { return ErrorCode::NOTSUP; } - virtual ErrorCode ioctl(std::uint64_t request, orbis::ptr argp, + virtual ErrorCode ioctl(std::uint64_t request, ptr argp, Thread *thread); virtual ErrorCode map(rx::AddressRange range, std::int64_t offset, - rx::EnumBitSet protection, - Process *process) { - return ErrorCode::NOTSUP; - } + rx::EnumBitSet protection, File *file, + Process *process); }; namespace ioctl { diff --git a/kernel/orbis/include/orbis/KernelContext.hpp b/kernel/orbis/include/orbis/KernelContext.hpp index 7ecd665db..4e4334601 100644 --- a/kernel/orbis/include/orbis/KernelContext.hpp +++ b/kernel/orbis/include/orbis/KernelContext.hpp @@ -122,8 +122,8 @@ public: rx::Ref deviceEventEmitter; rx::Ref shmDevice; - rx::Ref dmemDevice; - rx::Ref blockpoolDevice; + rx::Ref dmem; + rx::Ref blockpool; rx::Ref gpuDevice; rx::Ref dceDevice; rx::shared_mutex gpuDeviceMtx; diff --git a/kernel/orbis/include/orbis/MemoryType.hpp b/kernel/orbis/include/orbis/MemoryType.hpp new file mode 100644 index 000000000..fae3dd018 --- /dev/null +++ b/kernel/orbis/include/orbis/MemoryType.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace orbis { +enum class MemoryType : std::uint32_t { + Invalid = -1u, + WbOnion = 0, // write back, CPU bus + WCGarlic = 3, // combining, GPU bus + WbGarlic = 10, // write back, GPU bus +}; +} diff --git a/kernel/orbis/include/orbis/blockpool.hpp b/kernel/orbis/include/orbis/blockpool.hpp new file mode 100644 index 000000000..5bdaefc02 --- /dev/null +++ b/kernel/orbis/include/orbis/blockpool.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "dmem.hpp" +#include "error/ErrorCode.hpp" +#include "orbis-config.hpp" +#include "rx/AddressRange.hpp" + +namespace orbis { +struct Process; +} + +namespace orbis::blockpool { +#pragma pack(push, 1) +struct BlockStats { + sint availFlushedBlocks; + sint availCachedBlocks; + sint commitFlushedBlocks; + sint commitCachedBlocks; +}; +static_assert(sizeof(BlockStats) == 16); +#pragma pack(pop) + +void clear(); +ErrorCode expand(rx::AddressRange dmemRange); +ErrorCode allocateControlBlock(); +ErrorCode releaseControlBlock(); +ErrorCode commit(Process *process, rx::AddressRange vmemRange, MemoryType type, + rx::EnumBitSet protection); +void decommit(Process *process, rx::AddressRange vmemRange); +std::optional getType(std::uint64_t address); +BlockStats stats(); +} // namespace orbis::blockpool diff --git a/kernel/orbis/include/orbis/dmem.hpp b/kernel/orbis/include/orbis/dmem.hpp new file mode 100644 index 000000000..3dd91c639 --- /dev/null +++ b/kernel/orbis/include/orbis/dmem.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include "MemoryType.hpp" +#include "error/ErrorCode.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" +#include "vmem.hpp" +#include + +namespace orbis { +struct Process; +struct IoDevice; +} // namespace orbis + +namespace orbis::dmem { +static constexpr auto kPageSize = 64 * 1024; + +enum class QueryFlags { + LowerBound, + Pooled, + + bitset_last = Pooled, +}; + +struct QueryResult { + rx::AddressRange range; + MemoryType memoryType; +}; + +ErrorCode initialize(); +ErrorCode clear(unsigned dmemIndex); + +std::pair +allocate(unsigned dmemIndex, rx::AddressRange searchRange, std::uint64_t len, + MemoryType memoryType, std::uint64_t alignment = kPageSize, + bool pooled = false); + +ErrorCode release(unsigned dmemIndex, rx::AddressRange range, + bool pooled = false); + +std::pair reserveSystem(unsigned dmemIndex, + std::uint64_t size); +std::pair reservePooled(unsigned dmemIndex, + rx::AddressRange searchRange, + std::uint64_t size, + std::uint64_t alignment); + +ErrorCode setType(unsigned dmemIndex, rx::AddressRange range, MemoryType type, + rx::EnumBitSet flags = {}); +std::optional query(unsigned dmemIndex, std::uint64_t dmemOffset, + rx::EnumBitSet flags = {}); +std::uint64_t getSize(unsigned dmemIndex); +std::pair +getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t alignment); + +ErrorCode map(unsigned dmemIndex, rx::AddressRange range, std::uint64_t offset, + rx::EnumBitSet protection); + +std::pair getPmemOffset(unsigned dmemIndex, + std::uint64_t dmemOffset); +} // namespace orbis::dmem diff --git a/kernel/orbis/include/orbis/file.hpp b/kernel/orbis/include/orbis/file.hpp index 1be2edb5e..a6639bb13 100644 --- a/kernel/orbis/include/orbis/file.hpp +++ b/kernel/orbis/include/orbis/file.hpp @@ -4,6 +4,7 @@ #include "KernelAllocator.hpp" #include "error/ErrorCode.hpp" #include "note.hpp" +#include "rx/Mappable.hpp" #include "rx/Rc.hpp" #include "rx/SharedMutex.hpp" #include "stat.hpp" @@ -42,12 +43,6 @@ struct FileOps { // TODO: chown // TODO: chmod - ErrorCode (*mmap)(File *file, void **address, std::uint64_t size, - std::int32_t prot, std::int32_t flags, std::int64_t offset, - Thread *thread) = nullptr; - ErrorCode (*munmap)(File *file, void **address, std::uint64_t size, - Thread *thread) = nullptr; - ErrorCode (*bind)(orbis::File *file, SocketAddress *address, std::size_t addressLen, Thread *thread) = nullptr; ErrorCode (*listen)(orbis::File *file, int backlog, Thread *thread) = nullptr; @@ -84,7 +79,7 @@ struct File : rx::RcBase { std::uint64_t nextOff = 0; int flags = 0; int mode = 0; - int hostFd = -1; + rx::Mappable hostFd; kvector dirEntries; bool noBlock() const { return (flags & 4) != 0; } diff --git a/kernel/orbis/include/orbis/fmem.hpp b/kernel/orbis/include/orbis/fmem.hpp index b6edf1206..10f37f306 100644 --- a/kernel/orbis/include/orbis/fmem.hpp +++ b/kernel/orbis/include/orbis/fmem.hpp @@ -4,13 +4,9 @@ #include "rx/AddressRange.hpp" #include -namespace orbis { -struct Process; -} namespace orbis::fmem { -ErrorCode initialize(Process *process, std::uint64_t size); -void destroy(Process *process); -std::pair allocate(Process *process, - std::uint64_t size); -ErrorCode deallocate(Process *process, rx::AddressRange range); +ErrorCode initialize(std::uint64_t size); +void destroy(); +std::pair allocate(std::uint64_t size); +ErrorCode deallocate(rx::AddressRange range); } // namespace orbis::fmem diff --git a/kernel/orbis/include/orbis/pmem.hpp b/kernel/orbis/include/orbis/pmem.hpp index 516354116..f0994007b 100644 --- a/kernel/orbis/include/orbis/pmem.hpp +++ b/kernel/orbis/include/orbis/pmem.hpp @@ -9,30 +9,20 @@ namespace orbis { using kernel::AllocationFlags; struct IoDevice; +struct File; } // namespace orbis namespace orbis::pmem { -enum class MemoryType : std::uint32_t { - Invalid = -1u, - WbOnion = 0, // write back, CPU bus - WCGarlic = 3, // combining, GPU bus - WbGarlic = 10, // write back, GPU bus -}; - -struct AllocatedMemory { - rx::AddressRange range; - MemoryType memoryType; -}; - ErrorCode initialize(std::uint64_t size); void destroy(); std::pair -allocate(std::uint64_t addressHint, std::uint64_t size, MemoryType memoryType, +allocate(std::uint64_t addressHint, std::uint64_t size, rx::EnumBitSet flags, std::uint64_t alignment); ErrorCode deallocate(rx::AddressRange range); -std::optional query(std::uint64_t address); +std::optional query(std::uint64_t address); ErrorCode map(std::uint64_t virtualAddress, rx::AddressRange range, rx::EnumBitSet protection); std::size_t getSize(); IoDevice *getDevice(); +File *getFile(); } // namespace orbis::pmem diff --git a/kernel/orbis/include/orbis/sys/sysproto.hpp b/kernel/orbis/include/orbis/sys/sysproto.hpp index 6925c6185..d2f0b8ff9 100644 --- a/kernel/orbis/include/orbis/sys/sysproto.hpp +++ b/kernel/orbis/include/orbis/sys/sysproto.hpp @@ -1,4 +1,7 @@ #include "orbis-config.hpp" +#include "orbis/dmem.hpp" +#include "orbis/vmem.hpp" +#include "rx/EnumBitSet.hpp" #include #include #include @@ -18,7 +21,9 @@ using id_t = uint32_t; struct Thread; struct AuthInfo; +namespace vmem { struct MemoryProtection; +} struct ModuleInfo; struct ModuleInfoEx; struct KEvent; @@ -105,16 +110,16 @@ SysResult sys_execve(Thread *thread, ptr fname, ptr> argv, ptr> envv); SysResult sys_umask(Thread *thread, sint newmask); SysResult sys_chroot(Thread *thread, ptr path); -SysResult sys_msync(Thread *thread, ptr addr, size_t len, sint flags); +SysResult sys_msync(Thread *thread, uintptr_t addr, size_t len, sint flags); SysResult sys_vfork(Thread *thread); SysResult sys_sbrk(Thread *thread, sint incr); SysResult sys_sstk(Thread *thread, sint incr); SysResult sys_ovadvise(Thread *thread, sint anom); -SysResult sys_munmap(Thread *thread, ptr addr, size_t len); -SysResult sys_mprotect(Thread *thread, ptr addr, size_t len, - sint prot); -SysResult sys_madvise(Thread *thread, ptr addr, size_t len, sint behav); -SysResult sys_mincore(Thread *thread, ptr addr, size_t len, +SysResult sys_munmap(Thread *thread, uintptr_t addr, size_t len); +SysResult sys_mprotect(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot); +SysResult sys_madvise(Thread *thread, uintptr_t addr, size_t len, sint behav); +SysResult sys_mincore(Thread *thread, uintptr_t addr, size_t len, ptr vec); SysResult sys_getgroups(Thread *thread, uint gidsetsize, ptr gidset); SysResult sys_setgroups(Thread *thread, uint gidsetsize, ptr gidset); @@ -198,8 +203,10 @@ SysResult sys_getrlimit(Thread *thread, uint which, ptr rlp); SysResult sys_setrlimit(Thread *thread, uint which, ptr rlp); SysResult sys_getdirentries(Thread *thread, sint fd, ptr buf, uint count, ptr basep); -SysResult sys_freebsd6_mmap(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags, sint fd, sint pad, off_t pos); +SysResult sys_freebsd6_mmap(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, + sint pad, off_t pos); SysResult sys_freebsd6_lseek(Thread *thread, sint fd, sint pad, off_t offset, sint whence); SysResult sys_freebsd6_truncate(Thread *thread, ptr path, sint pad, @@ -209,8 +216,8 @@ SysResult sys_freebsd6_ftruncate(Thread *thread, sint fd, sint pad, SysResult sys___sysctl(Thread *thread, ptr name, uint namelen, ptr old, ptr oldenp, ptr new_, size_t newlen); -SysResult sys_mlock(Thread *thread, ptr addr, size_t len); -SysResult sys_munlock(Thread *thread, ptr addr, size_t len); +SysResult sys_mlock(Thread *thread, uintptr_t addr, size_t len); +SysResult sys_munlock(Thread *thread, uintptr_t addr, size_t len); SysResult sys_undelete(Thread *thread, ptr path); SysResult sys_futimes(Thread *thread, sint fd, ptr tptr); SysResult sys_getpgid(Thread *thread, pid_t pid); @@ -246,7 +253,7 @@ SysResult sys_ktimer_getoverrun(Thread *thread, sint timerid); SysResult sys_nanosleep(Thread *thread, cptr rqtp, ptr rmtp); SysResult sys_ntp_gettime(Thread *thread, ptr ntvp); -SysResult sys_minherit(Thread *thread, ptr addr, size_t len, +SysResult sys_minherit(Thread *thread, uintptr_t addr, size_t len, sint inherit); SysResult sys_rfork(Thread *thread, sint flags); SysResult sys_openbsd_poll(Thread *thread, ptr fds, uint nfds, @@ -519,8 +526,9 @@ SysResult sys_pread(Thread *thread, sint fd, ptr buf, size_t nbyte, off_t offset); SysResult sys_pwrite(Thread *thread, sint fd, ptr buf, size_t nbyte, off_t offset); -SysResult sys_mmap(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags, sint fd, off_t pos); +SysResult sys_mmap(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, off_t pos); SysResult sys_lseek(Thread *thread, sint fd, off_t offset, sint whence); SysResult sys_truncate(Thread *thread, ptr path, off_t length); SysResult sys_ftruncate(Thread *thread, sint fd, off_t length); @@ -621,7 +629,9 @@ SysResult sys_socketex(Thread *thread, ptr name, sint domain, SysResult sys_socketclose(Thread *thread, sint fd); SysResult sys_netgetiflist(Thread *thread /* TODO */); SysResult sys_kqueueex(Thread *thread, ptr name, sint flags); -SysResult sys_mtypeprotect(Thread *thread /* TODO */); +SysResult sys_mtypeprotect(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, + rx::EnumBitSet prot); SysResult sys_regmgr_call(Thread *thread, uint32_t op, uint32_t id, ptr result, ptr value, uint64_t len); SysResult sys_jitshm_create(Thread *thread /* TODO */); @@ -644,9 +654,10 @@ SysResult sys_evf_set(Thread *thread, sint id, uint64_t value); SysResult sys_evf_clear(Thread *thread, sint id, uint64_t value); SysResult sys_evf_cancel(Thread *thread, sint id, uint64_t value, ptr pNumWaitThreads); -SysResult sys_query_memory_protection(Thread *thread, ptr address, - ptr protection); -SysResult sys_batch_map(Thread *thread, sint unk, sint flags, +SysResult sys_query_memory_protection(Thread *thread, uintptr_t address, + ptr protection); +SysResult sys_batch_map(Thread *thread, sint unk, + rx::EnumBitSet flags, ptr entries, sint entriesCount, ptr processedCount); SysResult sys_osem_create(Thread *thread, ptr name, uint attrs, @@ -679,7 +690,7 @@ SysResult sys_budget_delete(Thread *thread, sint budget); SysResult sys_budget_get(Thread *thread, sint id, ptr budgetInfo, ptr count); SysResult sys_budget_set(Thread *thread, sint budget); -SysResult sys_virtual_query(Thread *thread, ptr addr, uint64_t unk, +SysResult sys_virtual_query(Thread *thread, uintptr_t addr, uint64_t unk, ptr info, size_t infosz); SysResult sys_mdbg_call(Thread *thread /* TODO */); SysResult sys_obs_sblock_create(Thread *thread /* TODO */); @@ -752,8 +763,10 @@ SysResult sys_opmc_set_hw(Thread *thread /* TODO */); SysResult sys_opmc_get_hw(Thread *thread /* TODO */); SysResult sys_get_cpu_usage_all(Thread *thread, uint32_t unk, ptr result); -SysResult sys_mmap_dmem(Thread *thread, caddr_t addr, size_t len, - sint memoryType, sint prot, sint flags, +SysResult sys_mmap_dmem(Thread *thread, uintptr_t addr, size_t len, + MemoryType memoryType, + rx::EnumBitSet prot, + rx::EnumBitSet flags, off_t directMemoryStart); SysResult sys_physhm_open(Thread *thread /* TODO */); SysResult sys_physhm_unlink(Thread *thread /* TODO */); @@ -785,9 +798,11 @@ SysResult sys_budget_get_ptype_of_budget(Thread *thread, sint budgetId); SysResult sys_prepare_to_resume_process(Thread *thread /* TODO */); SysResult sys_process_terminate(Thread *thread /* TODO */); SysResult sys_blockpool_open(Thread *thread); -SysResult sys_blockpool_map(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags); -SysResult sys_blockpool_unmap(Thread *thread, caddr_t addr, size_t len, +SysResult sys_blockpool_map(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, + rx::EnumBitSet prot, + rx::EnumBitSet flags); +SysResult sys_blockpool_unmap(Thread *thread, uintptr_t addr, size_t len, sint flags); SysResult sys_dynlib_get_info_for_libdbg(Thread *thread /* TODO */); SysResult sys_blockpool_batch(Thread *thread /* TODO */); diff --git a/kernel/orbis/include/orbis/thread/Process.hpp b/kernel/orbis/include/orbis/thread/Process.hpp index 245ddb7ae..6a8d8d19f 100644 --- a/kernel/orbis/include/orbis/thread/Process.hpp +++ b/kernel/orbis/include/orbis/thread/Process.hpp @@ -102,6 +102,8 @@ struct Process final { rx::RcIdMap threadsMap; rx::RcIdMap fileDescriptors; + rx::AddressRange libkernelRange; + // Named objects for debugging rx::shared_mutex namedObjMutex; kmap namedObjNames; @@ -109,10 +111,6 @@ struct Process final { kmap sigActions; - // Named memory ranges for debugging - rx::shared_mutex namedMemMutex; - kmap namedMem; - // FIXME: implement process destruction void incRef() {} void decRef() {} @@ -126,6 +124,8 @@ struct Process final { ref) { return ref.get(storage); } + + Budget *getBudget() const; }; pid_t allocatePid(); diff --git a/kernel/orbis/include/orbis/thread/ProcessOps.hpp b/kernel/orbis/include/orbis/thread/ProcessOps.hpp index 717fb8c55..7a3eeb4f6 100644 --- a/kernel/orbis/include/orbis/thread/ProcessOps.hpp +++ b/kernel/orbis/include/orbis/thread/ProcessOps.hpp @@ -10,34 +10,13 @@ struct Thread; struct Module; struct timespec; struct File; +namespace vmem { struct MemoryProtection; +} struct IoVec; struct UContext; struct ProcessOps { - SysResult (*mmap)(Thread *thread, caddr_t addr, size_t len, sint prot, - sint flags, sint fd, off_t pos); - SysResult (*dmem_mmap)(Thread *thread, caddr_t addr, size_t len, - sint memoryType, sint prot, sint flags, - off_t directMemoryStart); - SysResult (*munmap)(Thread *thread, ptr addr, size_t len); - SysResult (*msync)(Thread *thread, ptr addr, size_t len, sint flags); - SysResult (*mprotect)(Thread *thread, ptr addr, size_t len, - sint prot); - SysResult (*minherit)(Thread *thread, ptr addr, size_t len, - sint inherit); - SysResult (*madvise)(Thread *thread, ptr addr, size_t len, sint behav); - SysResult (*mincore)(Thread *thread, ptr addr, size_t len, - ptr vec); - SysResult (*mlock)(Thread *thread, ptr addr, size_t len); - SysResult (*mlockall)(Thread *thread, sint how); - SysResult (*munlockall)(Thread *thread); - SysResult (*munlock)(Thread *thread, ptr addr, size_t len); - SysResult (*virtual_query)(Thread *thread, ptr addr, sint flags, - ptr info, ulong infoSize); - SysResult (*query_memory_protection)(Thread *thread, ptr address, - ptr protection); - SysResult (*open)(Thread *thread, ptr path, sint flags, sint mode, rx::Ref *file); SysResult (*shm_open)(Thread *thread, const char *path, sint flags, sint mode, @@ -46,10 +25,6 @@ struct ProcessOps { SysResult (*mkdir)(Thread *thread, ptr path, sint mode); SysResult (*rmdir)(Thread *thread, ptr path); SysResult (*rename)(Thread *thread, ptr from, ptr to); - SysResult (*blockpool_open)(Thread *thread, rx::Ref *file); - SysResult (*blockpool_map)(Thread *thread, caddr_t addr, size_t len, - sint prot, sint flags); - SysResult (*blockpool_unmap)(Thread *thread, caddr_t addr, size_t len); SysResult (*socket)(Thread *thread, ptr name, sint domain, sint type, sint protocol, rx::Ref *file); SysResult (*socketpair)(Thread *thread, sint domain, sint type, sint protocol, diff --git a/kernel/orbis/include/orbis/thread/Thread.hpp b/kernel/orbis/include/orbis/thread/Thread.hpp index 7963c4335..9df49bf45 100644 --- a/kernel/orbis/include/orbis/thread/Thread.hpp +++ b/kernel/orbis/include/orbis/thread/Thread.hpp @@ -32,8 +32,8 @@ struct Thread final { uint64_t retval[2]{}; void *context{}; kvector altStack; - ptr stackStart; - ptr stackEnd; + uint64_t stackStart; + uint64_t stackEnd; uint64_t fsBase{}; uint64_t gsBase{}; rx::StaticString<31> name; @@ -97,6 +97,7 @@ struct Thread final { }; Thread *createThread(Process *process, std::string_view name); +uintptr_t getCallerAddress(Thread *thread); extern thread_local Thread *g_currentThread; diff --git a/kernel/orbis/include/orbis/thread/sysent.hpp b/kernel/orbis/include/orbis/thread/sysent.hpp index 01f37e1b3..719f45c70 100644 --- a/kernel/orbis/include/orbis/thread/sysent.hpp +++ b/kernel/orbis/include/orbis/thread/sysent.hpp @@ -1,6 +1,7 @@ #pragma once #include "orbis-config.hpp" +#include namespace orbis { struct Thread; @@ -9,6 +10,7 @@ using sy_call_t = SysResult(Thread *, uint64_t *); struct sysent { sint narg; sy_call_t *call; + std::string (*format)(uint64_t *); }; struct sysentvec { diff --git a/kernel/orbis/include/orbis/thread/types.hpp b/kernel/orbis/include/orbis/thread/types.hpp index 438b6fa01..c509d8943 100644 --- a/kernel/orbis/include/orbis/thread/types.hpp +++ b/kernel/orbis/include/orbis/thread/types.hpp @@ -15,7 +15,7 @@ struct rtprio { struct thr_param { ptr start_func; ptr arg; - ptr stack_base; + uint64_t stack_base; size_t stack_size; ptr tls_base; size_t tls_size; diff --git a/kernel/orbis/include/orbis/ucontext.hpp b/kernel/orbis/include/orbis/ucontext.hpp index fa47726a3..9066510ee 100644 --- a/kernel/orbis/include/orbis/ucontext.hpp +++ b/kernel/orbis/include/orbis/ucontext.hpp @@ -47,7 +47,7 @@ struct MContext { }; struct Stack { - ptr sp; + uintptr_t sp; size_t size; sint flags; sint align; diff --git a/kernel/orbis/include/orbis/vm.hpp b/kernel/orbis/include/orbis/vm.hpp deleted file mode 100644 index 8e55bc27d..000000000 --- a/kernel/orbis/include/orbis/vm.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "orbis-config.hpp" - -namespace orbis { -struct MemoryProtection { - uint64_t startAddress; - uint64_t endAddress; - int32_t prot; -}; - -static_assert(sizeof(MemoryProtection) == 24); -} // namespace orbis diff --git a/kernel/orbis/include/orbis/vmem.hpp b/kernel/orbis/include/orbis/vmem.hpp index 3a9724467..461eb274c 100644 --- a/kernel/orbis/include/orbis/vmem.hpp +++ b/kernel/orbis/include/orbis/vmem.hpp @@ -1,21 +1,22 @@ #pragma once +#include "MemoryType.hpp" #include "kernel/MemoryResource.hpp" #include "orbis-config.hpp" #include "rx/AddressRange.hpp" #include "rx/EnumBitSet.hpp" #include "rx/StaticString.hpp" +#include "rx/mem.hpp" #include namespace orbis { using kernel::AllocationFlags; -} -namespace orbis { -struct IoDevice; +struct File; struct Process; +} // namespace orbis -namespace vmem { +namespace orbis::vmem { static constexpr auto kPageSize = 16 * 1024; enum class Protection { @@ -28,33 +29,85 @@ enum class Protection { bitset_last = GpuWrite }; -enum class BlockFlags { +enum class BlockFlags : std::uint8_t { FlexibleMemory, DirectMemory, Stack, PooledMemory, Commited, - Allocated, - bitset_last = Allocated + bitset_last = Commited }; +enum class BlockFlagsEx : std::uint8_t { + Allocated, + Private, + Shared, + PoolControl, + Reserved, + + bitset_last = Reserved +}; + +enum class MapFlags { + Shared = 0, + Private = 1, + Fixed = 4, + Rename = 5, + NoReserve = 6, + NoOverwrite = 7, + Void = 8, + HasSemaphore = 9, + Stack = 10, + NoSync = 11, + Anon = 12, + System = 13, + AllAvailable = 14, + NoCore = 17, + PrefaultRead = 18, + Self = 19, + NoCoalesce = 22, + + bitset_last = NoCoalesce +}; + +inline constexpr std::uint32_t kMapFlagsAlignShift = 24; +inline constexpr std::uint32_t kMapFlagsAlignMask = 0x1f << kMapFlagsAlignShift; + inline constexpr auto kProtCpuReadWrite = Protection::CpuRead | Protection::CpuWrite; inline constexpr auto kProtCpuAll = Protection::CpuRead | Protection::CpuWrite | Protection::CpuExec; inline constexpr auto kProtGpuAll = Protection::GpuRead | Protection::GpuWrite; +inline std::pair> +unpackMapFlags(rx::EnumBitSet flags, std::uint64_t minAlignment) { + std::uint64_t alignment = minAlignment; + + if (auto align = + (flags.toUnderlying() & kMapFlagsAlignMask) >> kMapFlagsAlignShift) { + alignment = std::uint64_t(1) << align; + flags = rx::EnumBitSet::fromUnderlying( + flags.toUnderlying() & ~kMapFlagsAlignMask); + + if (alignment < minAlignment) { + alignment = minAlignment; + } + } + + return {alignment, flags}; +} + #pragma pack(push, 1) struct QueryResult { uint64_t start; uint64_t end; uint64_t offset; - uint32_t protection; - uint32_t memoryType; - uint32_t flags; + rx::EnumBitSet protection; + MemoryType memoryType; + rx::EnumBitSet flags; rx::StaticCString<32> name; - uint32_t _padding; + char _padding[7]; }; static_assert(sizeof(QueryResult) == 72); @@ -69,25 +122,81 @@ struct MemoryProtection { static_assert(sizeof(MemoryProtection) == 24); #pragma pack(pop) +inline constexpr rx::EnumBitSet +toCpuProtection(rx::EnumBitSet prot) { + rx::EnumBitSet result{}; + if (prot & Protection::CpuRead) { + result |= rx::mem::Protection::R; + } + if (prot & Protection::CpuWrite) { + result |= rx::mem::Protection::W; + } + if (prot & Protection::CpuExec) { + result |= rx::mem::Protection::X; + } + return result; +} + +inline constexpr rx::EnumBitSet +toGpuProtection(rx::EnumBitSet prot) { + rx::EnumBitSet result{}; + if (prot & Protection::GpuRead) { + result |= rx::mem::Protection::R; + } + if (prot & Protection::GpuWrite) { + result |= rx::mem::Protection::W; + } + return result; +} + void initialize(Process *process, bool force = false); void fork(Process *process, Process *parentThread); std::pair reserve(Process *process, std::uint64_t addressHint, std::uint64_t size, - rx::EnumBitSet allocFlags); + rx::EnumBitSet allocFlags, + rx::EnumBitSet blockFlagsEx = {}, + std::uint64_t alignment = kPageSize); std::pair -map(Process *process, std::uint64_t addressHint, std::uint64_t size, - rx::EnumBitSet allocFlags, - rx::EnumBitSet prot = {}, - rx::EnumBitSet blockFlags = {}, - std::uint64_t alignment = kPageSize, std::string_view name = {}, - IoDevice *device = nullptr, std::int64_t deviceOffset = 0); +mapFile(Process *process, std::uint64_t addressHint, std::uint64_t size, + rx::EnumBitSet allocFlags, + rx::EnumBitSet prot, rx::EnumBitSet blockFlags, + rx::EnumBitSet blockFlagsEx, File *file, + std::uint64_t fileOffset, std::string_view name = {}, + std::uint64_t alignment = kPageSize, + MemoryType type = MemoryType::Invalid); + +std::pair +mapDirect(Process *process, std::uint64_t addressHint, + rx::AddressRange directRange, rx::EnumBitSet prot, + rx::EnumBitSet allocFlags, + std::string_view name = {}, std::uint64_t alignment = kPageSize, + MemoryType type = MemoryType::Invalid); + +std::pair +mapFlex(Process *process, std::uint64_t size, rx::EnumBitSet prot, + std::uint64_t addressHint = 0, + rx::EnumBitSet allocFlags = {}, + rx::EnumBitSet blockFlags = {}, std::string_view name = {}, + std::uint64_t alignment = kPageSize); + +std::pair +commitPooled(Process *process, rx::AddressRange addressRange, MemoryType type, + rx::EnumBitSet prot); +ErrorCode decommitPooled(Process *process, rx::AddressRange addressRange); + +ErrorCode protect(Process *process, rx::AddressRange range, + rx::EnumBitSet prot); ErrorCode unmap(Process *process, rx::AddressRange range); ErrorCode setName(Process *process, rx::AddressRange range, std::string_view name); -std::optional query(Process *process, std::uint64_t address); +ErrorCode setType(Process *process, rx::AddressRange range, MemoryType type); +ErrorCode setTypeAndProtect(Process *process, rx::AddressRange range, + MemoryType type, rx::EnumBitSet prot); +std::optional query(Process *process, std::uint64_t address, + bool lowerBound = false); std::optional queryProtection(Process *process, - std::uint64_t address); -} // namespace vmem -} // namespace orbis + std::uint64_t address, + bool lowerBound = false); +} // namespace orbis::vmem diff --git a/kernel/orbis/src/IoDevice.cpp b/kernel/orbis/src/IoDevice.cpp index 0840f2a2e..bd0dfecd1 100644 --- a/kernel/orbis/src/IoDevice.cpp +++ b/kernel/orbis/src/IoDevice.cpp @@ -1,6 +1,9 @@ #include "IoDevice.hpp" +#include "file.hpp" +#include "rx/Mappable.hpp" #include "thread/Thread.hpp" #include "utils/Logs.hpp" +#include "vmem.hpp" static std::string iocGroupToString(unsigned iocGroup) { if (iocGroup >= 128) { @@ -64,6 +67,23 @@ static std::string iocGroupToString(unsigned iocGroup) { return "'?'"; } +orbis::ErrorCode +orbis::IoDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, File *file, + Process *) { + if (!file->dirEntries.empty()) { + return orbis::ErrorCode::ISDIR; + } + + if (!file->hostFd) { + return ErrorCode::NOTSUP; + } + + auto errc = file->hostFd.map(range, offset, vmem::toCpuProtection(protection), + orbis::vmem::kPageSize); + return orbis::toErrorCode(errc); +} + orbis::ErrorCode orbis::IoDevice::ioctl(std::uint64_t request, orbis::ptr argp, Thread *thread) { auto group = iocGroupToString(ioctl::group(request)); diff --git a/kernel/orbis/src/blockpool.cpp b/kernel/orbis/src/blockpool.cpp new file mode 100644 index 000000000..c2389e954 --- /dev/null +++ b/kernel/orbis/src/blockpool.cpp @@ -0,0 +1,370 @@ +#include "blockpool.hpp" +#include "KernelAllocator.hpp" +#include "KernelObject.hpp" +#include "dmem.hpp" +#include "pmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/MemoryTable.hpp" +#include "rx/die.hpp" +#include "thread/Process.hpp" +#include "vmem.hpp" +#include +#include +#include + +// FIXME: remove +namespace amdgpu { +void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset); +void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange); +void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot); +} // namespace amdgpu + +struct PooledMemoryResource { + struct CommitedBlock { + std::uint64_t pmemAddress; + orbis::MemoryType type; + bool operator==(const CommitedBlock &) const = default; + }; + + std::uint64_t total = 0; + std::uint64_t used = 0; + orbis::kvector freeBlocks; + rx::MemoryTableWithPayload usedBlocks; + orbis::kvector reservedPages; + + void clear() { + total = 0; + used = 0; + freeBlocks.clear(); + usedBlocks.clear(); + reservedPages.clear(); + } + + void addFreeBlock(rx::AddressRange dmemRange) { + if (freeBlocks.empty()) { + freeBlocks.push_back(dmemRange); + return; + } + + auto it = std::upper_bound( + freeBlocks.begin(), freeBlocks.end(), dmemRange.beginAddress(), + [](auto lhs, auto rhs) { + if constexpr (requires { lhs.beginAddress() < rhs; }) { + return lhs.beginAddress() < rhs; + } else { + return lhs < rhs.beginAddress(); + } + }); + + if (it != freeBlocks.end() && + dmemRange.endAddress() == it->beginAddress()) { + *it = rx::AddressRange::fromBeginEnd(it->beginAddress(), + dmemRange.endAddress()); + return; + } + + if (it != freeBlocks.begin()) { + auto prev = std::prev(it); + + if (prev->endAddress() == dmemRange.beginAddress()) { + *prev = rx::AddressRange::fromBeginEnd(prev->beginAddress(), + dmemRange.endAddress()); + return; + } + } + + freeBlocks.insert(it, dmemRange); + return; + } + + void expand(rx::AddressRange dmemRange) { + addFreeBlock(dmemRange); + total += dmemRange.size(); + } + + orbis::ErrorCode reserve(std::size_t size) { + if (size > total) { + return orbis::ErrorCode::INVAL; + } + + used += size; + return {}; + } + + std::pair reservePage() { + if (total - used < orbis::dmem::kPageSize) { + return {{}, orbis::ErrorCode::INVAL}; + } + + used += orbis::dmem::kPageSize; + auto &block = freeBlocks.back(); + auto allocatedPage = block.beginAddress(); + + if (block.endAddress() == allocatedPage + orbis::dmem::kPageSize) { + freeBlocks.pop_back(); + } else { + block = rx::AddressRange::fromBeginEnd( + allocatedPage + orbis::dmem::kPageSize, block.endAddress()); + } + + reservedPages.push_back(allocatedPage); + return {allocatedPage, {}}; + } + + std::pair releasePage() { + if (reservedPages.empty()) { + return {{}, orbis::ErrorCode::INVAL}; + } + + used -= orbis::dmem::kPageSize; + total -= orbis::dmem::kPageSize; + + auto address = reservedPages.back(); + reservedPages.pop_back(); + return {address, {}}; + } + + void moveTo(PooledMemoryResource &other, std::size_t size) { + while (size > 0 && total > 0) { + auto &block = freeBlocks.back(); + + if (block.size() > size) { + total -= size; + + auto moveRange = + rx::AddressRange::fromBeginSize(block.beginAddress(), size); + + other.expand(moveRange); + + block = rx::AddressRange::fromBeginEnd(moveRange.endAddress(), + block.endAddress()); + break; + } + + total -= block.size(); + size -= block.size(); + other.expand(block); + freeBlocks.pop_back(); + } + } + + void commit(orbis::Process *process, rx::AddressRange virtualRange, + orbis::MemoryType type, + rx::EnumBitSet protection) { + while (virtualRange.isValid()) { + auto &block = freeBlocks.back(); + + if (block.size() >= virtualRange.size()) [[likely]] { + auto mapPhysicalRange = rx::AddressRange::fromBeginSize( + block.beginAddress(), virtualRange.size()); + auto errc = + orbis::pmem::map(virtualRange.beginAddress(), mapPhysicalRange, + orbis::vmem::toCpuProtection(protection)); + + rx::dieIf(errc != orbis::ErrorCode{}, + "blockpool: failed to map physical memory"); + amdgpu::mapMemory(process->pid, virtualRange, type, protection, + block.beginAddress()); + + if (mapPhysicalRange.endAddress() == block.endAddress()) { + freeBlocks.pop_back(); + } else { + block = rx::AddressRange::fromBeginEnd(mapPhysicalRange.endAddress(), + block.endAddress()); + } + + usedBlocks.map(virtualRange, + {.pmemAddress = block.beginAddress(), .type = type}, + false); + used += virtualRange.size(); + break; + } + + auto mapVirtualRange = rx::AddressRange::fromBeginSize( + virtualRange.beginAddress(), block.size()); + + virtualRange = rx::AddressRange::fromBeginEnd( + mapVirtualRange.endAddress(), virtualRange.endAddress()); + + auto errc = orbis::pmem::map(mapVirtualRange.beginAddress(), block, + orbis::vmem::toCpuProtection(protection)); + + rx::dieIf(errc != orbis::ErrorCode{}, + "blockpool: failed to map physical memory"); + amdgpu::mapMemory(process->pid, mapVirtualRange, type, protection, + block.beginAddress()); + + usedBlocks.map(mapVirtualRange, + {.pmemAddress = block.beginAddress(), .type = type}, + false); + freeBlocks.pop_back(); + } + } + + void decommit(orbis::Process *process, rx::AddressRange virtualRange) { + auto it = usedBlocks.lowerBound(virtualRange.beginAddress()); + + if (it != usedBlocks.end() && + it.beginAddress() < virtualRange.beginAddress()) { + auto itRange = it.range(); + auto decommitRange = itRange.intersection(virtualRange); + used -= decommitRange.size(); + + auto decommitPmemRange = rx::AddressRange::fromBeginSize( + it->pmemAddress + + (virtualRange.beginAddress() - itRange.beginAddress()), + decommitRange.size()); + addFreeBlock(decommitPmemRange); + usedBlocks.unmap(decommitRange); + amdgpu::unmapMemory(process->pid, decommitRange); + ++it; + } + + while (it != usedBlocks.end() && + it.beginAddress() < virtualRange.endAddress()) { + auto itRange = it.range(); + auto decommitRange = itRange.intersection(virtualRange); + used -= decommitRange.size(); + + addFreeBlock(rx::AddressRange::fromBeginSize(it->pmemAddress, + decommitRange.size())); + amdgpu::unmapMemory(process->pid, decommitRange); + + if (itRange == decommitRange) { + it = usedBlocks.unmap(it); + } else { + usedBlocks.unmap(decommitRange); + break; + } + } + } + + std::optional getMemoryType(std::uint64_t address) { + auto it = usedBlocks.queryArea(address); + if (it == usedBlocks.end()) { + return {}; + } + return it->type; + } +}; + +static auto g_blockpool = orbis::createGlobalObject< + kernel::LockableKernelObject>(); + +static auto g_cachedBlockpool = orbis::createGlobalObject< + kernel::LockableKernelObject>(); + +void orbis::blockpool::clear() { + std::scoped_lock lock(*g_blockpool, *g_cachedBlockpool); + + g_blockpool->clear(); + g_cachedBlockpool->clear(); +} + +orbis::ErrorCode orbis::blockpool::expand(rx::AddressRange dmemRange) { + std::scoped_lock lock(*g_blockpool); + g_blockpool->expand(dmemRange); + return {}; +} + +orbis::ErrorCode orbis::blockpool::allocateControlBlock() { + std::scoped_lock cachedLock(*g_cachedBlockpool); + if (g_cachedBlockpool->used < g_cachedBlockpool->total) { + return g_cachedBlockpool->reservePage().second; + } + + std::scoped_lock lock(*g_blockpool); + + if (g_blockpool->total - g_blockpool->used < dmem::kPageSize) { + return ErrorCode::INVAL; + } + + g_blockpool->moveTo(*g_cachedBlockpool, dmem::kPageSize); + return g_cachedBlockpool->reservePage().second; +} + +orbis::ErrorCode orbis::blockpool::releaseControlBlock() { + std::scoped_lock lock(*g_cachedBlockpool); + + // control block is always cached + if (!g_cachedBlockpool->reservedPages.empty()) { + auto [page, errc] = g_cachedBlockpool->releasePage(); + if (errc != ErrorCode{}) { + return errc; + } + + g_cachedBlockpool->expand( + rx::AddressRange::fromBeginSize(page, dmem::kPageSize)); + return {}; + } + + return ErrorCode::INVAL; +} + +orbis::ErrorCode +orbis::blockpool::commit(Process *process, rx::AddressRange vmemRange, + MemoryType type, + rx::EnumBitSet protection) { + auto pool = + type == MemoryType::WbOnion ? g_cachedBlockpool : g_blockpool; + auto otherPool = + type == MemoryType::WbOnion ? g_blockpool : g_cachedBlockpool; + + std::scoped_lock lock(*pool); + + if (auto avail = pool->total - pool->used; avail < vmemRange.size()) { + // try to steal free blocks from other pool + + std::scoped_lock lock(*otherPool); + auto pullSize = vmemRange.size() - avail; + + if (otherPool->total - otherPool->used < pullSize) { + return ErrorCode::NOMEM; + } + + otherPool->moveTo(*pool, pullSize); + } + + pool->commit(process, vmemRange, type, protection); + return {}; +} + +void orbis::blockpool::decommit(Process *process, rx::AddressRange vmemRange) { + std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool); + g_cachedBlockpool->decommit(process, vmemRange); + g_blockpool->decommit(process, vmemRange); +} + +std::optional +orbis::blockpool::getType(std::uint64_t address) { + { + std::scoped_lock lock(*g_cachedBlockpool); + + if (auto result = g_cachedBlockpool->getMemoryType(address)) { + return result; + } + } + + std::scoped_lock lock(*g_blockpool); + return g_blockpool->getMemoryType(address); +} + +orbis::blockpool::BlockStats orbis::blockpool::stats() { + BlockStats result{}; + + { + std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool); + result.availFlushedBlocks = + (g_blockpool->total - g_blockpool->used) / dmem::kPageSize; + result.availCachedBlocks = + (g_cachedBlockpool->total - g_cachedBlockpool->used) / dmem::kPageSize; + result.commitFlushedBlocks = g_blockpool->used / dmem::kPageSize; + result.commitCachedBlocks = g_cachedBlockpool->used / dmem::kPageSize; + } + + return result; +} diff --git a/kernel/orbis/src/dmem.cpp b/kernel/orbis/src/dmem.cpp new file mode 100644 index 000000000..546c322d7 --- /dev/null +++ b/kernel/orbis/src/dmem.cpp @@ -0,0 +1,610 @@ +#include "dmem.hpp" +#include "KernelAllocator.hpp" +#include "KernelObject.hpp" +#include "error.hpp" +#include "kernel/KernelObject.hpp" +#include "pmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/FileLock.hpp" +#include "rx/Serializer.hpp" +#include "rx/die.hpp" +#include "rx/print.hpp" +#include "utils/Logs.hpp" +#include "vmem.hpp" +#include +#include +#include +#include + +struct DirectMemoryAllocation { + static constexpr std::uint32_t kAllocatedBit = 1 << 31; + static constexpr std::uint32_t kPooledBit = 1 << 30; + std::uint32_t type = 0; + + [[nodiscard]] bool isPooled() const { return type & kPooledBit; } + void markAsPooled() { type |= kPooledBit | kAllocatedBit; } + + void setMemoryType(orbis::MemoryType memoryType) { + type = (std::to_underlying(memoryType) & ~(kAllocatedBit | kPooledBit)) | + (type & kPooledBit) | kAllocatedBit; + } + + [[nodiscard]] orbis::MemoryType getMemoryType() const { + return static_cast(type & ~(kAllocatedBit | kPooledBit)); + } + + [[nodiscard]] bool isAllocated() const { return (type & kAllocatedBit) != 0; } + [[nodiscard]] bool isRelated(const DirectMemoryAllocation &other, + rx::AddressRange, rx::AddressRange) const { + return type == other.type; + } + + [[nodiscard]] DirectMemoryAllocation + merge(const DirectMemoryAllocation &other, rx::AddressRange, + rx::AddressRange) const { + return other; + } + + bool operator==(const DirectMemoryAllocation &) const = default; +}; + +struct DirectMemoryResourceState { + std::uint64_t pmemOffset{}; + std::uint64_t dmemTotalSize{}; + std::uint64_t dmemReservedSize{}; +}; + +struct DirectMemoryResource + : kernel::AllocableResource, + DirectMemoryResourceState { + using BaseResource = + kernel::AllocableResource; + + void create(std::size_t size) { + auto [pmemRange, errc] = orbis::pmem::allocate(0, size, {}, 64 * 1024); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to allocate direct memory: size {:x}, error {}", size, + static_cast(errc)); + + dmemTotalSize = pmemRange.size(); + pmemOffset = pmemRange.beginAddress(); + auto dmemResourceRange = + rx::AddressRange::fromBeginSize(0, pmemRange.size()); + + if (auto errc = BaseResource::create(dmemResourceRange); + errc != std::errc{}) { + rx::die("failed to create direct memory resource {:x}, error {}", + pmemRange.size(), errc); + } + } + + orbis::ErrorCode clear() { + auto result = BaseResource::create( + rx::AddressRange::fromBeginSize(0, dmemTotalSize + dmemReservedSize)); + + if (result != std::errc{}) { + return orbis::toErrorCode(result); + } + dmemReservedSize = 0; + return {}; + } + + void serialize(rx::Serializer &s) const { + s.serialize(static_cast(*this)); + BaseResource::serialize(s); + } + + void deserialize(rx::Deserializer &d) { + d.deserialize(static_cast(*this)); + BaseResource::deserialize(d); + } +}; + +static std::array g_dmemPools = { + orbis::createGlobalObject< + kernel::LockableKernelObject>(), + orbis::createGlobalObject< + kernel::LockableKernelObject>(), + orbis::createGlobalObject< + kernel::LockableKernelObject>(), +}; + +orbis::ErrorCode orbis::dmem::initialize() { + g_dmemPools[0]->create(0x120000000); + g_dmemPools[1]->create(0x1000000); + g_dmemPools[2]->create(0x1000000); + + return {}; +} + +orbis::ErrorCode orbis::dmem::clear(unsigned dmemIndex) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + { + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + auto result = dmem->clear(); + if (result != orbis::ErrorCode{}) { + return result; + } + } + + return {}; +} + +static void dmemDump(unsigned dmemIndex, std::string_view message = {}) { + auto dmem = g_dmemPools[dmemIndex]; + + rx::ScopedFileLock lock(stderr); + + rx::println(stderr, "dmem0 {}", message); + for (auto alloc : *dmem) { + rx::println(stderr, " {:012x}-{:012x}: {}{} {}", alloc.beginAddress(), + alloc.endAddress(), "_A"[alloc->isAllocated()], + "_P"[alloc->isPooled()], alloc->getMemoryType()); + } +} + +std::pair +orbis::dmem::allocate(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t len, MemoryType memoryType, + std::uint64_t alignment, bool pooled) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + if (searchRange.endAddress() != 0 && + (!searchRange.isValid() || searchRange.size() < len)) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress(), len); + return {{}, orbis::ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + alignment = alignment == 0 ? kPageSize : alignment; + len = rx::alignUp(len, dmem::kPageSize); + + if (searchRange.endAddress() == 0) { + searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(), + dmem->dmemTotalSize - + dmem->dmemReservedSize); + } + + if (!searchRange.isValid() || searchRange.size() < len) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress(), len); + return {{}, orbis::ErrorCode::INVAL}; + } + + if (len == 0) { + ORBIS_LOG_ERROR(__FUNCTION__, "len is 0", searchRange.beginAddress(), + searchRange.endAddress(), len); + + return {{}, ErrorCode::INVAL}; + } + + if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.beginAddress(), searchRange.endAddress(), + dmem->dmemTotalSize); + return {{}, ErrorCode::INVAL}; + } + + DirectMemoryAllocation allocation; + allocation.setMemoryType(memoryType); + if (pooled) { + allocation.markAsPooled(); + } + + auto allocResult = dmem->map(searchRange.beginAddress(), len, allocation, + AllocationFlags::Dry, alignment); + if (allocResult.errc != std::errc{}) { + return {{}, toErrorCode(allocResult.errc)}; + } + + auto result = allocResult.range.beginAddress(); + + auto commitResult = + dmem->map(result, len, allocation, AllocationFlags::Fixed, alignment); + + rx::dieIf(commitResult.errc != std::errc{}, + "dmem: failed to commit memory, error {}", commitResult.errc); + + dmemDump(dmemIndex, + rx::format("allocated {:x}-{:x}", allocResult.range.beginAddress(), + allocResult.range.endAddress())); + + return {result, {}}; +} + +orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex, + rx::AddressRange range, bool pooled) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + if (range.beginAddress() % kPageSize || range.endAddress() % kPageSize || + !range.isValid()) { + return ErrorCode::INVAL; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + constexpr auto razorGpuMemory = + rx::AddressRange::fromBeginSize(0x3000000000, 0x20000000); + + if (dmemIndex == 0 && razorGpuMemory.contains(range.beginAddress())) { + return ErrorCode::OPNOTSUPP; + } + + auto it = dmem->query(range.beginAddress()); + + if (it == dmem->end() || !it->isAllocated()) { + return ErrorCode::NOENT; + } + + if (it->isPooled() && !pooled) { + return ErrorCode::NOENT; + } + + DirectMemoryAllocation allocation{}; + auto result = dmem->map(range.beginAddress(), range.size(), allocation, + AllocationFlags::Fixed, vmem::kPageSize); + + if (result.errc != std::errc{}) { + return toErrorCode(result.errc); + } + + return {}; +} + +std::pair +orbis::dmem::reserveSystem(unsigned dmemIndex, std::uint64_t size) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + if (size == 0 || size % vmem::kPageSize) { + return {{}, ErrorCode::INVAL}; + } + + if (dmem->dmemReservedSize + size > dmem->dmemTotalSize) { + return {{}, ErrorCode::NOMEM}; + } + + DirectMemoryAllocation alloc; + alloc.setMemoryType(MemoryType::WbOnion); + auto result = dmem->map(0, size, alloc, AllocationFlags::Stack, kPageSize); + + if (result.errc != std::errc{}) { + return {{}, toErrorCode(result.errc)}; + } + + dmem->dmemReservedSize += size; + return {result.range.beginAddress() | 0x4000000000, {}}; +} + +std::pair +orbis::dmem::reservePooled(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t size, std::uint64_t alignment) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + alignment = alignment == 0 ? kPageSize : alignment; + + if (alignment % kPageSize) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + + if (searchRange.endAddress() > dmem->dmemTotalSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize); + return {{}, ErrorCode::INVAL}; + } + + if (!searchRange.isValid() && + searchRange.beginAddress() != searchRange.endAddress()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, ErrorCode::INVAL}; + } + + std::lock_guard lock(*dmem); + + if (searchRange.beginAddress() == searchRange.endAddress()) { + searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(), + dmem->dmemTotalSize - + dmem->dmemReservedSize); + + if (!searchRange.isValid()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, ErrorCode::INVAL}; + } + } + + if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize, + dmem->dmemReservedSize); + return {{}, ErrorCode::INVAL}; + } + + auto it = dmem->lowerBound(searchRange.beginAddress()); + rx::AddressRange result; + + while (it != dmem->end() && it.beginAddress() < searchRange.endAddress()) { + if (!it->isAllocated()) { + auto viewRange = searchRange.intersection(it.range()); + + viewRange = rx::AddressRange::fromBeginEnd( + rx::alignUp(viewRange.beginAddress(), alignment), + viewRange.endAddress()); + + if (viewRange.isValid() && viewRange.size() >= size) { + result = + rx::AddressRange::fromBeginSize(viewRange.beginAddress(), size); + break; + } + } + + ++it; + } + + if (!result.isValid()) { + return {{}, ErrorCode::NOMEM}; + } + + DirectMemoryAllocation allocation; + allocation.markAsPooled(); + + auto commitResult = dmem->map(result.beginAddress(), result.size(), + allocation, AllocationFlags::Fixed, alignment); + + if (commitResult.errc != std::errc{}) { + return {{}, toErrorCode(commitResult.errc)}; + } + + return {result.beginAddress(), {}}; +} + +orbis::ErrorCode orbis::dmem::setType(unsigned dmemIndex, + rx::AddressRange range, MemoryType type, + rx::EnumBitSet flags) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + bool expectedPooled = (flags & QueryFlags::Pooled) == QueryFlags::Pooled; + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + auto it = dmem->query(range.beginAddress()); + + if (it == dmem->end() || !it->isAllocated() || + it->isPooled() != expectedPooled) { + return ErrorCode::ACCES; + } + + if (it->getMemoryType() == type && it.range().contains(range)) { + return {}; + } + + DirectMemoryAllocation allocation; + allocation.setMemoryType(type); + auto [_it, errc, _range] = + dmem->map(range.beginAddress(), range.size(), allocation, + AllocationFlags::Fixed, vmem::kPageSize); + + return toErrorCode(errc); +} + +std::optional +orbis::dmem::query(unsigned dmemIndex, std::uint64_t dmemOffset, + rx::EnumBitSet flags) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + auto it = dmem->lowerBound(dmemOffset); + + if (flags & QueryFlags::LowerBound) { + while (it != dmem->end() && !it->isAllocated()) { + ++it; + } + + if (it == dmem->end()) { + return {}; + } + } else if (it == dmem->end() || !it->isAllocated()) { + return {}; + } + + if (!(flags & QueryFlags::Pooled)) { + if (it->isPooled()) { + return {}; + } + } + + return QueryResult{.range = it.range(), .memoryType = it->getMemoryType()}; +} + +std::uint64_t orbis::dmem::getSize(unsigned dmemIndex) { + if (dmemIndex >= std::size(g_dmemPools)) { + return 0; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + return dmem->dmemTotalSize - dmem->dmemReservedSize; +} + +std::pair +orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange, + std::uint64_t alignment) { + if (dmemIndex >= std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + alignment = alignment == 0 ? kPageSize : alignment; + + if (alignment % kPageSize) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + + if (searchRange.endAddress() > dmem->dmemTotalSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize); + return {{}, orbis::ErrorCode::INVAL}; + } + + if (!searchRange.isValid() && + searchRange.beginAddress() != searchRange.endAddress()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, orbis::ErrorCode::INVAL}; + } + + std::lock_guard lock(*dmem); + + if (searchRange.beginAddress() == searchRange.endAddress()) { + searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(), + dmem->dmemTotalSize - + dmem->dmemReservedSize); + + if (!searchRange.isValid()) { + ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(), + searchRange.endAddress()); + return {{}, orbis::ErrorCode::INVAL}; + } + } + + if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size", + searchRange.endAddress(), dmem->dmemTotalSize, + dmem->dmemReservedSize); + return {{}, orbis::ErrorCode::INVAL}; + } + + auto it = dmem->lowerBound(searchRange.beginAddress()); + rx::AddressRange result = rx::AddressRange::fromBeginEnd( + searchRange.beginAddress(), searchRange.beginAddress()); + + while (it != dmem->end() && it.beginAddress() < searchRange.endAddress()) { + if (!it->isAllocated()) { + auto viewRange = searchRange.intersection(it.range()); + + viewRange = rx::AddressRange::fromBeginEnd( + rx::alignUp(viewRange.beginAddress(), alignment), + viewRange.endAddress()); + + if (viewRange.isValid() && + (!result.isValid() || viewRange.size() > result.size())) { + result = viewRange; + } + } + + ++it; + } + + return {result, {}}; +} + +orbis::ErrorCode orbis::dmem::map(unsigned dmemIndex, rx::AddressRange range, + std::uint64_t offset, + rx::EnumBitSet protection) { + if (dmemIndex >= std::size(g_dmemPools)) { + return ErrorCode::INVAL; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + if (offset < 0 || offset + range.size() < offset || + offset + range.size() > dmem->dmemTotalSize) { + return orbis::ErrorCode::INVAL; + } + + if (offset + range.size() > dmem->dmemTotalSize - dmem->dmemReservedSize) { + return orbis::ErrorCode::ACCES; + } + + auto allocationInfoIt = dmem->query(offset); + + if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) { + if (allocationInfoIt != dmem->end()) { + dmemDump( + dmemIndex, + rx::format("map unallocated {:x}-{:x}, requested range {:x}-{:x}", + allocationInfoIt.beginAddress(), + allocationInfoIt.endAddress(), range.beginAddress(), + range.endAddress())); + } else { + dmemDump(dmemIndex, rx::format("map out of memory {:x}-{:x}", + range.beginAddress(), range.endAddress())); + } + return orbis::ErrorCode::ACCES; + } + + if (allocationInfoIt->isPooled()) { + return orbis::ErrorCode::ACCES; + } + + auto directRange = rx::AddressRange::fromBeginSize(offset, range.size()) + .intersection(allocationInfoIt.range()); + + if (range.size() > directRange.size()) { + return orbis::ErrorCode::INVAL; + } + + auto physicalRange = + rx::AddressRange::fromBeginSize(dmem->pmemOffset + offset, range.size()); + + return orbis::pmem::map(range.beginAddress(), physicalRange, + vmem::toCpuProtection(protection)); +} + +std::pair +orbis::dmem::getPmemOffset(unsigned dmemIndex, std::uint64_t dmemOffset) { + if (dmemIndex > std::size(g_dmemPools)) { + return {{}, ErrorCode::INVAL}; + } + + auto dmem = g_dmemPools[dmemIndex]; + std::lock_guard lock(*dmem); + + if (dmemOffset >= dmem->dmemTotalSize) { + return {{}, orbis::ErrorCode::INVAL}; + } + + if (dmemOffset >= dmem->dmemTotalSize - dmem->dmemReservedSize) { + return {{}, orbis::ErrorCode::ACCES}; + } + + auto allocationInfoIt = dmem->query(dmemOffset); + + if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) { + return {{}, orbis::ErrorCode::ACCES}; + } + + return {dmem->pmemOffset + dmemOffset, {}}; +} diff --git a/kernel/orbis/src/fmem.cpp b/kernel/orbis/src/fmem.cpp index cd6125e8d..47ed3bdf5 100644 --- a/kernel/orbis/src/fmem.cpp +++ b/kernel/orbis/src/fmem.cpp @@ -1,12 +1,14 @@ #include "fmem.hpp" +#include "KernelAllocator.hpp" #include "KernelObject.hpp" #include "error.hpp" #include "kernel/KernelObject.hpp" #include "kernel/MemoryResource.hpp" #include "pmem.hpp" #include "rx/AddressRange.hpp" -#include "thread/Process.hpp" +#include "rx/debug.hpp" +#include "rx/print.hpp" #include "vmem.hpp" #include #include @@ -30,43 +32,39 @@ struct FlexibleMemoryAllocation { }; using FlexibleMemoryResource = - kernel::AllocableResource; + kernel::AllocableResource; -static auto g_fmemInstance = orbis::createProcessLocalObject< +static auto g_fmemInstance = orbis::createGlobalObject< kernel::LockableKernelObject>(); -orbis::ErrorCode orbis::fmem::initialize(Process *process, std::uint64_t size) { +orbis::ErrorCode orbis::fmem::initialize(std::uint64_t size) { auto [range, errc] = - pmem::allocate(pmem::getSize() - 1, size, pmem::MemoryType::WbOnion, - kernel::AllocationFlags::Stack, vmem::kPageSize); + pmem::allocate(0, size, kernel::AllocationFlags::Stack, vmem::kPageSize); if (errc != ErrorCode{}) { return errc; } - auto fmem = process->get(g_fmemInstance); - std::lock_guard lock(*fmem); - return toErrorCode(fmem->create(range)); + rx::println("fmem: {:x}-{:x}", range.beginAddress(), range.endAddress()); + + std::lock_guard lock(*g_fmemInstance); + return toErrorCode(g_fmemInstance->create(range)); } -void orbis::fmem::destroy(Process *process) { - auto fmem = process->get(g_fmemInstance); +void orbis::fmem::destroy() { + std::lock_guard lock(*g_fmemInstance); - std::lock_guard lock(*fmem); - - for (auto allocation : fmem->allocations) { + for (auto allocation : g_fmemInstance->allocations) { pmem::deallocate(allocation); } - fmem->destroy(); + g_fmemInstance->destroy(); } std::pair -orbis::fmem::allocate(Process *process, std::uint64_t size) { - auto fmem = process->get(g_fmemInstance); - - std::lock_guard lock(*fmem); +orbis::fmem::allocate(std::uint64_t size) { + std::lock_guard lock(*g_fmemInstance); FlexibleMemoryAllocation allocation{.allocated = true}; - auto [it, errc, range] = fmem->map( + auto [it, errc, range] = g_fmemInstance->map( 0, size, allocation, kernel::AllocationFlags::NoMerge, vmem::kPageSize); if (errc != std::errc{}) { @@ -76,13 +74,12 @@ orbis::fmem::allocate(Process *process, std::uint64_t size) { return {range, {}}; } -orbis::ErrorCode orbis::fmem::deallocate(Process *process, - rx::AddressRange range) { +orbis::ErrorCode orbis::fmem::deallocate(rx::AddressRange range) { FlexibleMemoryAllocation allocation{}; - auto fmem = process->get(g_fmemInstance); - std::lock_guard lock(*fmem); - auto [it, errc, _] = fmem->map(range.beginAddress(), range.size(), allocation, - AllocationFlags::Fixed, 1); + std::lock_guard lock(*g_fmemInstance); + auto [it, errc, _] = + g_fmemInstance->map(range.beginAddress(), range.size(), allocation, + AllocationFlags::Fixed, 1); return toErrorCode(errc); } diff --git a/kernel/orbis/src/pmem.cpp b/kernel/orbis/src/pmem.cpp index 8a6b76e46..d7abc9bc1 100644 --- a/kernel/orbis/src/pmem.cpp +++ b/kernel/orbis/src/pmem.cpp @@ -1,30 +1,32 @@ #include "pmem.hpp" #include "IoDevice.hpp" +#include "KernelAllocator.hpp" #include "KernelObject.hpp" #include "error.hpp" #include "error/ErrorCode.hpp" +#include "file.hpp" #include "kernel/KernelObject.hpp" #include "kernel/MemoryResource.hpp" #include "rx/AddressRange.hpp" +#include "rx/Rc.hpp" +#include "rx/print.hpp" #include "vmem.hpp" #include #include struct PhysicalMemoryAllocation { - orbis::pmem::MemoryType type = orbis::pmem::MemoryType::Invalid; + bool allocated = false; - [[nodiscard]] bool isAllocated() const { - return type != orbis::pmem::MemoryType::Invalid; - } + [[nodiscard]] bool isAllocated() const { return allocated; } [[nodiscard]] bool isRelated(const PhysicalMemoryAllocation &left, rx::AddressRange, rx::AddressRange) const { - return type == left.type; + return allocated == left.allocated; } [[nodiscard]] PhysicalMemoryAllocation merge(const PhysicalMemoryAllocation &other, rx::AddressRange, rx::AddressRange) const { - assert(other.type == type); + assert(other.allocated == allocated); return other; } @@ -37,12 +39,22 @@ using MappableMemoryResource = })>; using PhysicalMemoryResource = - kernel::AllocableResource; + kernel::AllocableResource; static auto g_pmemInstance = orbis::createGlobalObject< kernel::LockableKernelObject>(); struct PhysicalMemory : orbis::IoDevice { + orbis::File file; + + PhysicalMemory() { + incRef(); // do not delete global object + + file.device = this; + file.incRef(); // do not delete property + } + orbis::ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override { @@ -50,11 +62,12 @@ struct PhysicalMemory : orbis::IoDevice { } orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, - rx::EnumBitSet protection, - orbis::Process *) override { + rx::EnumBitSet protection, + orbis::File *, orbis::Process *) override { return orbis::pmem::map( range.beginAddress(), - rx::AddressRange::fromBeginSize(offset, range.size()), protection); + rx::AddressRange::fromBeginSize(offset, range.size()), + orbis::vmem::toCpuProtection(protection)); } void serialize(rx::Serializer &s) const {} @@ -65,6 +78,8 @@ static auto g_phyMemory = orbis::createGlobalObject(); orbis::ErrorCode orbis::pmem::initialize(std::uint64_t size) { std::lock_guard lock(*g_pmemInstance); + rx::println("pmem: {:x}", size); + return toErrorCode( g_pmemInstance->create(rx::AddressRange::fromBeginSize(0, size))); } @@ -74,11 +89,12 @@ void orbis::pmem::destroy() { g_pmemInstance->destroy(); } -std::pair orbis::pmem::allocate( - std::uint64_t addressHint, std::uint64_t size, MemoryType memoryType, - rx::EnumBitSet flags, std::uint64_t alignment) { +std::pair +orbis::pmem::allocate(std::uint64_t addressHint, std::uint64_t size, + rx::EnumBitSet flags, + std::uint64_t alignment) { std::lock_guard lock(*g_pmemInstance); - PhysicalMemoryAllocation allocation{.type = memoryType}; + PhysicalMemoryAllocation allocation{.allocated = true}; auto [it, errc, range] = g_pmemInstance->map(addressHint, size, allocation, flags, alignment); @@ -99,8 +115,7 @@ orbis::ErrorCode orbis::pmem::deallocate(rx::AddressRange range) { return toErrorCode(errc); } -std::optional -orbis::pmem::query(std::uint64_t address) { +std::optional orbis::pmem::query(std::uint64_t address) { std::lock_guard lock(*g_pmemInstance); auto result = g_pmemInstance->query(address); @@ -108,7 +123,7 @@ orbis::pmem::query(std::uint64_t address) { return {}; } - return AllocatedMemory{.range = result.range(), .memoryType = result->type}; + return result.range(); } orbis::ErrorCode @@ -117,11 +132,11 @@ orbis::pmem::map(std::uint64_t virtualAddress, rx::AddressRange range, auto virtualRange = rx::AddressRange::fromBeginSize(virtualAddress, range.size()); auto errc = g_pmemInstance->mappable.map(virtualRange, range.beginAddress(), - protection, orbis::vmem::kPageSize); + protection, vmem::kPageSize); return toErrorCode(errc); } std::size_t orbis::pmem::getSize() { return g_pmemInstance->size; } orbis::IoDevice *orbis::pmem::getDevice() { return g_phyMemory.get(); } - +orbis::File *orbis::pmem::getFile() { return &g_phyMemory->file; } diff --git a/kernel/orbis/src/sys/sys_event.cpp b/kernel/orbis/src/sys/sys_event.cpp index eab4e0d7d..5e591ccf1 100644 --- a/kernel/orbis/src/sys/sys_event.cpp +++ b/kernel/orbis/src/sys/sys_event.cpp @@ -40,31 +40,33 @@ orbis::SysResult orbis::sys_kqueueex(Thread *thread, ptr name, return {}; } -static bool isReadEventTriggered(int hostFd) { +static bool isReadEventTriggered(const rx::Mappable &hostFd) { #ifdef __linux fd_set fds{}; - FD_SET(hostFd, &fds); + FD_SET(hostFd.native_handle(), &fds); timeval timeout{}; - if (::select(hostFd + 1, &fds, nullptr, nullptr, &timeout) < 0) { + if (::select(hostFd.native_handle() + 1, &fds, nullptr, nullptr, &timeout) < + 0) { return false; } - return FD_ISSET(hostFd, &fds); + return FD_ISSET(hostFd.native_handle(), &fds); #else #warning "Not implemented" return false; #endif } -static bool isWriteEventTriggered(int hostFd) { +static bool isWriteEventTriggered(const rx::Mappable &hostFd) { #ifdef __linux fd_set fds{}; - FD_SET(hostFd, &fds); + FD_SET(hostFd.native_handle(), &fds); timeval timeout{}; - if (::select(hostFd + 1, nullptr, &fds, nullptr, &timeout) < 0) { + if (::select(hostFd.native_handle() + 1, nullptr, &fds, nullptr, &timeout) < + 0) { return false; } - return FD_ISSET(hostFd, &fds); + return FD_ISSET(hostFd.native_handle(), &fds); #else #warning "Not implemented" return false; @@ -131,7 +133,7 @@ static SysResult keventChange(KQueue *kq, KEvent &change, Thread *thread) { eventEmitter->subscribe(&*nodeIt); nodeIt->triggered = true; kq->cv.notify_all(kq->mtx); - } else if (note.file->hostFd < 0) { + } else if (!note.file->hostFd) { ORBIS_LOG_ERROR("Unimplemented event emitter", change.ident); } } else if (change.filter == kEvFiltGraphicsCore || @@ -303,7 +305,7 @@ orbis::SysResult orbis::sys_kevent(Thread *thread, sint fd, if (!note.triggered) { if (note.event.filter == kEvFiltRead) { - if (note.file->hostFd >= 0) { + if (note.file->hostFd) { if (isReadEventTriggered(note.file->hostFd)) { note.triggered = true; } else { @@ -311,7 +313,7 @@ orbis::SysResult orbis::sys_kevent(Thread *thread, sint fd, } } } else if (note.event.filter == kEvFiltWrite) { - if (note.file->hostFd >= 0) { + if (note.file->hostFd) { if (isWriteEventTriggered(note.file->hostFd)) { note.triggered = true; } else { diff --git a/kernel/orbis/src/sys/sys_sce.cpp b/kernel/orbis/src/sys/sys_sce.cpp index a25d3ce66..e13530cc9 100644 --- a/kernel/orbis/src/sys/sys_sce.cpp +++ b/kernel/orbis/src/sys/sys_sce.cpp @@ -7,6 +7,9 @@ #include "module/ModuleInfoEx.hpp" #include "orbis/time.hpp" #include "osem.hpp" +#include "pmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/ProcessOps.hpp" @@ -14,6 +17,7 @@ #include "ucontext.hpp" #include "uio.hpp" #include "utils/Logs.hpp" +#include "vmem.hpp" #include #include #include @@ -69,8 +73,12 @@ orbis::SysResult orbis::sys_netgetiflist(Thread *thread /* TODO */) { return {}; } -orbis::SysResult orbis::sys_mtypeprotect(Thread *thread /* TODO */) { - return ErrorCode::NOSYS; +orbis::SysResult +orbis::sys_mtypeprotect(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, + rx::EnumBitSet prot) { + auto range = rx::AddressRange::fromBeginSize(addr, len); + return vmem::setTypeAndProtect(thread->tproc, range, type, prot); } orbis::SysResult orbis::sys_regmgr_call(Thread *thread, uint32_t op, uint32_t id, ptr result, @@ -393,19 +401,20 @@ orbis::SysResult orbis::sys_evf_cancel(Thread *thread, sint id, uint64_t value, return {}; } orbis::SysResult -orbis::sys_query_memory_protection(Thread *thread, ptr address, - ptr protection) { - if (auto query_memory_protection = - thread->tproc->ops->query_memory_protection) { - return query_memory_protection(thread, address, protection); +orbis::sys_query_memory_protection(Thread *thread, uintptr_t address, + ptr protection) { + auto result = vmem::queryProtection(thread->tproc, address); + + if (!result) { + return ErrorCode::ACCES; } - return ErrorCode::NOSYS; + return uwrite(protection, *result); } namespace orbis { struct BatchMapEntry { - ptr start; + uintptr_t start; off_t offset; size_t length; char protection; @@ -419,14 +428,14 @@ struct BatchMapEntry { }; } // namespace orbis -orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, sint flags, +orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, + rx::EnumBitSet flags, ptr entries, sint entriesCount, ptr processedCount) { - auto ops = thread->tproc->ops; SysResult result = ErrorCode{}; - ORBIS_LOG_ERROR(__FUNCTION__, unk, flags, entries, entriesCount, - processedCount); + ORBIS_LOG_ERROR(__FUNCTION__, unk, flags.toUnderlying(), entries, + entriesCount, processedCount); int processed = 0; for (int i = 0; i < entriesCount; ++i) { @@ -438,16 +447,29 @@ orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, sint flags, switch (_entry.operation) { case 0: - result = ops->dmem_mmap(thread, _entry.start, _entry.length, _entry.type, - _entry.protection, flags, _entry.offset); - break; + result = sys_mmap_dmem( + thread, _entry.start, _entry.length, + static_cast(_entry.type), + rx::EnumBitSet::fromUnderlying(_entry.protection), + flags, _entry.offset); case 1: - result = ops->munmap(thread, _entry.start, _entry.length); + result = sys_munmap(thread, _entry.start, _entry.length); break; case 2: - result = - ops->mprotect(thread, _entry.start, _entry.length, _entry.protection); + result = sys_mprotect( + thread, _entry.start, _entry.length, + rx::EnumBitSet::fromUnderlying(_entry.protection)); break; + case 3: + result = sys_mmap( + thread, _entry.start, _entry.length, + rx::EnumBitSet::fromUnderlying(_entry.protection), + vmem::MapFlags::Void | vmem::MapFlags::Anon | flags, -1, 0); + break; + case 4: + ORBIS_LOG_ERROR(__FUNCTION__, "unimplemented type protect"); + break; + default: result = ErrorCode::INVAL; break; @@ -880,14 +902,22 @@ orbis::SysResult orbis::sys_budget_set(Thread *thread, sint budgetId) { thread->tproc->budgetId = budgetId; return {}; } -orbis::SysResult orbis::sys_virtual_query(Thread *thread, ptr addr, - uint64_t unk, ptr info, + +orbis::SysResult orbis::sys_virtual_query(Thread *thread, uintptr_t addr, + uint64_t flags, ptr info, size_t infosz) { - if (auto virtual_query = thread->tproc->ops->virtual_query) { - return virtual_query(thread, addr, unk, info, infosz); + + if (infosz != sizeof(vmem::QueryResult)) { + return ErrorCode::INVAL; } - return ErrorCode::NOSYS; + auto result = vmem::query(thread->tproc, addr, flags & 1); + + if (!result || result->start >= 0x800000000) { + return ErrorCode::ACCES; + } + + return uwrite(ptr(info), *result); } orbis::SysResult orbis::sys_mdbg_call(Thread *thread /* TODO */) { return {}; } orbis::SysResult orbis::sys_obs_sblock_create(Thread *thread /* TODO */) { @@ -930,7 +960,7 @@ orbis::SysResult orbis::sys_is_in_sandbox(Thread *thread /* TODO */) { } orbis::SysResult orbis::sys_dmem_container(Thread *thread, uint id) { ORBIS_LOG_NOTICE(__FUNCTION__, id); - thread->retval[0] = 1; // returns default direct memory device + thread->retval[0] = 0; // returns default direct memory device if (id + 1) return ErrorCode::PERM; return {}; @@ -955,24 +985,11 @@ orbis::SysResult orbis::sys_mname(Thread *thread, uint64_t addr, uint64_t len, return result; } - NamedMemoryRange range; - range.begin = addr & ~0x3fffull; - range.end = range.begin; - range.end += ((addr & 0x3fff) + 0x3fff + len) & ~0x3fffull; - - std::lock_guard lock(thread->tproc->namedMemMutex); - auto [it, end] = thread->tproc->namedMem.equal_range(range); - while (it != end) { - auto [addr2, end2] = it->first; - auto len2 = end2 - addr2; - ORBIS_LOG_NOTICE("sys_mname: removed overlapped", it->second, addr2, len2); - it = thread->tproc->namedMem.erase(it); + auto range = rx::AddressRange::fromBeginSize(addr, len); + if (auto error = vmem::setName(thread->tproc, range, _name); + error != ErrorCode{}) { + ORBIS_LOG_ERROR(__FUNCTION__, "failure:", static_cast(error)); } - if (!thread->tproc->namedMem.try_emplace(range, _name).second) - std::abort(); - if (!thread->tproc->namedMem.count(addr)) - std::abort(); - return {}; } orbis::SysResult orbis::sys_dynlib_dlopen(Thread *thread /* TODO */) { @@ -1019,7 +1036,7 @@ orbis::SysResult orbis::sys_dynlib_get_info(Thread *thread, ModuleInfo result = {}; result.size = sizeof(ModuleInfo); - std::strncpy(result.name, module->moduleName, sizeof(result.name)); + std::strncpy(result.name, module->soName, sizeof(result.name)); std::memcpy(result.segments, module->segments, sizeof(ModuleSegment) * module->segmentCount); result.segmentCount = module->segmentCount; @@ -1485,14 +1502,56 @@ orbis::SysResult orbis::sys_get_cpu_usage_all(Thread *thread, uint32_t unk, ORBIS_LOG_TODO(__FUNCTION__, unk, result); return {}; } -orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, caddr_t addr, size_t len, - sint memoryType, sint prot, sint flags, +orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr, + size_t len, MemoryType memoryType, + rx::EnumBitSet prot, + rx::EnumBitSet flags, off_t directMemoryStart) { - if (auto dmem_mmap = thread->tproc->ops->dmem_mmap) { - return dmem_mmap(thread, addr, len, memoryType, prot, flags, - directMemoryStart); + + auto callerAddress = getCallerAddress(thread); + auto alignment = dmem::kPageSize; + + { + auto unpacked = unpackMapFlags(flags, dmem::kPageSize); + alignment = unpacked.first; + flags = unpacked.second; } - return ErrorCode::NOSYS; + + len = rx::alignUp(len, dmem::kPageSize); + + rx::EnumBitSet allocFlags{}; + + if (!prot) { + // HACK + // FIXME: implement protect for pid + prot = vmem::Protection::CpuRead | vmem::Protection::CpuWrite | + vmem::Protection::GpuRead | vmem::Protection::GpuWrite; + } + + if (flags & vmem::MapFlags::Fixed) { + allocFlags = AllocationFlags::Fixed; + } else if (addr == 0) { + addr = 0xfe0000000; + } + + if (flags & vmem::MapFlags::NoOverwrite) { + allocFlags |= AllocationFlags::NoOverwrite; + } + + auto name = + callerAddress ? rx::format("anon:{:012x}", callerAddress) : ""; + + auto [range, errc] = + vmem::mapDirect(thread->tproc, addr, + rx::AddressRange::fromBeginSize(directMemoryStart, len), + prot, allocFlags, name, alignment, memoryType); + + if (errc != ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress(); + return {}; } orbis::SysResult orbis::sys_physhm_open(Thread *thread /* TODO */) { return ErrorCode::NOSYS; @@ -1694,31 +1753,53 @@ orbis::SysResult orbis::sys_process_terminate(Thread *thread /* TODO */) { return ErrorCode::NOSYS; } orbis::SysResult orbis::sys_blockpool_open(Thread *thread) { - if (auto blockpool_open = thread->tproc->ops->blockpool_open) { - rx::Ref file; - auto result = blockpool_open(thread, &file); - if (result.isError()) { - return result; - } + if (!g_context->blockpool) { + return ErrorCode::NOSYS; + } - thread->retval[0] = thread->tproc->fileDescriptors.insert(file); - return {}; + auto fd = thread->tproc->fileDescriptors.insert(g_context->blockpool); + + if (fd == thread->tproc->fileDescriptors.npos) { + return ErrorCode::MFILE; } - return ErrorCode::NOSYS; + + thread->retval[0] = fd; + return {}; } -orbis::SysResult orbis::sys_blockpool_map(Thread *thread, caddr_t addr, - size_t len, sint prot, sint flags) { - if (auto blockpool_map = thread->tproc->ops->blockpool_map) { - return blockpool_map(thread, addr, len, prot, flags); +orbis::SysResult +orbis::sys_blockpool_map(Thread *thread, uintptr_t addr, size_t len, + MemoryType type, rx::EnumBitSet prot, + rx::EnumBitSet flags) { + rx::EnumBitSet allocFlags{}; + + if (flags & vmem::MapFlags::Fixed) { + allocFlags |= AllocationFlags::Fixed; } - return ErrorCode::NOSYS; + if (flags & vmem::MapFlags::NoOverwrite) { + allocFlags |= AllocationFlags::NoOverwrite; + } + if (flags & vmem::MapFlags::NoCoalesce) { + allocFlags |= AllocationFlags::NoMerge; + } + + auto [range, errc] = vmem::commitPooled( + thread->tproc, rx::AddressRange::fromBeginSize(addr, len), type, prot); + + if (errc != ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress(); + return {}; } -orbis::SysResult orbis::sys_blockpool_unmap(Thread *thread, caddr_t addr, +orbis::SysResult orbis::sys_blockpool_unmap(Thread *thread, uintptr_t addr, size_t len, sint flags) { - if (auto blockpool_unmap = thread->tproc->ops->blockpool_unmap) { - return blockpool_unmap(thread, addr, len); + if (flags != 0) { + return ErrorCode::INVAL; } - return ErrorCode::NOSYS; + + return vmem::decommitPooled(thread->tproc, + rx::AddressRange::fromBeginSize(addr, len)); } orbis::SysResult orbis::sys_dynlib_get_info_for_libdbg(Thread *thread /* TODO */) { diff --git a/kernel/orbis/src/sys/sys_sysctl.cpp b/kernel/orbis/src/sys/sys_sysctl.cpp index 5015f136c..4515cd6b9 100644 --- a/kernel/orbis/src/sys/sys_sysctl.cpp +++ b/kernel/orbis/src/sys/sys_sysctl.cpp @@ -1,4 +1,5 @@ #include "KernelContext.hpp" +#include "rx/print.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/Thread.hpp" @@ -749,9 +750,8 @@ SysResult kern_sysctl(Thread *thread, ptr name, uint namelen, return ErrorCode::INVAL; } - std::printf("Reporting stack at %p\n", thread->stackEnd); - *(ptr *)old = thread->stackEnd; - return {}; + rx::println(stderr, "Reporting stack at {:x}", thread->stackEnd); + return uwrite(ptr(old), thread->stackEnd); } case sysctl_kern::smp_cpus: diff --git a/kernel/orbis/src/sys/sys_vm_mmap.cpp b/kernel/orbis/src/sys/sys_vm_mmap.cpp index 0c8fa67fe..309807761 100644 --- a/kernel/orbis/src/sys/sys_vm_mmap.cpp +++ b/kernel/orbis/src/sys/sys_vm_mmap.cpp @@ -1,8 +1,16 @@ +#include "KernelContext.hpp" #include "error.hpp" +#include "rx/AddressRange.hpp" +#include "rx/align.hpp" +#include "rx/debug.hpp" +#include "rx/format.hpp" +#include "rx/print.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/ProcessOps.hpp" #include "thread/Thread.hpp" +#include "utils/Logs.hpp" +#include "vmem.hpp" orbis::SysResult orbis::sys_sbrk(Thread *, sint) { return ErrorCode::OPNOTSUPP; @@ -11,94 +19,179 @@ orbis::SysResult orbis::sys_sstk(Thread *, sint) { return ErrorCode::OPNOTSUPP; } -orbis::SysResult orbis::sys_mmap(Thread *thread, caddr_t addr, size_t len, - sint prot, sint flags, sint fd, off_t pos) { - if (auto impl = thread->tproc->ops->mmap) { - return impl(thread, addr, len, prot, flags, fd, pos); +orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, + off_t pos) { + + std::uint64_t callerAddress = getCallerAddress(thread); + + auto shift = addr & (vmem::kPageSize - 1); + addr = rx::alignDown(addr, vmem::kPageSize); + rx::EnumBitSet allocFlags{}; + rx::EnumBitSet blockFlags{}; + rx::EnumBitSet blockFlagsEx{}; + std::uint64_t alignment = vmem::kPageSize; + + { + auto unpacked = unpackMapFlags(flags, vmem::kPageSize); + alignment = unpacked.first; + flags = unpacked.second; } - return ErrorCode::NOSYS; + len = rx::alignUp(len, vmem::kPageSize); + + if (flags & vmem::MapFlags::Stack) { + allocFlags |= AllocationFlags::Stack; + } + + if (flags & vmem::MapFlags::Fixed) { + allocFlags |= AllocationFlags::Fixed; + } + + if (flags & vmem::MapFlags::NoOverwrite) { + allocFlags |= AllocationFlags::NoOverwrite; + } + + if (flags & vmem::MapFlags::NoCoalesce) { + allocFlags |= AllocationFlags::NoMerge; + } + + if (flags & vmem::MapFlags::Shared) { + blockFlagsEx |= vmem::BlockFlagsEx::Shared; + + if (flags & vmem::MapFlags::Private) { + return ErrorCode::INVAL; + } + } + + if (flags & vmem::MapFlags::Private) { + blockFlagsEx |= vmem::BlockFlagsEx::Private; + } + + if (addr == 0) { + addr = flags & vmem::MapFlags::System ? 0xfc0000000 : 0x200000000; + } + + if (flags & vmem::MapFlags::Void) { + flags |= vmem::MapFlags::Anon; + + if (fd != -1 || pos != 0) { + return ErrorCode::INVAL; + } + } + + if (flags & vmem::MapFlags::Stack) { + flags |= vmem::MapFlags::Anon; + blockFlags |= vmem::BlockFlags::Stack; + + if (fd != -1 || pos != 0) { + return ErrorCode::INVAL; + } + + if ((prot & (vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) != + (vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) { + return ErrorCode::INVAL; + } + } + + auto name = callerAddress ? rx::format("anon:{:012x}", callerAddress) : ""; + + if (flags & vmem::MapFlags::Anon) { + if (fd != -1 || pos != 0) { + return ErrorCode::INVAL; + } + + if (prot & (vmem::Protection::GpuRead | vmem::Protection::GpuWrite)) { + return ErrorCode::INVAL; + } + + auto [range, errc] = vmem::mapFlex(thread->tproc, len, prot, addr, + allocFlags, blockFlags, name, alignment); + + if (errc != orbis::ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress() + shift; + return {}; + } + + auto file = thread->tproc->fileDescriptors.get(fd); + if (file == nullptr) { + return ErrorCode::BADF; + } + + if (!file->device->blockFlags) { + blockFlags |= vmem::BlockFlags::FlexibleMemory; + } + + prot &= ~vmem::Protection::CpuExec; + + auto [range, errc] = + vmem::mapFile(thread->tproc, addr, len, allocFlags, prot, blockFlags, + blockFlagsEx, file.get(), pos, name, alignment); + + if (errc != ErrorCode{}) { + return errc; + } + + thread->retval[0] = range.beginAddress() + shift; + return {}; } -orbis::SysResult orbis::sys_freebsd6_mmap(Thread *thread, caddr_t addr, - size_t len, sint prot, sint flags, +orbis::SysResult orbis::sys_freebsd6_mmap(Thread *thread, uintptr_t addr, + size_t len, + rx::EnumBitSet prot, + rx::EnumBitSet flags, sint fd, sint, off_t pos) { return sys_mmap(thread, addr, len, prot, flags, fd, pos); } -orbis::SysResult orbis::sys_msync(Thread *thread, ptr addr, size_t len, +orbis::SysResult orbis::sys_msync(Thread *thread, uintptr_t addr, size_t len, sint flags) { - if (auto impl = thread->tproc->ops->msync) { - return impl(thread, addr, len, flags); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__, addr, len, flags); + return {}; } -orbis::SysResult orbis::sys_munmap(Thread *thread, ptr addr, size_t len) { - if (auto impl = thread->tproc->ops->munmap) { - return impl(thread, addr, len); - } +orbis::SysResult orbis::sys_munmap(Thread *thread, uintptr_t addr, size_t len) { + auto range = rx::AddressRange::fromBeginSize(addr, len); - return ErrorCode::NOSYS; + return vmem::unmap(thread->tproc, range); } -orbis::SysResult orbis::sys_mprotect(Thread *thread, ptr addr, - size_t len, sint prot) { - if (auto impl = thread->tproc->ops->mprotect) { - return impl(thread, addr, len, prot); - } - - return ErrorCode::NOSYS; +orbis::SysResult orbis::sys_mprotect(Thread *thread, uintptr_t addr, size_t len, + rx::EnumBitSet prot) { + auto range = rx::AddressRange::fromBeginSize(addr, len); + return vmem::protect(thread->tproc, range, prot); } -orbis::SysResult orbis::sys_minherit(Thread *thread, ptr addr, size_t len, +orbis::SysResult orbis::sys_minherit(Thread *thread, uintptr_t addr, size_t len, sint inherit) { - if (auto impl = thread->tproc->ops->minherit) { - return impl(thread, addr, len, inherit); - } + ORBIS_LOG_TODO(__FUNCTION__, addr, len, inherit); return ErrorCode::NOSYS; } -orbis::SysResult orbis::sys_madvise(Thread *thread, ptr addr, size_t len, +orbis::SysResult orbis::sys_madvise(Thread *thread, uintptr_t addr, size_t len, sint behav) { - if (auto impl = thread->tproc->ops->madvise) { - return impl(thread, addr, len, behav); - } - + ORBIS_LOG_TODO(__FUNCTION__, addr, len, behav); return ErrorCode::NOSYS; } -orbis::SysResult orbis::sys_mincore(Thread *thread, ptr addr, - size_t len, ptr vec) { - if (auto impl = thread->tproc->ops->mincore) { - return impl(thread, addr, len, vec); - } - +orbis::SysResult orbis::sys_mincore(Thread *thread, uintptr_t addr, size_t len, + ptr vec) { + ORBIS_LOG_TODO(__FUNCTION__, addr, len, vec); return ErrorCode::NOSYS; } -orbis::SysResult orbis::sys_mlock(Thread *thread, ptr addr, - size_t len) { - if (auto impl = thread->tproc->ops->mlock) { - return impl(thread, addr, len); - } - - return ErrorCode::NOSYS; +orbis::SysResult orbis::sys_mlock(Thread *thread, uintptr_t addr, size_t len) { + ORBIS_LOG_TODO(__FUNCTION__, addr, len); + return {}; } orbis::SysResult orbis::sys_mlockall(Thread *thread, sint how) { - if (auto impl = thread->tproc->ops->mlockall) { - return impl(thread, how); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__, how); + return {}; } orbis::SysResult orbis::sys_munlockall(Thread *thread) { - if (auto impl = thread->tproc->ops->munlockall) { - return impl(thread); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__); + return {}; } -orbis::SysResult orbis::sys_munlock(Thread *thread, ptr addr, +orbis::SysResult orbis::sys_munlock(Thread *thread, uintptr_t addr, size_t len) { - if (auto impl = thread->tproc->ops->munlock) { - return impl(thread, addr, len); - } - - return ErrorCode::NOSYS; + ORBIS_LOG_TODO(__FUNCTION__, addr, len); + return {}; } diff --git a/kernel/orbis/src/sysvec.cpp b/kernel/orbis/src/sysvec.cpp index 786531131..c000ae180 100644 --- a/kernel/orbis/src/sysvec.cpp +++ b/kernel/orbis/src/sysvec.cpp @@ -1,10 +1,15 @@ +#include "orbis-config.hpp" +#include "rx/EnumBitSet.hpp" +#include "rx/format.hpp" #include "sys/syscall.hpp" #include "sys/sysentry.hpp" #include "sys/sysproto.hpp" #include "thread/Process.hpp" #include "thread/Thread.hpp" #include +#include #include +#include enum { PSL_C = 0x1 }; @@ -101,6 +106,72 @@ struct WrapImpl { sysent result; result.narg = sizeof...(Args); result.call = &WrapImpl::call; + result.format = [](uint64_t *values) -> std::string { + std::string result = getSysentName(&WrapImpl::call); + result += "("; + + auto formatArg = + [&](std::integral_constant, + T value, std::uint64_t raw) { + if (I != 0) { + result += ", "; + } + + using type = std::remove_cvref_t; + + if constexpr (std::is_same_v) { + if (value) { + result += "true"; + } else { + result += "false"; + } + } else if constexpr (std::is_integral_v) { + if (value > 9) { + result += rx::format("{:#x}", value); + } else { + result += rx::format("{}", value); + } + } else if constexpr (std::is_pointer_v) { + result += rx::format("{}", (void *)value); + + // using pointee = std::remove_cvref_t>; + // if constexpr (requires(rx::format_parse_context &ctx) { + // rx::formatter().parse(ctx); + // }) { + // if (value) { + // pointee kernelValue; + // auto errc = orbis::uread(kernelValue, value); + + // if (errc == ErrorCode{}) { + // result += rx::format("={}", kernelValue); + // } else { + // result += rx::format("={}", errc); + // } + // } + // } + } else if constexpr (requires(rx::format_parse_context &ctx) { + rx::formatter().parse(ctx); + }) { + result += rx::format("{}", value); + } else { + if (raw > 9) { + result += rx::format("{:#x}", raw); + } else { + result += rx::format("{}", raw); + } + } + }; + auto formatArgs = [&](std::index_sequence, + uint64_t *values) { + (formatArg(std::integral_constant{}, + makeArg(values[I]), values[I]), + ...); + }; + + formatArgs(std::make_index_sequence{}, values); + result += ")"; + return result; + }; return result; } @@ -113,7 +184,28 @@ private: template static SysResult callImpl(Thread *thread, uint64_t *args, std::index_sequence) { - return Fn(thread, Args(args[I])...); + return Fn(thread, makeArg(args[I])...); + } + + template + static T makeArg(uint64_t raw) + requires requires { std::bit_cast(raw); } + { + return std::bit_cast(raw); + } + + template + static T makeArg(uint64_t raw) + requires requires { T(raw); } && (!requires { std::bit_cast(raw); }) + { + return T(raw); + } + + template + static T makeArg(uint64_t raw) + requires requires { T(T::fromUnderlying(raw)); } + { + return T::fromUnderlying(raw); } }; } // namespace detail diff --git a/kernel/orbis/src/thread/Process.cpp b/kernel/orbis/src/thread/Process.cpp index 5d6cbc605..9d1bebdec 100644 --- a/kernel/orbis/src/thread/Process.cpp +++ b/kernel/orbis/src/thread/Process.cpp @@ -177,3 +177,9 @@ orbis::Process *orbis::createProcess(Process *parentProcess, pid_t pid) { g_processList->list = result; return &result->object; } + +orbis::Budget *orbis::Process::getBudget() const { + auto result = g_context->budgets.get(budgetId); + + return result != nullptr ? result.get() : nullptr; +} diff --git a/kernel/orbis/src/thread/Thread.cpp b/kernel/orbis/src/thread/Thread.cpp index 579a2cd47..124cab0d5 100644 --- a/kernel/orbis/src/thread/Thread.cpp +++ b/kernel/orbis/src/thread/Thread.cpp @@ -36,3 +36,35 @@ orbis::Thread *orbis::createThread(Process *process, std::string_view name) { return result; } + +uintptr_t orbis::getCallerAddress(Thread *thread) { + if (!thread->context) { + return 0; + } + + auto rbp = readRegister(thread->context, RegisterId::rbp); + std::uint64_t result = 0; + + while (rbp < 0x8000'0000'0000) { + auto framePtr = std::bit_cast>(rbp); + + std::uint64_t nextFrame; + std::uint64_t retAddress; + if (orbis::uread(retAddress, framePtr + 1) != orbis::ErrorCode{}) { + break; + } + + if (orbis::uread(nextFrame, framePtr) != orbis::ErrorCode{}) { + break; + } + + if (!thread->tproc->libkernelRange.contains(retAddress)) { + result = retAddress; + break; + } + + rbp = nextFrame; + } + + return result; +} diff --git a/kernel/orbis/src/vmem.cpp b/kernel/orbis/src/vmem.cpp index 70f663e7b..545b4ed83 100644 --- a/kernel/orbis/src/vmem.cpp +++ b/kernel/orbis/src/vmem.cpp @@ -1,11 +1,25 @@ #include "vmem.hpp" +#include "Budget.hpp" #include "IoDevice.hpp" +#include "KernelAllocator.hpp" +#include "KernelContext.hpp" #include "KernelObject.hpp" +#include "MemoryType.hpp" +#include "blockpool.hpp" +#include "dmem.hpp" #include "error.hpp" +#include "fmem.hpp" #include "pmem.hpp" -#include "rx/Mappable.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" +#include "rx/FileLock.hpp" +#include "rx/FunctionRef.hpp" #include "rx/Rc.hpp" +#include "rx/Serializer.hpp" #include "rx/StaticString.hpp" +#include "rx/align.hpp" +#include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/mem.hpp" #include "rx/print.hpp" #include "thread/Process.hpp" @@ -13,28 +27,48 @@ #include #include +// FIXME: remove +namespace amdgpu { +void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset); +void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange); +void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot); +} // namespace amdgpu + struct VirtualMemoryAllocation { rx::EnumBitSet flags{}; + rx::EnumBitSet flagsEx{}; rx::EnumBitSet prot{}; + orbis::MemoryType type = orbis::MemoryType::Invalid; rx::Ref device; std::uint64_t deviceOffset = 0; rx::StaticString<31> name; [[nodiscard]] bool isAllocated() const { - return (flags & orbis::vmem::BlockFlags::Allocated) == - orbis::vmem::BlockFlags::Allocated; + return (flagsEx & orbis::vmem::BlockFlagsEx::Allocated) == + orbis::vmem::BlockFlagsEx::Allocated; } [[nodiscard]] bool isRelated(const VirtualMemoryAllocation &other, rx::AddressRange selfRange, [[maybe_unused]] rx::AddressRange rightRange) const { - if (flags != other.flags || prot != other.prot || device != other.device || + if (flags != other.flags || flagsEx != other.flagsEx || + prot != other.prot || type != other.type || device != other.device || name != other.name) { return false; } + if (!isAllocated()) { + return true; + } - return !isAllocated() || - deviceOffset + selfRange.size() == other.deviceOffset; + if (device == nullptr || flags == orbis::vmem::BlockFlags::PooledMemory) { + return true; + } + + return deviceOffset + selfRange.size() == other.deviceOffset; } [[nodiscard]] VirtualMemoryAllocation merge(const VirtualMemoryAllocation &, @@ -43,10 +77,10 @@ struct VirtualMemoryAllocation { return *this; } - std::pair - truncate(rx::AddressRange selfRange, rx::AddressRange, - rx::AddressRange rightRange) { - if (!isAllocated() || !rightRange.isValid() || device == nullptr) { + [[nodiscard]] std::pair + truncate(rx::AddressRange selfRange, rx::AddressRange leftRange, + rx::AddressRange rightRange) const { + if (!rightRange.isValid() || device == nullptr) { return {}; } @@ -59,19 +93,197 @@ struct VirtualMemoryAllocation { } bool operator==(const VirtualMemoryAllocation &) const = default; + + void setName(orbis::Process *process, std::string_view newName) { + if (!newName.empty()) { + name = newName; + return; + } + + name = "(NoName)"; + + if (auto executable = process->modulesMap.get(orbis::ModuleHandle{})) { + name += executable->soName; + } + } + + void serialize(rx::Serializer &s) const {} + void deserialize(rx::Deserializer &d) {} }; -using MappableMemoryResource = - kernel::MappableResource; - struct VirtualMemoryResource - : kernel::AllocableResource {}; + : kernel::AllocableResource {}; static auto g_vmInstance = orbis::createProcessLocalObject< kernel::LockableKernelObject>(); +static void vmemDump(orbis::Process *process, std::string_view message = {}) { + rx::ScopedFileLock lock(stderr); + auto vmem = process->get(g_vmInstance); + + rx::println(stderr, "vmem {} {}", process->pid, message); + for (auto alloc : *vmem) { + if (!alloc->isAllocated()) { + continue; + } + + rx::print(stderr, " {:012x}-{:012x}: {:08x}", alloc.beginAddress(), + alloc.endAddress(), + alloc->flags & orbis::vmem::BlockFlags::FlexibleMemory + ? 0 + : alloc->deviceOffset); + + rx::print( + stderr, " {}{}{}{}{}", + "_P"[alloc->flags.any_of(orbis::vmem::BlockFlags::PooledMemory)], + "_D"[alloc->flags.any_of(orbis::vmem::BlockFlags::DirectMemory)], + "_F"[alloc->flags.any_of(orbis::vmem::BlockFlags::FlexibleMemory)], + "_S"[alloc->flags.any_of(orbis::vmem::BlockFlags::Stack)], + "_C"[alloc->flags.any_of(orbis::vmem::BlockFlags::Commited)]); + + rx::print(stderr, " {}{}{} {}{}", + "_R"[alloc->prot.any_of(orbis::vmem::Protection::CpuRead)], + "_W"[alloc->prot.any_of(orbis::vmem::Protection::CpuWrite)], + "_X"[alloc->prot.any_of(orbis::vmem::Protection::CpuExec)], + "_R"[alloc->prot.any_of(orbis::vmem::Protection::GpuRead)], + "_W"[alloc->prot.any_of(orbis::vmem::Protection::GpuWrite)]); + + rx::print( + stderr, " {}{}{}{}{} {}", + "_A"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Allocated)], + "_P"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Private)], + "_S"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Shared)], + "_C"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::PoolControl)], + "_R"[alloc->flagsEx.any_of(orbis::vmem::BlockFlagsEx::Reserved)], + alloc->type); + + rx::println(stderr, " {}", alloc->name); + } +} + +static orbis::ErrorCode validateOverwrite(decltype(g_vmInstance)::type *vmem, + rx::AddressRange range, + bool isUnmap) { + for (auto it = vmem->lowerBound(range.beginAddress()); + it != vmem->end() && it.beginAddress() < range.endAddress(); ++it) { + if (!it->isAllocated()) { + continue; + } + + if (it->flagsEx & orbis::vmem::BlockFlagsEx::Reserved) { + return orbis::ErrorCode::ACCES; + } + + if (it->flags & orbis::vmem::BlockFlags::PooledMemory) { + if (!isUnmap) { + return orbis::ErrorCode::ACCES; + } + + while (!(it->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl)) { + ++it; + + if (it == vmem->end()) { + return orbis::ErrorCode::ACCES; + } + + if (!it->isAllocated() || + !(it->flags & orbis::vmem::BlockFlags::PooledMemory)) { + --it; + break; + } + } + + if ((it->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl) && + it->deviceOffset != 0) { + // control block has allocations, verify that whole reserved range going + // to be unmapped + + auto reservedRange = rx::AddressRange::fromBeginEnd( + it.beginAddress() - it->deviceOffset, it.endAddress()); + + if (!range.contains(reservedRange)) { + return orbis::ErrorCode::ACCES; + } + } + } + } + + return {}; +} + +static void release(orbis::Process *process, decltype(g_vmInstance)::type *vmem, + orbis::Budget *budget, rx::AddressRange range) { + for (auto it = vmem->lowerBound(range.beginAddress()); + it != vmem->end() && it.beginAddress() < range.endAddress(); ++it) { + if (!it->isAllocated()) { + continue; + } + + auto blockRange = range.intersection(it.range()); + + if (it->flags & orbis::vmem::BlockFlags::FlexibleMemory) { + if (it->device == nullptr) { + orbis::fmem::deallocate(blockRange); + } + budget->release(orbis::BudgetResource::Fmem, blockRange.size()); + } + + if (it->flags & orbis::vmem::BlockFlags::DirectMemory) { + budget->release(orbis::BudgetResource::Dmem, blockRange.size()); + } + + if (it->flags & orbis::vmem::BlockFlags::PooledMemory) { + if (it->flags & orbis::vmem::BlockFlags::Commited) { + orbis::blockpool::decommit(process, it.range()); + } else if (it->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl) { + orbis::blockpool::releaseControlBlock(); + } + } + + if (orbis::vmem::toGpuProtection(it->prot) && + (it->flags & orbis::vmem::BlockFlags::DirectMemory | + orbis::vmem::BlockFlags::PooledMemory)) { + amdgpu::unmapMemory(process->pid, blockRange); + } + } +} + +static orbis::ErrorCode validateRange( + decltype(g_vmInstance)::type *vmem, + decltype(g_vmInstance)::type::iterator it, rx::AddressRange range, + rx::FunctionRef cb) { + while (it != vmem->end() && it.beginAddress() < range.endAddress()) { + if (auto errc = cb(it.get()); errc != orbis::ErrorCode{}) { + return errc; + } + + ++it; + } + + return {}; +} + +static void modifyRange( + decltype(g_vmInstance)::type *vmem, + decltype(g_vmInstance)::type::iterator it, rx::AddressRange range, + rx::FunctionRef cb) { + while (it != vmem->end() && it.beginAddress() < range.endAddress()) { + auto mapRange = range.intersection(it.range()); + auto allocInfo = it.get(); + + if (allocInfo.device != nullptr && + !(allocInfo.flags & orbis::vmem::BlockFlags::PooledMemory)) { + allocInfo.deviceOffset += mapRange.beginAddress() - it.beginAddress(); + } + + cb(allocInfo, mapRange); + vmem->map(mapRange.beginAddress(), mapRange.size(), allocInfo, + orbis::AllocationFlags::Fixed, orbis::vmem::kPageSize); + + ++it; + } +} + void orbis::vmem::initialize(Process *process, bool force) { auto vmem = process->get(g_vmInstance); @@ -125,37 +337,60 @@ void orbis::vmem::fork(Process *process, Process *parentThread) { // FIXME: implement } -std::pair orbis::vmem::map( +std::pair orbis::vmem::reserve( Process *process, std::uint64_t addressHint, std::uint64_t size, - rx::EnumBitSet allocFlags, rx::EnumBitSet prot, - rx::EnumBitSet blockFlags, std::uint64_t alignment, - std::string_view name, IoDevice *device, std::int64_t deviceOffset) { - auto vmem = process->get(g_vmInstance); + rx::EnumBitSet allocFlags, + rx::EnumBitSet blockFlagsEx, std::uint64_t alignment) { + VirtualMemoryAllocation allocationInfo; + allocationInfo.flagsEx = blockFlagsEx | BlockFlagsEx::Allocated; + auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - VirtualMemoryAllocation allocationInfo; - allocationInfo.flags = blockFlags | orbis::vmem::BlockFlags::Allocated; - allocationInfo.device = device; - allocationInfo.prot = prot; - allocationInfo.deviceOffset = deviceOffset; - allocationInfo.name = name; + auto [_, errc, range] = + vmem->map(addressHint, size, allocationInfo, allocFlags, alignment); - if (device == nullptr) { - if (deviceOffset != 0 || prot) { - return {{}, ErrorCode::INVAL}; - } - - auto [_, errc, range] = - vmem->map(addressHint, size, allocationInfo, allocFlags, alignment); - - if (errc != std::errc{}) { - return {{}, toErrorCode(errc)}; - } - - return {range, {}}; + if (errc != std::errc{}) { + return {{}, toErrorCode(errc)}; } + return {range, {}}; +} + +std::pair orbis::vmem::mapFile( + Process *process, std::uint64_t addressHint, std::uint64_t size, + rx::EnumBitSet allocFlags, rx::EnumBitSet prot, + rx::EnumBitSet blockFlags, + rx::EnumBitSet blockFlagsEx, File *file, + std::uint64_t fileOffset, std::string_view name, std::uint64_t alignment, + MemoryType type) { + blockFlags |= file->device->blockFlags; + + if (blockFlags & BlockFlags::PooledMemory) { + if (size < dmem::kPageSize * 2 || size % dmem::kPageSize) { + return {{}, orbis::ErrorCode::INVAL}; + } + } + + if (blockFlags & (BlockFlags::DirectMemory | BlockFlags::PooledMemory)) { + if (alignment < dmem::kPageSize) { + alignment = dmem::kPageSize; + } + } + + bool canOverwrite = (allocFlags & AllocationFlags::Fixed) && + !(allocFlags & AllocationFlags::NoOverwrite); + + VirtualMemoryAllocation allocationInfo; + allocationInfo.flagsEx = blockFlagsEx | BlockFlagsEx::Allocated; + allocationInfo.device = file->device; + allocationInfo.prot = prot; + allocationInfo.deviceOffset = fileOffset; + allocationInfo.setName(process, name); + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + auto [_, errc, range] = vmem->map(addressHint, size, allocationInfo, allocFlags | AllocationFlags::Dry, alignment); @@ -163,48 +398,515 @@ std::pair orbis::vmem::map( if (errc != std::errc{}) { if (errc == std::errc::not_enough_memory) { // virtual memory shouldn't care about physical memory - return {{}, orbis::ErrorCode::INVAL}; + return {{}, ErrorCode::INVAL}; } return {{}, toErrorCode(errc)}; } - rx::EnumBitSet deviceProtection = {}; - - if (prot & Protection::CpuRead) { - deviceProtection |= rx::mem::Protection::R; + if (canOverwrite) { + if (auto errc = validateOverwrite(vmem, range, false); + errc != ErrorCode{}) { + return {{}, errc}; + } } - if (prot & Protection::CpuWrite) { - deviceProtection |= rx::mem::Protection::W; + auto budget = process->getBudget(); + + if (blockFlags & BlockFlags::PooledMemory) { + prot = {}; + allocationInfo.prot = prot; + blockFlags &= ~BlockFlags::Commited; } - if (prot & Protection::CpuExec) { - deviceProtection |= rx::mem::Protection::X; + if (blockFlags & BlockFlags::FlexibleMemory) { + if (!budget->acquire(BudgetResource::Fmem, size)) { + return {{}, ErrorCode::INVAL}; + } + + if (prot) { + blockFlags |= BlockFlags::Commited; + } } - if (auto error = device->map(range, deviceOffset, deviceProtection, process); + allocFlags = AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge); + + if (blockFlags & BlockFlags::DirectMemory) { + if (!budget->acquire(BudgetResource::Dmem, size)) { + return {{}, ErrorCode::INVAL}; + } + + allocFlags |= AllocationFlags::NoMerge; + + if (prot) { + blockFlags |= BlockFlags::Commited; + } + } + + if (blockFlags & BlockFlags::PooledMemory) { + if (auto errc = blockpool::allocateControlBlock(); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + allocationInfo.flagsEx |= BlockFlagsEx::PoolControl; + } + + if (auto error = file->device->map(range, fileOffset, prot, file, process); error != ErrorCode{}) { + if (blockFlags & BlockFlags::FlexibleMemory) { + budget->release(BudgetResource::Fmem, size); + } + + if (blockFlags & BlockFlags::DirectMemory) { + budget->release(BudgetResource::Dmem, size); + } + + if (blockFlags & BlockFlags::PooledMemory) { + blockpool::releaseControlBlock(); + } + return {{}, error}; } - auto [it, _errc, _range] = - vmem->map(range.beginAddress(), range.size(), allocationInfo, - AllocationFlags::Fixed | AllocationFlags::NoMerge, alignment); - - if (name.empty()) { - it->name = rx::format("anon:{:012x}", it.beginAddress()); + if (canOverwrite) { + release(process, vmem, budget, range); } + allocationInfo.flags = blockFlags; + + if (type == MemoryType::Invalid) { + if (blockFlags & BlockFlags::FlexibleMemory) { + type = MemoryType::WbOnion; + } else if (blockFlags & BlockFlags::DirectMemory) { + auto queryResult = dmem::query(0, fileOffset); + if (queryResult) { + type = queryResult->memoryType; + } else { + type = MemoryType::WbOnion; + } + } + allocationInfo.type = type; + } else { + if (blockFlags & BlockFlags::DirectMemory) { + if (auto errc = dmem::setType( + 0, rx::AddressRange::fromBeginSize(fileOffset, size), type); + errc != ErrorCode{}) { + rx::die("mapDirect: failed to set dmem type"); + } + } + } + + { + auto [_it, errc, _range] = vmem->map(range.beginAddress(), range.size(), + allocationInfo, allocFlags, alignment); + + rx::dieIf(errc != std::errc{}, "failed to commit virtual memory {}", errc); + } + + // vmemDump(process, rx::format("mapped {:x}-{:x}", range.beginAddress(), + // range.endAddress())); + return {range, {}}; } +std::pair orbis::vmem::mapDirect( + Process *process, std::uint64_t addressHint, rx::AddressRange directRange, + rx::EnumBitSet prot, rx::EnumBitSet allocFlags, + std::string_view name, std::uint64_t alignment, MemoryType type) { + ScopedBudgetAcquire dmemResource(process->getBudget(), BudgetResource::Dmem, + directRange.size()); + if (!dmemResource) { + return {{}, ErrorCode::INVAL}; + } + + VirtualMemoryAllocation allocationInfo; + allocationInfo.flags = orbis::vmem::BlockFlags::DirectMemory; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.prot = prot; + allocationInfo.device = g_context->dmem->device; + allocationInfo.deviceOffset = directRange.beginAddress(); + allocationInfo.type = type; + allocationInfo.setName(process, name); + + if (prot) { + allocationInfo.flags |= orbis::vmem::BlockFlags::Commited; + } + + bool canOverwrite = (allocFlags & AllocationFlags::Fixed) && + !(allocFlags & AllocationFlags::NoOverwrite); + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + rx::AddressRange range; + + { + auto [_, errc, vmemRange] = + vmem->map(addressHint, directRange.size(), allocationInfo, + allocFlags | AllocationFlags::Dry, alignment); + if (errc != std::errc{}) { + return {{}, toErrorCode(errc)}; + } + + range = vmemRange; + } + + if (canOverwrite) { + if (auto errc = validateOverwrite(vmem, range, false); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + } + + if (auto errc = dmem::map(0, range, directRange.beginAddress(), prot); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + + if (canOverwrite) { + release(process, vmem, process->getBudget(), range); + } + + if (type == MemoryType::Invalid) { + auto queryResult = dmem::query(0, directRange.beginAddress()); + rx::dieIf(!queryResult, "mapDirect: failed to query dmem type"); + type = queryResult->memoryType; + allocationInfo.type = type; + } else { + if (auto errc = dmem::setType(0, directRange, type); errc != ErrorCode{}) { + rx::die("mapDirect: failed to set dmem type"); + } + } + + vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, alignment); + + dmemResource.commit(); + + auto [pmemOffset, errc] = dmem::getPmemOffset(0, directRange.beginAddress()); + rx::dieIf(errc != ErrorCode{}, + "mapDirect: failed to query physical offset {}", errc); + + amdgpu::mapMemory(process->pid, range, type, prot, pmemOffset); + + // vmemDump(process, + // rx::format("mapped dmem {:x}-{:x}", range.beginAddress(), + // range.endAddress())); + + return {range, {}}; +} + +std::pair +orbis::vmem::mapFlex(Process *process, std::uint64_t size, + rx::EnumBitSet prot, std::uint64_t addressHint, + rx::EnumBitSet allocFlags, + rx::EnumBitSet blockFlags, + std::string_view name, std::uint64_t alignment) { + ScopedBudgetAcquire fmemResource(process->getBudget(), BudgetResource::Fmem, + size); + if (!fmemResource) { + return {{}, ErrorCode::INVAL}; + } + + bool canOverwrite = (allocFlags & AllocationFlags::Fixed) && + !(allocFlags & AllocationFlags::NoOverwrite); + + VirtualMemoryAllocation allocationInfo; + allocationInfo.flags = orbis::vmem::BlockFlags::FlexibleMemory | blockFlags; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.prot = prot; + allocationInfo.setName(process, name); + + if (prot) { + allocationInfo.flags |= orbis::vmem::BlockFlags::Commited; + } + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + rx::AddressRange vmemRange; + + { + auto [_, errc, range] = + vmem->map(addressHint, size, allocationInfo, + allocFlags | AllocationFlags::Dry, alignment); + if (errc != std::errc{}) { + return {{}, toErrorCode(errc)}; + } + + vmemRange = range; + } + + if (canOverwrite) { + if (auto errc = validateOverwrite(vmem, vmemRange, false); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + } + + rx::AddressRange flexRange; + + { + auto [range, errc] = fmem::allocate(size); + + if (errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + + flexRange = range; + } + + allocationInfo.deviceOffset = flexRange.beginAddress(); + + if (auto errc = + pmem::map(vmemRange.beginAddress(), flexRange, toCpuProtection(prot)); + errc != orbis::ErrorCode{}) { + fmem::deallocate(flexRange); + return {{}, errc}; + } + + if (canOverwrite) { + release(process, vmem, process->getBudget(), vmemRange); + } + + auto [it, _errc, _range] = vmem->map( + vmemRange.beginAddress(), vmemRange.size(), allocationInfo, + AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge), + alignment); + + // vmemDump(process, rx::format("mapFlex {:x}-{:x}", vmemRange.beginAddress(), + // vmemRange.endAddress())); + fmemResource.commit(); + return {vmemRange, {}}; +} + +std::pair +orbis::vmem::commitPooled(Process *process, rx::AddressRange range, + MemoryType type, rx::EnumBitSet prot) { + VirtualMemoryAllocation allocationInfo; + allocationInfo.flags = BlockFlags::PooledMemory | BlockFlags::Commited; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.prot = prot; + allocationInfo.type = type; + + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end() || !it->isAllocated() || + it->flags != BlockFlags::PooledMemory || !it.range().contains(range)) { + return {{}, ErrorCode::INVAL}; + } + + allocationInfo.name = it->name; + + auto controlBlockIt = it; + + while (controlBlockIt != vmem->end() && controlBlockIt->isAllocated() && + (controlBlockIt->flags & BlockFlags::PooledMemory)) { + if (!controlBlockIt->prot && + controlBlockIt->flags == BlockFlags::PooledMemory && + (controlBlockIt->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl)) { + break; + } + + ++controlBlockIt; + } + + if (controlBlockIt != vmem->end() && + controlBlockIt->flags != BlockFlags::PooledMemory) { + controlBlockIt = vmem->end(); + } + + if (controlBlockIt == vmem->end()) { + auto rangeWithoutCtrl = it.range(); + rangeWithoutCtrl = rx::AddressRange::fromBeginEnd( + rangeWithoutCtrl.beginAddress(), + rangeWithoutCtrl.endAddress() - dmem::kPageSize); + + if (!rangeWithoutCtrl.contains(range)) { + return {{}, ErrorCode::INVAL}; + } + + if (auto errc = blockpool::commit(process, range, type, prot); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + + auto controlAllocationInfo = it.get(); + controlAllocationInfo.flags = BlockFlags::PooledMemory; + controlAllocationInfo.flagsEx = + BlockFlagsEx::Allocated | BlockFlagsEx::PoolControl; + controlAllocationInfo.deviceOffset = range.endAddress() - it.beginAddress(); + controlAllocationInfo.type = MemoryType::WbOnion; + + vmem->map(range.endAddress(), it.endAddress() - range.endAddress(), + controlAllocationInfo, AllocationFlags::Fixed, kPageSize); + } else { + if (auto errc = blockpool::commit(process, range, type, prot); + errc != orbis::ErrorCode{}) { + return {{}, errc}; + } + } + + vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + + // vmemDump(process, rx::format("commitPooled {:x}-{:x}", + // range.beginAddress(), + // range.endAddress())); + return {range, {}}; +} + +orbis::ErrorCode orbis::vmem::decommitPooled(Process *process, + rx::AddressRange range) { + auto vmem = process->get(g_vmInstance); + std::lock_guard lock(*vmem); + + auto it = vmem->query(range.beginAddress()); + + while (it != vmem->end() && it.beginAddress() < range.endAddress()) { + if (!it->isAllocated() || + it->flags != (BlockFlags::PooledMemory | BlockFlags::Commited)) { + if (it->flagsEx & BlockFlagsEx::PoolControl) { + if (it.range().contains(range)) { + // ignore decommits of control block + return {}; + } + + if (it.endAddress() <= range.endAddress()) { + // if requested range overlaps with control block, adjust range to + // ignore control block + range = rx::AddressRange::fromBeginEnd(range.beginAddress(), + it.beginAddress()); + --it; + break; + } + } + + return ErrorCode::INVAL; + } + + ++it; + } + + if (it == vmem->end()) { + return ErrorCode::INVAL; + } + + blockpool::decommit(process, range); + auto allocationInfo = it.get(); + allocationInfo.prot = {}; + allocationInfo.flags = BlockFlags::PooledMemory; + allocationInfo.flagsEx = BlockFlagsEx::Allocated; + allocationInfo.deviceOffset = 0; + auto unmapped = vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + + auto controlBlockIt = unmapped.it; + ++controlBlockIt; + + if (controlBlockIt != vmem->end() && + controlBlockIt->flags == BlockFlags::PooledMemory && + controlBlockIt->flagsEx & BlockFlagsEx::PoolControl) { + allocationInfo.flagsEx |= BlockFlagsEx::PoolControl; + // compress control block + + if (controlBlockIt.beginAddress() - controlBlockIt->deviceOffset != + unmapped.it.beginAddress()) { + // if not whole reserved space was decommited, preserve offset + allocationInfo.deviceOffset = controlBlockIt->deviceOffset; + } else { + allocationInfo.deviceOffset = 0; + } + + auto controlRange = unmapped.it.range().merge(controlBlockIt.range()); + + vmem->map(controlRange.beginAddress(), controlRange.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + } + + // vmemDump(process, rx::format("decommitPooled {:x}-{:x}", + // range.beginAddress(), + // range.endAddress())); + + return {}; +} + +orbis::ErrorCode orbis::vmem::protect(Process *process, rx::AddressRange range, + rx::EnumBitSet prot) { + auto vmem = process->get(g_vmInstance); + + range = rx::AddressRange::fromBeginEnd( + rx::alignDown(range.beginAddress(), kPageSize), + rx::alignUp(range.endAddress(), kPageSize)); + { + std::lock_guard lock(*vmem); + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end()) { + rx::println(stderr, + "vmem: attempt to set protection of invalid address range: " + "{:x}-{:x}", + range.beginAddress(), range.endAddress()); + return orbis::ErrorCode::INVAL; + } + + auto errc = validateRange(vmem, it, range, + [](const VirtualMemoryAllocation &alloc) { + if (alloc.flags == BlockFlags::PooledMemory) { + return ErrorCode::ACCES; + } + + if (alloc.flagsEx & BlockFlagsEx::Reserved) { + return ErrorCode::ACCES; + } + + return ErrorCode{}; + }); + + if (errc != ErrorCode{}) { + return errc; + } + + if (auto errc = + toErrorCode(rx::mem::protect(range, vmem::toCpuProtection(prot))); + errc != ErrorCode{}) { + return errc; + } + + modifyRange(vmem, it, range, + [prot](VirtualMemoryAllocation &alloc, rx::AddressRange) { + if (!alloc.isAllocated()) { + return; + } + + if (alloc.device != nullptr && + alloc.flags & BlockFlags::FlexibleMemory) { + alloc.prot = prot & ~Protection::CpuExec; + } + + alloc.flags = alloc.prot + ? alloc.flags | BlockFlags::Commited + : alloc.flags & ~BlockFlags::Commited; + }); + } + + amdgpu::protectMemory(process->pid, range, prot); + + // vmemDump(process, rx::format("protected {:x}-{:x} {}", + // range.beginAddress(), + // range.endAddress(), prot)); + + return {}; +} + orbis::ErrorCode orbis::vmem::setName(Process *process, rx::AddressRange range, std::string_view name) { auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - auto it = vmem->query(range.beginAddress()); if (it == vmem->end()) { @@ -215,48 +917,162 @@ orbis::ErrorCode orbis::vmem::setName(Process *process, rx::AddressRange range, return orbis::ErrorCode::INVAL; } - if (!it->isAllocated()) { - rx::println(stderr, - "vmem: attempt to set name of unallocated range: request " - "{:x}-{:x}, node: {:x}-{:x}, name {}", - range.beginAddress(), range.endAddress(), it.beginAddress(), - it.endAddress(), name); + modifyRange(vmem, it, range, + [name](VirtualMemoryAllocation &alloc, rx::AddressRange) { + if (alloc.isAllocated()) { + alloc.name = name; + } + }); + + return {}; +} + +orbis::ErrorCode orbis::vmem::setType(Process *process, rx::AddressRange range, + MemoryType type) { + auto vmem = process->get(g_vmInstance); + + std::lock_guard lock(*vmem); + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end()) { return orbis::ErrorCode::INVAL; } - if (it.range() != range) { - rx::println(stderr, - "vmem: set name range mismatch " - "{:x}-{:x}, node: {:x}-{:x}, name {}", - range.beginAddress(), range.endAddress(), it.beginAddress(), - it.endAddress(), name); + modifyRange( + vmem, it, range, + [type](VirtualMemoryAllocation &alloc, rx::AddressRange range) { + if (!alloc.isAllocated()) { + return; + } + + if (alloc.flags & BlockFlags::DirectMemory) { + dmem::setType( + 0, + rx::AddressRange::fromBeginSize(alloc.deviceOffset, range.size()), + type); + alloc.type = type; + return; + } + + if (alloc.flags != (BlockFlags::PooledMemory | BlockFlags::Commited)) { + alloc.type = type; + return; + } + }); + + return {}; +} + +orbis::ErrorCode +orbis::vmem::setTypeAndProtect(Process *process, rx::AddressRange range, + MemoryType type, + rx::EnumBitSet prot) { + auto vmem = process->get(g_vmInstance); + + std::lock_guard lock(*vmem); + auto it = vmem->query(range.beginAddress()); + + if (it == vmem->end()) { + return orbis::ErrorCode::INVAL; } - it->name = name; + auto errc = + validateRange(vmem, it, range, [](const VirtualMemoryAllocation &alloc) { + if (alloc.flags == BlockFlags::PooledMemory) { + return ErrorCode::ACCES; + } + + if (alloc.flagsEx & BlockFlagsEx::Reserved) { + return ErrorCode::ACCES; + } + + return ErrorCode{}; + }); + + if (errc != ErrorCode{}) { + return errc; + } + + if (auto errc = + toErrorCode(rx::mem::protect(range, vmem::toCpuProtection(prot))); + errc != ErrorCode{}) { + return errc; + } + + modifyRange( + vmem, it, range, + [type, prot](VirtualMemoryAllocation &alloc, rx::AddressRange range) { + if (alloc.isAllocated()) { + alloc.type = type; + alloc.prot = prot; + + if (alloc.flags & BlockFlags::DirectMemory) { + dmem::setType(0, + rx::AddressRange::fromBeginSize(alloc.deviceOffset, + range.size()), + type); + return; + } + } + }); + + amdgpu::protectMemory(process->pid, range, prot); return {}; } orbis::ErrorCode orbis::vmem::unmap(Process *process, rx::AddressRange range) { auto vmem = process->get(g_vmInstance); - - std::lock_guard lock(*vmem); + auto budget = process->getBudget(); VirtualMemoryAllocation allocationInfo{}; - auto [it, errc, _] = - vmem->map(range.beginAddress(), range.endAddress(), allocationInfo, - AllocationFlags::Fixed, kPageSize); + + range = rx::AddressRange::fromBeginEnd( + rx::alignDown(range.beginAddress(), kPageSize), + rx::alignUp(range.endAddress(), kPageSize)); + + orbis::ErrorCode result; + { + std::lock_guard lock(*vmem); + + if (auto errc = validateOverwrite(vmem, range, true); + errc != orbis::ErrorCode{}) { + return errc; + } + + release(process, vmem, budget, range); + + auto [it, errc, _] = + vmem->map(range.beginAddress(), range.size(), allocationInfo, + AllocationFlags::Fixed, kPageSize); + + result = toErrorCode(errc); + } rx::mem::release(range, kPageSize); - return toErrorCode(errc); + amdgpu::unmapMemory(process->pid, range); + + // vmemDump(process, rx::format("unmap {:x}-{:x}", range.beginAddress(), + // range.endAddress())); + return result; } std::optional -orbis::vmem::query(Process *process, std::uint64_t address) { +orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) { auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - auto it = vmem->query(address); - if (it == vmem->end()) { + auto it = vmem->lowerBound(address); + + if (lowerBound) { + while (it != vmem->end() && !it->isAllocated()) { + ++it; + } + + if (it == vmem->end()) { + return {}; + } + } else if (it == vmem->end() || !it.range().contains(address) || + !it->isAllocated()) { return {}; } @@ -264,30 +1080,54 @@ orbis::vmem::query(Process *process, std::uint64_t address) { result.start = it.beginAddress(); result.end = it.endAddress(); - if (it->isAllocated()) { - if (it->flags == BlockFlags::DirectMemory) { - result.offset = it->deviceOffset; - } - - result.protection = it->prot.toUnderlying(); - result.flags = it->flags.toUnderlying(); - result.name = it->name; + if (!(it->flags & BlockFlags::FlexibleMemory) || it->device != nullptr) { + result.offset = it->deviceOffset; } + if (it->flags & BlockFlags::DirectMemory) { + // if (auto queryResult = dmem::query(0, it->deviceOffset)) { + result.memoryType = it->type; + // } + } else if (it->flags == (BlockFlags::PooledMemory | BlockFlags::Commited)) { + result.memoryType = it->type; + } + + result.protection = it->prot; + result.flags = it->flags; + result.name = it->name; + + // vmemDump(process, rx::format("query result {:x}-{:x} {:x} {} {} {}", + // result.start, result.end, result.offset, + // result.protection, result.flags, + // result.name)); + return result; } std::optional -orbis::vmem::queryProtection(Process *process, std::uint64_t address) { +orbis::vmem::queryProtection(Process *process, std::uint64_t address, + bool lowerBound) { auto vmem = process->get(g_vmInstance); std::lock_guard lock(*vmem); - auto it = vmem->query(address); + auto it = vmem->lowerBound(address); if (it == vmem->end()) { return {}; } + if (lowerBound) { + while (it != vmem->end() && !it->isAllocated()) { + ++it; + } + + if (it == vmem->end()) { + return {}; + } + } else if (!it.range().contains(address) || !it->isAllocated()) { + return {}; + } + orbis::vmem::MemoryProtection result{}; result.startAddress = it.beginAddress(); result.endAddress = it.endAddress(); diff --git a/rpcsx/CMakeLists.txt b/rpcsx/CMakeLists.txt index b6830bc30..96ae16ff9 100644 --- a/rpcsx/CMakeLists.txt +++ b/rpcsx/CMakeLists.txt @@ -72,7 +72,6 @@ if(LINUX AND WITH_PS4) main.cpp AudioOut.cpp backtrace.cpp - vm.cpp ops.cpp linker.cpp io-device.cpp diff --git a/rpcsx/gpu/Device.cpp b/rpcsx/gpu/Device.cpp index 5c2f764e1..858083414 100644 --- a/rpcsx/gpu/Device.cpp +++ b/rpcsx/gpu/Device.cpp @@ -7,6 +7,8 @@ #include "orbis-config.hpp" #include "orbis/KernelContext.hpp" #include "orbis/note.hpp" +#include "orbis/pmem.hpp" +#include "orbis/vmem.hpp" #include "rx/AddressRange.hpp" #include "rx/Config.hpp" #include "rx/bits.hpp" @@ -135,8 +137,8 @@ static vk::Context createVkContext(Device *device) { &device->debugMessenger)); } - glfwCreateWindowSurface(vk::context->instance, device->window, nullptr, - &device->surface); + VK_VERIFY(glfwCreateWindowSurface(vk::context->instance, device->window, + nullptr, &device->surface)); result.createDevice(device->surface, rx::g_config.gpuIndex, { @@ -293,18 +295,6 @@ Device::~Device() { vk::context->allocator); } - for (auto fd : dmemFd) { - if (fd >= 0) { - ::close(fd); - } - } - - for (auto &[pid, info] : processInfo) { - if (info.vmFd >= 0) { - ::close(info.vmFd); - } - } - for (auto &cachePage : cachePages) { orbis::kfree(cachePage, kCachePageSize); } @@ -321,28 +311,6 @@ void Device::start() { }); } - for (std::size_t i = 0; i < std::size(dmemFd); ++i) { - if (dmemFd[i] != -1) { - continue; - } - - auto path = rx::format("{}/dmem-{}", rx::getShmPath(), i); - if (!std::filesystem::exists(path)) { - std::println("Waiting for dmem {}", i); - - while (!std::filesystem::exists(path)) { - std::this_thread::sleep_for(std::chrono::milliseconds(300)); - } - } - - dmemFd[i] = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR); - - if (dmemFd[i] < 0) { - std::println(stderr, "failed to open dmem {}", path); - std::abort(); - } - } - std::jthread vblankThread([](const std::stop_token &stopToken) { orbis::g_context->deviceEventEmitter->emit( orbis::kEvFiltDisplay, @@ -573,45 +541,27 @@ void Device::mapProcess(std::uint32_t pid, int vmId) { auto memory = amdgpu::RemoteMemory{vmId}; - std::string pidVmName = rx::format("{}/memory-{}", rx::getShmPath(), pid); - int memoryFd = ::open(pidVmName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); - process.vmFd = memoryFd; - - if (memoryFd < 0) { - std::println("failed to open shared memory of process {}", (int)pid); - std::abort(); - } - for (auto slot : process.vmTable) { - auto gpuProt = slot->prot >> 4; - if (gpuProt == 0) { + auto gpuProt = orbis::vmem::toGpuProtection(slot->prot); + if (!gpuProt) { continue; } auto devOffset = slot->offset + slot.beginAddress() - slot->baseAddress; - int mapFd = memoryFd; - if (slot->memoryType >= 0) { - mapFd = dmemFd[slot->memoryType]; - } + auto errc = orbis::pmem::map( + memory.getVirtualAddress(slot.beginAddress()), + rx::AddressRange::fromBeginSize(devOffset, slot.size()), gpuProt); + rx::dieIf( + errc != orbis::ErrorCode{}, + "failed to map process {} memory, address {}-{}, type {}, vmId {}", + (int)pid, memory.getPointer(slot.beginAddress()), + memory.getPointer(slot.endAddress()), slot->memoryType, vmId); - auto mmapResult = - ::mmap(memory.getPointer(slot.beginAddress()), slot.size(), gpuProt, - MAP_FIXED | MAP_SHARED, mapFd, devOffset); - - if (mmapResult == MAP_FAILED) { - std::println( - stderr, - "failed to map process {} memory, address {}-{}, type {:x}, vmId {}", - (int)pid, memory.getPointer(slot.beginAddress()), - memory.getPointer(slot.endAddress()), slot->memoryType, vmId); - std::abort(); - } - - // std::println(stderr, - // "map process {} memory, address {}-{}, type {:x}, vmId {}", - // (int)pid, memory.getPointer(startAddress), - // memory.getPointer(endAddress), slot.memoryType, vmId); + std::println(stderr, "map process {} memory, address {}-{}, vmId {}", + (int)pid, memory.getVirtualAddress(slot.beginAddress()), + memory.getVirtualAddress(slot.beginAddress()) + slot.size(), + vmId); } } @@ -629,13 +579,12 @@ void Device::unmapProcess(std::uint32_t pid) { rx::die("failed to release userspace memory: {}", (int)errc); } - ::close(process.vmFd); - process.vmFd = -1; process.vmId = -1; } void Device::protectMemory(std::uint32_t pid, std::uint64_t address, - std::uint64_t size, int prot) { + std::uint64_t size, + rx::EnumBitSet prot) { auto &process = processInfo[pid]; auto vmSlotIt = process.vmTable.queryArea(address); @@ -648,22 +597,20 @@ void Device::protectMemory(std::uint32_t pid, std::uint64_t address, process.vmTable.map(rx::AddressRange::fromBeginSize(address, size), VmMapSlot{ .memoryType = vmSlot.memoryType, - .prot = static_cast(prot), + .prot = prot, .offset = vmSlot.offset, .baseAddress = vmSlot.baseAddress, }); if (process.vmId >= 0) { auto memory = amdgpu::RemoteMemory{process.vmId}; - rx::mem::protect( - rx::AddressRange::fromBeginSize(memory.getVirtualAddress(address), - size), - rx::EnumBitSet::fromUnderlying(prot >> 4)); + rx::mem::protect(rx::AddressRange::fromBeginSize( + memory.getVirtualAddress(address), size), + orbis::vmem::toGpuProtection(prot)); - // std::println(stderr, "protect process {} memory, address {}-{}, prot - // {:x}", - // (int)pid, memory.getPointer(address), - // memory.getPointer(address + size), prot); + std::println(stderr, "protect process {} memory, address {}-{}, prot {}", + (int)pid, memory.getPointer(address), + memory.getPointer(address + size), prot); } } @@ -1006,17 +953,18 @@ void Device::waitForIdle() { } } -void Device::mapMemory(std::uint32_t pid, std::uint64_t address, - std::uint64_t size, int memoryType, int dmemIndex, - int prot, std::int64_t offset) { +void Device::mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t physicalOffset) { auto &process = processInfo[pid]; - process.vmTable.map(rx::AddressRange::fromBeginSize(address, size), + process.vmTable.map(virtualRange, VmMapSlot{ - .memoryType = memoryType >= 0 ? dmemIndex : -1, + .memoryType = memoryType, .prot = prot, - .offset = offset, - .baseAddress = address, + .offset = physicalOffset, + .baseAddress = virtualRange.beginAddress(), }); if (process.vmId < 0) { @@ -1025,33 +973,29 @@ void Device::mapMemory(std::uint32_t pid, std::uint64_t address, auto memory = amdgpu::RemoteMemory{process.vmId}; - int mapFd = process.vmFd; - - if (memoryType >= 0) { - mapFd = dmemFd[dmemIndex]; + auto vmemAddress = memory.getVirtualAddress(virtualRange.beginAddress()); + auto errc = orbis::pmem::map(vmemAddress, + rx::AddressRange::fromBeginSize( + physicalOffset, virtualRange.beginAddress()), + orbis::vmem::toGpuProtection(prot)); + if (errc != orbis::ErrorCode{}) { + rx::die("failed to map process {} memory, address {:x}-{:x}, type {}, " + "offset {:x}, prot {}, error {}", + pid, vmemAddress, vmemAddress + virtualRange.size(), memoryType, + physicalOffset, prot, errc); } - auto mmapResult = ::mmap(memory.getPointer(address), size, prot >> 4, - MAP_FIXED | MAP_SHARED, mapFd, offset); - - if (mmapResult == MAP_FAILED) { - perror("::mmap"); - - rx::die("failed to map process {} memory, address {}-{}, type {:x}, offset " - "{:x}, prot {:x}", - pid, memory.getPointer(address), memory.getPointer(address + size), - memoryType, offset, prot); - } - - // std::println(stderr, "map memory of process {}, address {}-{}, prot {:x}", - // (int)pid, memory.getPointer(address), - // memory.getPointer(address + size), prot); + std::println( + stderr, + "map memory of process {}, address {:x}-{:x}, prot {}, phy memory {:x}", + (int)pid, vmemAddress, vmemAddress + virtualRange.size(), prot, + physicalOffset); } void Device::unmapMemory(std::uint32_t pid, std::uint64_t address, std::uint64_t size) { // TODO - protectMemory(pid, address, size, 0); + protectMemory(pid, address, size, {}); } static void notifyPageChanges(Device *device, int vmId, std::uint32_t firstPage, diff --git a/rpcsx/gpu/Device.hpp b/rpcsx/gpu/Device.hpp index 5a17f169b..200e5d990 100644 --- a/rpcsx/gpu/Device.hpp +++ b/rpcsx/gpu/Device.hpp @@ -5,6 +5,11 @@ #include "Pipe.hpp" #include "amdgpu/tiler_vulkan.hpp" #include "orbis/KernelAllocator.hpp" +#include "orbis/dmem.hpp" +#include "orbis/thread/Process.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/MemoryTable.hpp" #include "rx/Rc.hpp" #include "rx/SharedMutex.hpp" @@ -37,9 +42,9 @@ std::array createPm4Packet(std::uint32_t op, } struct VmMapSlot { - int memoryType; - int prot; - std::int64_t offset; + orbis::MemoryType memoryType; + rx::EnumBitSet prot; + std::uint64_t offset; std::uint64_t baseAddress; auto operator<=>(const VmMapSlot &) const = default; @@ -47,7 +52,6 @@ struct VmMapSlot { struct ProcessInfo { int vmId = -1; - int vmFd = -1; BufferAttribute bufferAttributes[10]; Buffer buffers[10]; rx::MemoryTableWithPayload vmTable; @@ -90,7 +94,6 @@ struct Device : rx::RcBase, DeviceContext { std::jthread cacheUpdateThread; - int dmemFd[3] = {-1, -1, -1}; orbis::kmap processInfo; Cache caches[kMaxProcessCount]{ @@ -123,7 +126,8 @@ struct Device : rx::RcBase, DeviceContext { void mapProcess(std::uint32_t pid, int vmId); void unmapProcess(std::uint32_t pid); void protectMemory(std::uint32_t pid, std::uint64_t address, - std::uint64_t size, int prot); + std::uint64_t size, + rx::EnumBitSet prot); void onCommandBuffer(std::uint32_t pid, int cmdHeader, std::uint64_t address, std::uint64_t size); bool processPipes(); @@ -131,8 +135,10 @@ struct Device : rx::RcBase, DeviceContext { VkImage swapchainImage, VkImageView swapchainImageView); void flip(std::uint32_t pid, int bufferIndex, std::uint64_t arg); void waitForIdle(); - void mapMemory(std::uint32_t pid, std::uint64_t address, std::uint64_t size, - int memoryType, int dmemIndex, int prot, std::int64_t offset); + void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t physicalOffset); void unmapMemory(std::uint32_t pid, std::uint64_t address, std::uint64_t size); void watchWrites(int vmId, std::uint64_t address, std::uint64_t size); diff --git a/rpcsx/gpu/DeviceCtl.cpp b/rpcsx/gpu/DeviceCtl.cpp index a9232e4a1..4750fc600 100644 --- a/rpcsx/gpu/DeviceCtl.cpp +++ b/rpcsx/gpu/DeviceCtl.cpp @@ -1,6 +1,7 @@ #include "DeviceCtl.hpp" #include "Device.hpp" #include "gnm/pm4.hpp" +#include "orbis/KernelContext.hpp" #include "orbis/error/ErrorCode.hpp" #include "rx/bits.hpp" #include "rx/die.hpp" @@ -230,3 +231,26 @@ void DeviceCtl::submitComputeQueue(std::uint32_t meId, std::uint32_t pipeId, void DeviceCtl::start() { mDevice->start(); } void DeviceCtl::waitForIdle() { mDevice->waitForIdle(); } + +void amdgpu::mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset) { + if (auto gpu = DeviceCtl{orbis::g_context->gpuDevice.rawCast()}) { + gpu.submitMapMemory(pid, virtualRange.beginAddress(), virtualRange.size(), + (int)memoryType, 0, prot.toUnderlying(), offset); + } +} +void amdgpu::unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange) { + if (auto gpu = DeviceCtl{orbis::g_context->gpuDevice.rawCast()}) { + gpu.submitUnmapMemory(pid, virtualRange.beginAddress(), + virtualRange.size()); + } +} +void amdgpu::protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot) { + if (auto gpu = DeviceCtl{orbis::g_context->gpuDevice.rawCast()}) { + gpu.submitProtectMemory(pid, virtualRange.beginAddress(), + virtualRange.size(), prot.toUnderlying()); + } +} diff --git a/rpcsx/gpu/DeviceCtl.hpp b/rpcsx/gpu/DeviceCtl.hpp index bdd4e1c7f..e3b41ea88 100644 --- a/rpcsx/gpu/DeviceCtl.hpp +++ b/rpcsx/gpu/DeviceCtl.hpp @@ -2,6 +2,10 @@ #include "DeviceContext.hpp" #include "orbis-config.hpp" +#include "orbis/dmem.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/Rc.hpp" #include #include @@ -58,4 +62,12 @@ public: explicit operator bool() const { return mDevice != nullptr; } }; + +void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange, + orbis::MemoryType memoryType, + rx::EnumBitSet prot, + std::uint64_t offset); +void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange); +void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange, + rx::EnumBitSet prot); } // namespace amdgpu diff --git a/rpcsx/gpu/Pipe.cpp b/rpcsx/gpu/Pipe.cpp index 741cb6376..9158e9198 100644 --- a/rpcsx/gpu/Pipe.cpp +++ b/rpcsx/gpu/Pipe.cpp @@ -5,6 +5,9 @@ #include "gnm/mmio.hpp" #include "gnm/pm4.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/print.hpp" #include "vk.hpp" #include @@ -2221,9 +2224,10 @@ void CommandPipe::mapMemory(Ring &ring) { auto addressHi = ring.rptr[3]; auto sizeLo = ring.rptr[4]; auto sizeHi = ring.rptr[5]; - auto memoryType = ring.rptr[6]; - auto dmemIndex = ring.rptr[7]; - auto prot = ring.rptr[8]; + auto memoryType = orbis::MemoryType(ring.rptr[6]); + // auto dmemIndex = ring.rptr[7]; + auto prot = + rx::EnumBitSet::fromUnderlying(ring.rptr[8]); auto offsetLo = ring.rptr[9]; auto offsetHi = ring.rptr[10]; @@ -2231,7 +2235,8 @@ void CommandPipe::mapMemory(Ring &ring) { auto size = sizeLo | (static_cast(sizeHi) << 32); auto offset = offsetLo | (static_cast(offsetHi) << 32); - device->mapMemory(pid, address, size, memoryType, dmemIndex, prot, offset); + device->mapMemory(pid, rx::AddressRange::fromBeginSize(address, size), + memoryType, prot, offset); } void CommandPipe::unmapMemory(Ring &ring) { auto pid = ring.rptr[1]; @@ -2250,7 +2255,8 @@ void CommandPipe::protectMemory(Ring &ring) { auto addressHi = ring.rptr[3]; auto sizeLo = ring.rptr[4]; auto sizeHi = ring.rptr[5]; - auto prot = ring.rptr[6]; + auto prot = + rx::EnumBitSet::fromUnderlying(ring.rptr[6]); auto address = addressLo | (static_cast(addressHi) << 32); auto size = sizeLo | (static_cast(sizeHi) << 32); diff --git a/rpcsx/io-device.cpp b/rpcsx/io-device.cpp index b42134158..cea81ce8c 100644 --- a/rpcsx/io-device.cpp +++ b/rpcsx/io-device.cpp @@ -1,14 +1,16 @@ #include "io-device.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/SocketAddress.hpp" +#include "orbis/error.hpp" #include "orbis/file.hpp" #include "orbis/stat.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/uio.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/Mappable.hpp" #include "vfs.hpp" -#include "vm.hpp" #include #include #include @@ -33,8 +35,8 @@ struct HostFile : orbis::File { bool alignTruncate = false; ~HostFile() { - if (hostFd > 0 && closeOnExit) { - ::close(hostFd); + if (!closeOnExit && hostFd) { + static_cast(hostFd.release()); } } }; @@ -46,12 +48,6 @@ struct SocketFile : orbis::File { int prot = -1; orbis::kmap> options; - - ~SocketFile() { - if (hostFd > 0) { - ::close(hostFd); - } - } }; static orbis::ErrorCode convertErrc(std::errc errc) { @@ -319,7 +315,7 @@ static orbis::ErrorCode host_read(orbis::File *file, orbis::Uio *uio, if (!hostFile->dirEntries.empty()) return orbis::ErrorCode::ISDIR; - return host_fd_read(hostFile->hostFd, uio); + return host_fd_read(hostFile->hostFd.native_handle(), uio); } static orbis::ErrorCode host_write(orbis::File *file, orbis::Uio *uio, @@ -328,73 +324,14 @@ static orbis::ErrorCode host_write(orbis::File *file, orbis::Uio *uio, if (!hostFile->dirEntries.empty()) return orbis::ErrorCode::ISDIR; - return host_fd_write(hostFile->hostFd, uio); -} - -static orbis::ErrorCode host_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - auto hostFile = static_cast(file); - if (!hostFile->dirEntries.empty()) - return orbis::ErrorCode::ISDIR; - - auto result = vm::map(*address, size, prot, flags, - vm::kMapInternalReserveOnly, hostFile->device.get(), offset); - - if (result == (void *)-1) { - return orbis::ErrorCode::NOMEM; - } - - size = rx::alignUp(size, vm::kPageSize); - - result = - ::mmap(result, size, prot & vm::kMapProtCpuAll, - ((flags & vm::kMapFlagPrivate) != 0 ? MAP_PRIVATE : MAP_SHARED) | - MAP_FIXED, - hostFile->hostFd, offset); - if (result == (void *)-1) { - auto errc = convertErrno(); - std::printf("Failed to map file at %p-%p\n", *address, - (char *)*address + size); - return errc; - } - - std::printf("file mapped at %p-%p:%lx\n", result, (char *)result + size, - offset); - - struct stat stat; - fstat(hostFile->hostFd, &stat); - if (stat.st_size < offset + size) { - std::size_t rest = std::min(offset + size - stat.st_size, vm::kPageSize); - - if (rest > rx::mem::pageSize) { - auto fillSize = rx::alignUp(rest, rx::mem::pageSize) - rx::mem::pageSize; - - std::printf("adding dummy mapping %p-%p, file ends at %p\n", - (char *)result + size - fillSize, (char *)result + size, - (char *)result + (stat.st_size - offset)); - - auto ptr = ::mmap((char *)result + size - fillSize, fillSize, - prot & vm::kMapProtCpuAll, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); - - if (ptr == (void *)-1) { - std::printf("failed to add dummy mapping %p-%p\n", result, - (char *)result + size); - } - } - } - - *address = result; - return {}; + return host_fd_write(hostFile->hostFd.native_handle(), uio); } static orbis::ErrorCode host_stat(orbis::File *file, orbis::Stat *sb, orbis::Thread *thread) { auto hostFile = static_cast(file); struct stat hostStat; - ::fstat(hostFile->hostFd, &hostStat); + ::fstat(hostFile->hostFd.native_handle(), &hostStat); sb->dev = hostStat.st_dev; // TODO sb->ino = hostStat.st_ino; sb->mode = hostStat.st_mode; @@ -437,10 +374,10 @@ static orbis::ErrorCode host_truncate(orbis::File *file, std::uint64_t len, } if (hostFile->alignTruncate) { - len = rx::alignUp(len, vm::kPageSize); + len = rx::alignUp(len, orbis::vmem::kPageSize); } - if (::ftruncate(hostFile->hostFd, len)) { + if (::ftruncate(hostFile->hostFd.native_handle(), len)) { return convertErrno(); } @@ -458,7 +395,7 @@ static orbis::ErrorCode socket_read(orbis::File *file, orbis::Uio *uio, orbis::Thread *) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { while (true) { std::this_thread::sleep_for(std::chrono::days(1)); } @@ -468,14 +405,14 @@ static orbis::ErrorCode socket_read(orbis::File *file, orbis::Uio *uio, if (uio->iov->len) { ORBIS_LOG_FATAL(__FUNCTION__, file, uio->iov->len); } - return host_fd_read(socket->hostFd, uio); + return host_fd_read(socket->hostFd.native_handle(), uio); } static orbis::ErrorCode socket_write(orbis::File *file, orbis::Uio *uio, orbis::Thread *) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { for (auto io : std::span(uio->iov, uio->iovcnt)) { uio->offset += io.len; } @@ -483,7 +420,7 @@ static orbis::ErrorCode socket_write(orbis::File *file, orbis::Uio *uio, } ORBIS_LOG_FATAL(__FUNCTION__, file, uio->iov->len); - return host_fd_write(socket->hostFd, uio); + return host_fd_write(socket->hostFd.native_handle(), uio); } static orbis::ErrorCode socket_bind(orbis::File *file, @@ -492,7 +429,7 @@ static orbis::ErrorCode socket_bind(orbis::File *file, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { return {}; } @@ -512,8 +449,8 @@ static orbis::ErrorCode socket_bind(orbis::File *file, sockaddr_un un{.sun_family = AF_UNIX}; std::strncpy(un.sun_path, socketPath.c_str(), sizeof(un.sun_path)); - if (::bind(socket->hostFd, reinterpret_cast<::sockaddr *>(&un), - sizeof(un)) < 0) { + if (::bind(socket->hostFd.native_handle(), + reinterpret_cast<::sockaddr *>(&un), sizeof(un)) < 0) { return convertErrno(); } @@ -528,11 +465,11 @@ static orbis::ErrorCode socket_listen(orbis::File *file, int backlog, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { return {}; } - if (::listen(socket->hostFd, backlog) < 0) { + if (::listen(socket->hostFd.native_handle(), backlog) < 0) { return convertErrno(); } @@ -545,7 +482,7 @@ static orbis::ErrorCode socket_accept(orbis::File *file, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { ORBIS_LOG_ERROR(__FUNCTION__, socket->name, "wait forever"); while (true) { @@ -558,8 +495,8 @@ static orbis::ErrorCode socket_accept(orbis::File *file, if (socket->dom == 1 && socket->type == 1 && socket->prot == 0) { sockaddr_un un{.sun_family = AF_UNIX}; socklen_t len = sizeof(un); - int result = - ::accept(socket->hostFd, reinterpret_cast(&un), &len); + int result = ::accept(socket->hostFd.native_handle(), + reinterpret_cast(&un), &len); if (result < 0) { return convertErrno(); @@ -585,7 +522,7 @@ static orbis::ErrorCode socket_connect(orbis::File *file, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (socket->hostFd.native_handle()) { return orbis::ErrorCode::CONNREFUSED; } @@ -604,8 +541,8 @@ static orbis::ErrorCode socket_connect(orbis::File *file, sockaddr_un un{.sun_family = AF_UNIX}; std::strncpy(un.sun_path, socketPath.c_str(), sizeof(un.sun_path)); - if (::connect(socket->hostFd, reinterpret_cast<::sockaddr *>(&un), - sizeof(un)) < 0) { + if (::connect(socket->hostFd.native_handle(), + reinterpret_cast<::sockaddr *>(&un), sizeof(un)) < 0) { return convertErrno(); } @@ -650,12 +587,13 @@ orbis::ErrorCode socket_recvfrom(orbis::File *file, void *buf, orbis::Thread *thread) { auto socket = static_cast(file); - if (socket->hostFd < 0) { + if (!socket->hostFd) { return orbis::ErrorCode::CONNREFUSED; } if (socket->dom == 1 && socket->type == 1 && socket->prot == 0) { - auto count = ::recvfrom(socket->hostFd, buf, len, flags, nullptr, nullptr); + auto count = ::recvfrom(socket->hostFd.native_handle(), buf, len, flags, + nullptr, nullptr); if (count < 0) { return convertErrno(); } @@ -678,7 +616,6 @@ static const orbis::FileOps hostOps = { .write = host_write, .truncate = host_truncate, .stat = host_stat, - .mmap = host_mmap, }; static const orbis::FileOps socketOps = { @@ -710,7 +647,7 @@ rx::Ref wrapSocket(int hostFd, orbis::kstring name, int dom, s->dom = dom; s->type = type; s->prot = prot; - s->hostFd = hostFd; + s->hostFd = rx::Mappable::CreateFromNativeHandle(hostFd); s->ops = &socketOps; return s; } @@ -736,7 +673,7 @@ orbis::ErrorCode createSocket(rx::Ref *file, orbis::kstring name, static std::optional findFileInDir(const std::filesystem::path &dir, const char *name) { - for (auto entry : std::filesystem::directory_iterator(dir)) { + for (auto &entry : std::filesystem::directory_iterator(dir)) { auto entryName = entry.path().filename(); if (strcasecmp(entryName.c_str(), name) == 0) { return entryName; @@ -752,7 +689,7 @@ toRealPath(const std::filesystem::path &inp) { } std::filesystem::path result; - for (auto elem : inp) { + for (auto &elem : inp) { if (result.empty() || std::filesystem::exists(result / elem)) { result /= elem; continue; @@ -887,7 +824,7 @@ orbis::ErrorCode HostFsDevice::open(rx::Ref *file, } auto newFile = orbis::knew(); - newFile->hostFd = hostFd; + newFile->hostFd = rx::Mappable::CreateFromNativeHandle(hostFd); newFile->dirEntries = std::move(dirEntries); newFile->ops = &hostOps; newFile->device = this; @@ -936,10 +873,10 @@ orbis::ErrorCode HostFsDevice::rename(const char *from, const char *to, return convertErrorCode(ec); } -orbis::File *createHostFile(int hostFd, rx::Ref device, +orbis::File *createHostFile(int hostFd, orbis::IoDevice *device, bool alignTruncate) { auto newFile = orbis::knew(); - newFile->hostFd = hostFd; + newFile->hostFd = rx::Mappable::CreateFromNativeHandle(hostFd); newFile->ops = &hostOps; newFile->device = device; newFile->alignTruncate = alignTruncate; diff --git a/rpcsx/io-device.hpp b/rpcsx/io-device.hpp index bf0394251..b4bd66cfc 100644 --- a/rpcsx/io-device.hpp +++ b/rpcsx/io-device.hpp @@ -35,6 +35,6 @@ rx::Ref wrapSocket(int hostFd, orbis::kstring name, int dom, int type, int prot); orbis::ErrorCode createSocket(rx::Ref *file, orbis::kstring name, int dom, int type, int prot); -orbis::File *createHostFile(int hostFd, rx::Ref device, +orbis::File *createHostFile(int hostFd, orbis::IoDevice *device, bool alignTruncate = false); orbis::IoDevice *createFdWrapDevice(int fd); diff --git a/rpcsx/io-devices.hpp b/rpcsx/io-devices.hpp index 6c6e260cb..05e55d31f 100644 --- a/rpcsx/io-devices.hpp +++ b/rpcsx/io-devices.hpp @@ -5,9 +5,10 @@ namespace orbis { struct IoDevice; +struct Process; } -orbis::IoDevice *createDceCharacterDevice(); +orbis::IoDevice *createDceCharacterDevice(orbis::Process *process); orbis::IoDevice *createDipswCharacterDevice(); orbis::IoDevice *createDmemCharacterDevice(int index); orbis::IoDevice *createGcCharacterDevice(); diff --git a/rpcsx/iodev/ajm.cpp b/rpcsx/iodev/ajm.cpp index 858a1e7a4..9c2820b5c 100644 --- a/rpcsx/iodev/ajm.cpp +++ b/rpcsx/iodev/ajm.cpp @@ -68,7 +68,7 @@ struct AjmDevice AjmDevice(); }; -AVSampleFormat ajmToAvFormat(AJMFormat ajmFormat) { +static AVSampleFormat ajmToAvFormat(AJMFormat ajmFormat) { switch (ajmFormat) { case AJM_FORMAT_S16: return AV_SAMPLE_FMT_S16; @@ -81,7 +81,7 @@ AVSampleFormat ajmToAvFormat(AJMFormat ajmFormat) { } } -void reset(Instance *instance) { +static void reset(Instance *instance) { instance->gapless.skipSamples = 0; instance->gapless.totalSamples = 0; instance->gapless.totalSkippedSamples = 0; @@ -90,7 +90,7 @@ void reset(Instance *instance) { instance->aac.framesSkipped = 0; } -void resetAt9(Instance *instance) { +static void resetAt9(Instance *instance) { if (instance->at9.configData) { Atrac9ReleaseHandle(instance->at9.handle); instance->at9.estimatedSizeUsed = 0; @@ -316,7 +316,8 @@ ajm_ioctl_start_batch_buffer(orbis::Thread *, AjmDevice *device, args.result = 0; args.batchId = device->batchId++; // ORBIS_LOG_ERROR(__FUNCTION__, args.result, args.unk0, args.pBatch, - // args.batchSize, args.priority, args.batchError, args.batchId); + // args.batchSize, args.priority, args.batchError, + // args.batchId); // thread->where(); auto ptr = args.pBatch; diff --git a/rpcsx/iodev/blockpool.cpp b/rpcsx/iodev/blockpool.cpp index c9f1f1a80..974ff463f 100644 --- a/rpcsx/iodev/blockpool.cpp +++ b/rpcsx/iodev/blockpool.cpp @@ -1,107 +1,114 @@ -#include "blockpool.hpp" -#include "dmem.hpp" +#include "orbis/blockpool.hpp" #include "orbis/IoDevice.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/dmem.hpp" #include "orbis/file.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" -#include "vm.hpp" -#include -#include -#include +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" + +enum { + BLOCKPOOL_IOCTL_EXPAND = 0xc020a801, + BLOCKPOOL_IOCTL_GET_BLOCK_STATS = 0x4010a802, +}; + +struct BlockPoolAllocation { + bool allocated = false; + + [[nodiscard]] bool isAllocated() const { return allocated; } + [[nodiscard]] bool isRelated(const BlockPoolAllocation &, rx::AddressRange, + rx::AddressRange) const { + return true; + } + + [[nodiscard]] BlockPoolAllocation merge(const BlockPoolAllocation &other, + rx::AddressRange, + rx::AddressRange) const { + return other; + } + + bool operator==(const BlockPoolAllocation &) const = default; +}; + +using DirectMemoryResource = + kernel::AllocableResource; struct BlockPoolFile : public orbis::File {}; +struct BlockPoolDevice + : orbis::IoDeviceWithIoctl { + rx::shared_mutex mtx; + rx::MemoryAreaTable<> pool; + orbis::sint availBlocks{}; + orbis::sint commitBlocks{}; -static orbis::ErrorCode blockpool_ioctl(orbis::File *file, - std::uint64_t request, void *argp, - orbis::Thread *thread) { - auto blockPool = static_cast(file->device.get()); - std::lock_guard lock(blockPool->mtx); + BlockPoolDevice(); - switch (request) { - case 0xc020a801: { - struct Args { - std::uint64_t len; - std::uint64_t searchStart; - std::uint64_t searchEnd; - std::uint32_t flags; - }; - auto args = reinterpret_cast(argp); - ORBIS_LOG_TODO("blockpool expand", args->len, args->searchStart, - args->searchEnd, args->flags); - - auto dmem = orbis::g_context->dmemDevice.rawStaticCast(); - std::lock_guard lock(dmem->mtx); - std::uint64_t start = args->searchStart; - ORBIS_RET_ON_ERROR( - dmem->allocate(&start, args->searchEnd, args->len, 1, args->flags)); - - blockPool->pool.map(rx::AddressRange::fromBeginSize(start, args->len)); - return {}; - } - } - - ORBIS_LOG_FATAL("Unhandled blockpool ioctl", request); - thread->where(); - return {}; -} - -static orbis::ErrorCode blockpool_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - auto blockPool = static_cast(file->device.get()); - std::lock_guard lock(blockPool->mtx); - ORBIS_LOG_FATAL("blockpool mmap", *address, size, offset); - - std::size_t totalBlockPoolSize = 0; - for (auto entry : blockPool->pool) { - totalBlockPoolSize += entry.endAddress - entry.beginAddress; - } - - if (totalBlockPoolSize < size) { - return orbis::ErrorCode::NOMEM; - } - - auto dmem = orbis::g_context->dmemDevice.rawStaticCast(); - auto mapped = reinterpret_cast(vm::map( - *address, size, prot, flags, vm::kMapInternalReserveOnly, blockPool)); - - if (mapped == MAP_FAILED) { - return orbis::ErrorCode::NOMEM; - } - - auto result = mapped; - - flags |= vm::kMapFlagFixed; - flags &= ~vm::kMapFlagNoOverwrite; - while (true) { - auto entry = *blockPool->pool.begin(); - auto blockSize = std::min(entry.endAddress - entry.beginAddress, size); - void *mapAddress = mapped; - ORBIS_LOG_FATAL("blockpool mmap", mapAddress, blockSize, entry.beginAddress, - blockSize); - ORBIS_RET_ON_ERROR( - dmem->mmap(&mapAddress, blockSize, prot, flags, entry.beginAddress)); - - mapped += blockSize; - size -= blockSize; - blockPool->pool.unmap(entry.beginAddress, entry.beginAddress + blockSize); - if (size == 0) { - break; - } - } - - *address = result; - return {}; -} - -static const orbis::FileOps ops = { - .ioctl = blockpool_ioctl, - .mmap = blockpool_mmap, + orbis::ErrorCode open(rx::Ref *file, const char *path, + std::uint32_t flags, std::uint32_t mode, + orbis::Thread *thread) override; + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override; }; +#pragma pack(push, 1) + +struct BlockPoolIoctlExpand { + orbis::uint64_t len; + orbis::uint64_t searchStart; + orbis::uint64_t searchEnd; + orbis::uint32_t flags; + orbis::uint32_t _padding; +}; +#pragma pack(pop) + +static orbis::ErrorCode blockpool_ioctl_expand(orbis::Thread *thread, + BlockPoolDevice *device, + BlockPoolIoctlExpand &args) { + ORBIS_LOG_TODO(__FUNCTION__, args.len, args.searchStart, args.searchEnd, + args.flags); + + if (args.len % orbis::dmem::kPageSize || args.len == 0) { + return orbis::ErrorCode::INVAL; + } + + auto alignment = args.flags == 0 ? 0 : 1ull << ((args.flags >> 24) & 0x1f); + + auto [dmemOffset, dmemErrc] = orbis::dmem::allocate( + 0, rx::AddressRange::fromBeginEnd(args.searchStart, args.searchEnd), + args.len, orbis::MemoryType::WbOnion, alignment, true); + + if (dmemErrc != orbis::ErrorCode{}) { + return dmemErrc; + } + + auto [pmemOffset, pmemErrc] = orbis::dmem::getPmemOffset(0, dmemOffset); + + if (pmemErrc != orbis::ErrorCode{}) { + return pmemErrc; + } + + args.searchStart = dmemOffset; + return orbis::blockpool::expand( + rx::AddressRange::fromBeginSize(pmemOffset, args.len)); +} + +static std::pair +blockpool_ioctl_get_block_stats(orbis::Thread *, BlockPoolDevice *) { + return {{}, orbis::blockpool::stats()}; +} + +static const orbis::FileOps ops = {}; + +BlockPoolDevice::BlockPoolDevice() { + blockFlags = orbis::vmem::BlockFlags::PooledMemory; + + addIoctl(blockpool_ioctl_expand); + addIoctl(blockpool_ioctl_get_block_stats); +} + orbis::ErrorCode BlockPoolDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, @@ -113,32 +120,17 @@ orbis::ErrorCode BlockPoolDevice::open(rx::Ref *file, return {}; } -orbis::ErrorCode BlockPoolDevice::map(void **address, std::uint64_t len, - std::int32_t prot, std::int32_t flags, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("blockpool device map", *address, len); - if (prot == 0) { - // FIXME: investigate it - prot = 0x33; +orbis::ErrorCode +BlockPoolDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) { + if (protection || offset != 0) { + return orbis::ErrorCode::INVAL; } - auto result = vm::map(*address, len, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::NOMEM; - } - - *address = result; return {}; } -orbis::ErrorCode BlockPoolDevice::unmap(void *address, std::uint64_t len, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("blockpool device unmap", address, len); - if (vm::unmap(address, len)) { - return {}; - } - return orbis::ErrorCode::INVAL; -} + orbis::IoDevice *createBlockPoolDevice() { return orbis::knew(); } diff --git a/rpcsx/iodev/blockpool.hpp b/rpcsx/iodev/blockpool.hpp deleted file mode 100644 index ec4ab6256..000000000 --- a/rpcsx/iodev/blockpool.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "orbis/IoDevice.hpp" -#include "orbis/error/ErrorCode.hpp" -#include "orbis/file.hpp" -#include "rx/MemoryTable.hpp" -#include "rx/Rc.hpp" -#include "rx/SharedMutex.hpp" -#include - -struct BlockPoolDevice : public orbis::IoDevice { - rx::shared_mutex mtx; - rx::MemoryAreaTable<> pool; - - orbis::ErrorCode open(rx::Ref *file, const char *path, - std::uint32_t flags, std::uint32_t mode, - orbis::Thread *thread) override; - orbis::ErrorCode map(void **address, std::uint64_t len, std::int32_t prot, - std::int32_t flags, orbis::Thread *thread); - orbis::ErrorCode unmap(void *address, std::uint64_t len, - orbis::Thread *thread); -}; diff --git a/rpcsx/iodev/dce.cpp b/rpcsx/iodev/dce.cpp index dcafe6b9e..da2f1c01d 100644 --- a/rpcsx/iodev/dce.cpp +++ b/rpcsx/iodev/dce.cpp @@ -1,18 +1,22 @@ #include "dce.hpp" #include "gpu/DeviceCtl.hpp" -#include "io-device.hpp" -#include "iodev/dmem.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/dmem.hpp" #include "orbis/error/ErrorCode.hpp" #include "orbis/file.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" #include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/mem.hpp" +#include "rx/print.hpp" #include "rx/watchdog.hpp" -#include "vm.hpp" #include #include #include @@ -144,7 +148,7 @@ struct ResolutionStatus { // refreshRate = 0x23, result.refreshHz = 0x42b3d1ec( 89.91) REFRESH_RATE_89_91HZ // clang-format on -static void runBridge(int vmId) { +static void runBridge(int vmId, orbis::Process *process) { std::thread{[=] { pthread_setname_np(pthread_self(), "Bridge"); @@ -186,29 +190,23 @@ static void runBridge(int vmId) { gpuCtx.cachePages[vmId][page].load(std::memory_order::relaxed); auto address = static_cast(page) * rx::mem::pageSize; - auto origVmProt = vm::getPageProtection(address); - int prot = 0; + auto range = + rx::AddressRange::fromBeginSize(address, rx::mem::pageSize * count); + auto origVmProt = orbis::vmem::queryProtection(process, address); + if (origVmProt && origVmProt->prot) { + auto prot = orbis::vmem::toCpuProtection(origVmProt->prot); - if (origVmProt & vm::kMapProtCpuRead) { - prot |= PROT_READ; - } - if (origVmProt & vm::kMapProtCpuWrite) { - prot |= PROT_WRITE; - } - if (origVmProt & vm::kMapProtCpuExec) { - prot |= PROT_EXEC; - } + if (pageFlags & amdgpu::kPageReadWriteLock) { + prot = prot & ~(rx::mem::Protection::R | rx::mem::Protection::W); + } else if (pageFlags & amdgpu::kPageWriteWatch) { + prot = prot & ~(rx::mem::Protection::W); + } - if (pageFlags & amdgpu::kPageReadWriteLock) { - prot &= ~(PROT_READ | PROT_WRITE); - } else if (pageFlags & amdgpu::kPageWriteWatch) { - prot &= ~PROT_WRITE; - } - - if (::mprotect(reinterpret_cast(address), - rx::mem::pageSize * count, prot)) { - perror("protection failed"); - std::abort(); + if (auto errc = rx::mem::protect(range, prot); errc != std::errc{}) { + rx::die("gpu cache: failed to protect memory {:x}-{:x}, error {}", + range.beginAddress(), range.endAddress(), + static_cast(errc)); + } } } @@ -237,43 +235,6 @@ int DceDevice::allocateVmId() { void DceDevice::deallocateVmId(int vmId) { freeVmIds |= (1 << vmId); } -static void initDceMemory(DceDevice *device) { - if (device->dmemOffset + 1) { - return; - } - - std::lock_guard lock(device->mtx); - if (device->dmemOffset + 1) { - return; - } - - auto dmem = orbis::g_context->dmemDevice.cast(); - std::uint64_t start = 0; - if (dmem->allocate(&start, ~0ull, kDceControlMemorySize, 0x100000, 1) != - orbis::ErrorCode{}) { - std::abort(); - } - - void *address = nullptr; - if (dmem->mmap(&address, kDceControlMemorySize, vm::kMapProtCpuWrite, 0, - start) != orbis::ErrorCode{}) { - std::abort(); - } - - auto dceControl = reinterpret_cast(address); - *reinterpret_cast(dceControl + 0x130) = 0; - *reinterpret_cast(dceControl + 0x138) = 1; - *reinterpret_cast(dceControl + 0x140) = - orbis::kEvFiltDisplay; - vm::unmap(address, kDceControlMemorySize); - device->dmemOffset = start; -} - -static orbis::ErrorCode dce_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread); - static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, void *argp, orbis::Thread *thread) { auto device = static_cast(file->device.get()); @@ -312,21 +273,30 @@ static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, } if (args->id == 9) { - ORBIS_LOG_NOTICE("dce: FlipControl allocate", args->id, args->padding, + ORBIS_LOG_NOTICE("dce: FlipControl map", args->id, args->padding, args->arg2, args->ptr, args->size, args->arg5, args->arg6); - void *address; - ORBIS_RET_ON_ERROR( - dce_mmap(file, &address, vm::kPageSize, - vm::kMapProtCpuReadWrite | vm::kMapProtGpuAll, - vm::kMapFlagShared, 0, thread)); + auto [range, errc] = orbis::vmem::mapDirect( + thread->tproc, 0, + rx::AddressRange::fromBeginSize(device->dmemRange.beginAddress(), + orbis::vmem::kPageSize), + orbis::vmem::Protection::CpuRead | + orbis::vmem::Protection::CpuWrite | + orbis::vmem::Protection::GpuRead | + orbis::vmem::Protection::GpuWrite, + {}, "DCE"); - *(void **)args->ptr = address; - *(std::uint64_t *)args->arg5 = vm::kPageSize; + if (errc != orbis::ErrorCode{}) { + return errc; + } + + *(std::uint64_t *)args->ptr = range.beginAddress(); + *(std::uint64_t *)args->arg5 = range.size(); return {}; } + if (args->id == 0x38) { auto attrs = (RegisterBufferAttributeArgs *)args->ptr; @@ -527,11 +497,11 @@ static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, if (request == 0xc0308207) { // SCE_SYS_DCE_IOCTL_REGISTER_BUFFER_ATTRIBUTE auto args = reinterpret_cast(argp); - ORBIS_LOG_ERROR("dce: RegisterBufferAttributes", args->canary, args->attrid, - args->submit, args->unk3, args->pixelFormat, - args->tilingMode, args->pitch, args->width, args->height, - args->unk4_zero, args->unk5_zero, args->options, - args->reserved1, args->reserved2); + ORBIS_LOG_ERROR( + "dce: RegisterBufferAttributes", args->canary, (int)args->attrid, + (int)args->submit, args->unk3, args->pixelFormat, args->tilingMode, + args->pitch, args->width, args->height, (int)args->unk4_zero, + (int)args->unk5_zero, args->options, args->reserved1, args->reserved2); gpu.registerBufferAttribute(thread->tproc->pid, { @@ -596,20 +566,30 @@ static orbis::ErrorCode dce_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode dce_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("dce mmap", address, size, offset); - auto dce = file->device.cast(); - initDceMemory(dce.get()); - auto dmem = orbis::g_context->dmemDevice.cast(); - return dmem->mmap(address, size, prot, flags, dce->dmemOffset + offset); +orbis::ErrorCode +DceDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *, orbis::Process *process) { + if (offset + range.size() > dmemRange.size()) { + return orbis::ErrorCode::INVAL; + } + + rx::println(stderr, "map dce {:x}-{:x} {:04x} {}", range.beginAddress(), + range.endAddress(), offset, protection); + + auto result = + orbis::dmem::map(0, range, dmemRange.beginAddress() + offset, protection); + + if (result == orbis::ErrorCode{}) { + amdgpu::mapMemory(process->pid, range, orbis::MemoryType::WbGarlic, + protection, dmemRange.beginAddress() + offset); + } + + return result; } static const orbis::FileOps ops = { .ioctl = dce_ioctl, - .mmap = dce_mmap, }; static void createGpu() { @@ -626,6 +606,8 @@ static void createGpu() { } } +DceDevice::~DceDevice() { orbis::dmem::release(0, dmemRange); } + orbis::ErrorCode DceDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) { @@ -649,8 +631,32 @@ void DceDevice::initializeProcess(orbis::Process *process) { process->vmId = vmId; } - runBridge(vmId); + runBridge(vmId, process); } } -orbis::IoDevice *createDceCharacterDevice() { return orbis::knew(); } +orbis::IoDevice *createDceCharacterDevice(orbis::Process *process) { + auto result = orbis::knew(); + auto dmemSize = orbis::dmem::getSize(0); + auto [dmemOffset, errc] = orbis::dmem::allocate( + 0, + rx::AddressRange::fromBeginEnd(dmemSize - orbis::dmem::kPageSize * 2, + dmemSize), + orbis::dmem::kPageSize, orbis::MemoryType::WbGarlic); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to allocate DCE memory, error {}", errc); + result->dmemRange = + rx::AddressRange::fromBeginSize(dmemOffset, orbis::dmem::kPageSize); + + auto [vmem, mapErrc] = orbis::vmem::mapDirect( + process, 0, result->dmemRange, orbis::vmem::Protection::CpuWrite, {}); + auto dceControl = reinterpret_cast(vmem.beginAddress()); + *reinterpret_cast(dceControl + 0x130) = 0; + *reinterpret_cast(dceControl + 0x138) = 1; + *reinterpret_cast(dceControl + 0x140) = + orbis::kEvFiltDisplay; + + orbis::vmem::unmap(process, vmem); + return result; +} diff --git a/rpcsx/iodev/dce.hpp b/rpcsx/iodev/dce.hpp index 38dd03741..1af40a03a 100644 --- a/rpcsx/iodev/dce.hpp +++ b/rpcsx/iodev/dce.hpp @@ -1,7 +1,7 @@ #pragma once -#include "io-device.hpp" #include "orbis-config.hpp" +#include "orbis/IoDevice.hpp" #include "orbis/error/ErrorCode.hpp" #include "orbis/file.hpp" #include "orbis/thread/Process.hpp" @@ -12,14 +12,19 @@ static constexpr auto kVmIdCount = 6; struct DceDevice : orbis::IoDevice { rx::shared_mutex mtx; + rx::AddressRange dmemRange; std::uint32_t eopCount = 0; std::uint32_t freeVmIds = (1 << (kVmIdCount + 1)) - 1; - orbis::uint64_t dmemOffset = ~static_cast(0); + + DceDevice() { blockFlags = orbis::vmem::BlockFlags::DirectMemory; } + ~DceDevice(); orbis::ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override; - + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override; int allocateVmId(); void deallocateVmId(int vmId); void initializeProcess(orbis::Process *process); diff --git a/rpcsx/iodev/dmem.cpp b/rpcsx/iodev/dmem.cpp index 1a3120c0a..d5eb755a5 100644 --- a/rpcsx/iodev/dmem.cpp +++ b/rpcsx/iodev/dmem.cpp @@ -1,390 +1,353 @@ -#include "dmem.hpp" +#include "orbis/dmem.hpp" #include "gpu/DeviceCtl.hpp" -#include "io-device.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/error.hpp" #include "orbis/file.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" -#include "rx/align.hpp" -#include "rx/format.hpp" -#include "rx/watchdog.hpp" -#include "vm.hpp" -#include -#include -#include -#include -#include +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/EnumBitSet.hpp" + +enum { + DMEM_IOCTL_ALLOCATE = 0xc0288001, + DMEM_IOCTL_RELEASE = 0x80108002, + DMEM_IOCTL_SET_TYPE = 0x80188003, + DMEM_IOCTL_GET_TYPE = 0xc0208004, + DMEM_IOCTL_GET_TOTAL_SIZE = 0x4008800a, + DMEM_IOCTL_CLEAR = 0x2000800b, + DMEM_IOCTL_TRANSFER_BUDGET = 0xc018800d, + DMEM_IOCTL_CONTROL_RELEASE = 0xc018800e, + DMEM_IOCTL_SET_PID_AND_PROTECT = 0xc018800f, + DMEM_IOCTL_ALLOCATE_FOR_MINI_APP = 0xc0288010, + DMEM_IOCTL_ALLOCATE_MAIN = 0xc0288011, + DMEM_IOCTL_QUERY = 0x80288012, + DMEM_IOCTL_CHECKED_RELEASE = 0x80108015, + DMEM_IOCTL_GET_AVAIL_SIZE = 0xc0208016, + DMEM_IOCTL_RESERVE = 0xc010801a, +}; +struct DmemDevice + : orbis::IoDeviceWithIoctl { + int index; + DmemDevice(int index); + + orbis::ErrorCode open(rx::Ref *file, const char *path, + std::uint32_t flags, std::uint32_t mode, + orbis::Thread *thread) override; + + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override; +}; struct DmemFile : public orbis::File {}; -struct AllocateDirectMemoryArgs { - std::uint64_t searchStart; - std::uint64_t searchEnd; - std::uint64_t len; - std::uint64_t alignment; - std::uint32_t memoryType; +#pragma pack(push, 1) +struct DmemIoctlAllocate { + orbis::uintptr_t searchStart; + orbis::uintptr_t searchEnd; + orbis::size_t len; + orbis::size_t alignment; + orbis::MemoryType memoryType; + orbis::uint32_t padding; }; -static constexpr auto dmemSize = 0x5000000000; -// static const std::uint64_t nextOffset = 0; -// static const std::uint64_t memBeginAddress = 0xfe0000000; +struct DmemIoctlRelease { + orbis::uintptr_t address; + orbis::size_t size; +}; +struct DmemIoctlSetType { + orbis::uintptr_t start; + orbis::uintptr_t end; + orbis::MemoryType memoryType; + orbis::uint32_t padding; +}; +struct DmemIoctlGetType { + orbis::uintptr_t start; + orbis::uintptr_t regionStart; + orbis::uintptr_t regionEnd; + orbis::MemoryType memoryType; + orbis::uint32_t padding; +}; +struct DmemIoctlTransferBudget { + orbis::uint64_t unk0; + orbis::uint64_t unk1; + orbis::uint64_t unk2; +}; +struct DmemIoctlControlRelease { + orbis::uint64_t unk0; + orbis::uint64_t unk1; + orbis::uint64_t unk2; +}; +struct DmemIoctlSetPidAndProtect { + orbis::uintptr_t address; + orbis::size_t size; + orbis::pid_t pid; // 0 if all + rx::EnumBitSet prot; +}; -DmemDevice::~DmemDevice() { - if (shmFd > 0) { - close(shmFd); +struct DirectMemoryQueryInfo { + orbis::uintptr_t start; + orbis::uintptr_t end; + orbis::MemoryType memoryType; + orbis::uint32_t padding; +}; +struct DmemIoctlQuery { + orbis::uint32_t devIndex; + orbis::uint32_t flags; + orbis::uint32_t unk; + orbis::uint32_t _padding; + orbis::uint64_t offset; + orbis::ptr info; + orbis::uint64_t infoSize; +}; +struct DmemIoctlGetAvailSize { + orbis::uintptr_t searchStart; + orbis::uintptr_t searchEnd; + orbis::size_t alignment; + orbis::size_t size; +}; + +struct DmemIoctlReserve { + orbis::size_t size; + orbis::uint32_t flags; + orbis::uint32_t padding; +}; +#pragma pack(pop) + +static orbis::ErrorCode dmem_ioctl_allocate(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlAllocate &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.searchStart, args.searchEnd, + args.alignment, args.len, (int)args.memoryType); + auto [offset, errc] = orbis::dmem::allocate( + device->index, + rx::AddressRange::fromBeginEnd(args.searchStart, args.searchEnd), + args.len, args.memoryType); + + if (errc != orbis::ErrorCode{}) { + return errc; } - std::filesystem::remove(rx::format("{}/dmem-{}", rx::getShmPath(), index)); + args.searchStart = offset; + return {}; } -orbis::ErrorCode DmemDevice::mmap(void **address, std::uint64_t len, - std::int32_t prot, std::int32_t flags, - std::int64_t directMemoryStart) { - if (prot == 0) { - // hack - // fixme: implement protect for pid - prot = vm::kMapProtCpuReadWrite | vm::kMapProtGpuAll; +static orbis::ErrorCode dmem_ioctl_release(orbis::Thread *thread, + DmemDevice *device, + const DmemIoctlRelease &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.address, args.size); + + return orbis::dmem::release( + device->index, rx::AddressRange::fromBeginSize(args.address, args.size)); +} + +static orbis::ErrorCode dmem_ioctl_set_type(orbis::Thread *thread, + DmemDevice *device, + const DmemIoctlSetType &args) { + // removed ioctl + return orbis::ErrorCode::INVAL; +} + +static orbis::ErrorCode dmem_ioctl_get_type(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlGetType &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.start); + + auto result = orbis::dmem::query(device->index, args.start); + + if (!result) { + return orbis::ErrorCode::NOENT; } - if (*address == nullptr) { - *address = std::bit_cast(0x80000000ull); - flags &= ~vm::kMapFlagFixed; + args.regionStart = result->range.beginAddress(); + args.regionEnd = result->range.endAddress(); + args.memoryType = result->memoryType; + return {}; +} + +static std::pair +dmem_ioctl_get_total_size(orbis::Thread *thread, DmemDevice *device) { + auto result = orbis::dmem::getSize(device->index); + ORBIS_LOG_WARNING(__FUNCTION__, result); + + auto limit = thread->tproc->getBudget()->get(orbis::BudgetResource::Dmem); + return {{}, orbis::uint64_t(std::min(result, limit.total))}; +} + +static orbis::ErrorCode dmem_ioctl_clear(orbis::Thread *thread, + DmemDevice *device) { + ORBIS_LOG_WARNING(__FUNCTION__); + auto result = orbis::dmem::clear(device->index); + if (result == orbis::ErrorCode{}) { + thread->tproc->getBudget()->release(orbis::BudgetResource::Dmem, -1); + } + return result; +} + +static orbis::ErrorCode +dmem_ioctl_transfer_budget(orbis::Thread *thread, DmemDevice *device, + DmemIoctlTransferBudget &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.unk0, args.unk1, args.unk2); + return {}; +} + +static orbis::ErrorCode +dmem_ioctl_control_release(orbis::Thread *thread, DmemDevice *device, + DmemIoctlControlRelease &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.unk0, args.unk1, args.unk2); + return {}; +} + +static orbis::ErrorCode +dmem_ioctl_set_pid_and_protect(orbis::Thread *thread, DmemDevice *device, + DmemIoctlSetPidAndProtect &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.pid, args.address, args.size, + args.prot.toUnderlying()); + return {}; +} + +static orbis::ErrorCode +dmem_ioctl_allocate_for_mini_app(orbis::Thread *thread, DmemDevice *device, + DmemIoctlAllocate &args) { + // FIXME: implement + return dmem_ioctl_allocate(thread, device, args); +} + +static orbis::ErrorCode dmem_ioctl_allocate_main(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlAllocate &args) { + // FIXME: implement + return dmem_ioctl_allocate(thread, device, args); +} + +static orbis::ErrorCode dmem_ioctl_query(orbis::Thread *thread, + DmemDevice *device, + const DmemIoctlQuery &args) { + ORBIS_LOG_WARNING(__FUNCTION__, device->index, args.devIndex, args.unk, + args.flags, args.offset, args.info, args.infoSize); + + if (args.devIndex != device->index) { + ORBIS_LOG_WARNING(__FUNCTION__, "device mismatch", device->index, + args.devIndex, args.unk, args.flags, args.offset, + args.info, args.infoSize); } - int memoryType = 0; - if (auto allocationInfoIt = allocations.queryArea(directMemoryStart); - allocationInfoIt != allocations.end()) { - memoryType = allocationInfoIt->memoryType; - } - - auto result = vm::map(*address, len, prot, flags, vm::kMapInternalReserveOnly, - this, directMemoryStart); - - ORBIS_LOG_WARNING("dmem mmap", index, directMemoryStart, len, prot, flags, - result, *address); - if (result == (void *)-1) { - return orbis::ErrorCode::NOMEM; // TODO - } - - if (::mmap(result, len, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, shmFd, - directMemoryStart) == (void *)-1) { + if (args.infoSize != sizeof(DirectMemoryQueryInfo) || args.devIndex >= 3) { return orbis::ErrorCode::INVAL; } - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitMapMemory(orbis::g_currentThread->tproc->pid, - reinterpret_cast(result), len, - memoryType, index, prot, directMemoryStart); + rx::EnumBitSet queryFlags = {}; + + if (args.flags & 1) { + queryFlags |= orbis::dmem::QueryFlags::LowerBound; } - *address = result; + auto result = orbis::dmem::query(args.devIndex, args.offset, queryFlags); + if (!result) { + return orbis::ErrorCode::ACCES; + } + + DirectMemoryQueryInfo info{ + .start = result->range.beginAddress(), + .end = result->range.endAddress(), + .memoryType = result->memoryType, + }; + + ORBIS_LOG_WARNING(__FUNCTION__, device->index, args.devIndex, args.unk, + args.flags, args.offset, args.info, args.infoSize, + info.start, info.end, (int)info.memoryType); + + return orbis::uwrite(args.info, info); +} + +static orbis::ErrorCode +dmem_ioctl_checked_release(orbis::Thread *thread, DmemDevice *device, + const DmemIoctlRelease &args) { + return dmem_ioctl_release(thread, device, args); +} + +static orbis::ErrorCode dmem_ioctl_get_avail_size(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlGetAvailSize &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.searchStart, args.searchEnd, + args.alignment, args.size); + + auto [range, errc] = orbis::dmem::getAvailSize( + device->index, + rx::AddressRange::fromBeginEnd(args.searchStart, args.searchEnd), + args.alignment); + + if (errc != orbis::ErrorCode{}) { + return errc; + } + + ORBIS_LOG_WARNING(__FUNCTION__, args.searchStart, args.searchEnd, + args.alignment, args.size, range.beginAddress(), + range.size()); + + args.searchStart = range.beginAddress(); + args.size = range.size(); return {}; } -static orbis::ErrorCode dmem_ioctl(orbis::File *file, std::uint64_t request, - void *argp, orbis::Thread *thread) { - auto device = file->device.rawStaticCast(); +static orbis::ErrorCode dmem_ioctl_reserve(orbis::Thread *thread, + DmemDevice *device, + DmemIoctlReserve &args) { + ORBIS_LOG_WARNING(__FUNCTION__, args.size, (int)args.flags); + auto [offset, errc] = orbis::dmem::reserveSystem(device->index, args.size); - std::lock_guard lock(device->mtx); - switch (request) { - case 0x4008800a: // get size - ORBIS_LOG_WARNING("dmem getTotalSize", device->index, argp); - *(std::uint64_t *)argp = device->dmemTotalSize / 0x10; - return {}; - - case 0xc0208016: { // get available size - struct Args { - std::uint64_t searchStart; - std::uint64_t searchEnd; - std::uint64_t alignment; - std::uint64_t size; - }; - - auto args = reinterpret_cast(argp); - - return device->queryMaxFreeChunkSize(&args->searchStart, args->searchEnd, - args->alignment, &args->size); + if (errc == orbis::ErrorCode{}) { + args.size = offset | 0x4000000000; } - - case 0xc0288010: // sceKernelAllocateDirectMemoryForMiniApp - case 0xc0288011: - case 0xc0288001: { // sceKernelAllocateDirectMemory - auto args = reinterpret_cast(argp); - - return device->allocate(&args->searchStart, args->searchEnd, args->len, - args->alignment, args->memoryType); - } - - case 0xc018800d: { // transfer budget - return {}; - } - - case 0xc018800f: { // protect memory for pid - struct Args { - std::uint64_t address; - std::uint64_t size; - std::uint32_t pid; // 0 if all - std::uint32_t prot; - }; - return {}; - } - - case 0x80108015: // sceKernelCheckedReleaseDirectMemory - case 0x80108002: { // sceKernelReleaseDirectMemory - struct Args { - std::uint64_t address; - std::uint64_t size; - }; - - auto args = reinterpret_cast(argp); - - ORBIS_LOG_WARNING("dmem releaseDirectMemory", device->index, args->address, - args->size); - - device->allocations.map( - rx::AddressRange::fromBeginSize(args->address, args->size), - {.memoryType = -1u}); - return {}; - } - - case 0xc0208004: { // get direct memory type - struct Args { - std::uint64_t start; - std::uint64_t regionStart; - std::uint64_t regionEnd; - std::uint32_t memoryType; - }; - - auto args = reinterpret_cast(argp); - - auto it = device->allocations.lowerBound(args->start); - - if (it == device->allocations.end() || it->memoryType == -1u) { - return orbis::ErrorCode::SRCH; - } - - args->regionStart = it.beginAddress(); - args->regionEnd = it.endAddress(); - args->memoryType = it->memoryType; - return {}; - } - - case 0x80288012: { // direct memory query - struct DirectMemoryQueryInfo { - std::uint64_t start; - std::uint64_t end; - std::uint32_t memoryType; - }; - - struct Args { - std::uint32_t devIndex; - std::uint32_t flags; - std::uint32_t unk; - std::uint64_t offset; - orbis::ptr info; - std::uint64_t infoSize; - }; - - auto args = reinterpret_cast(argp); - - ORBIS_LOG_WARNING("dmem directMemoryQuery", device->index, args->devIndex, - args->unk, args->flags, args->offset, args->info, - args->infoSize); - - if (args->devIndex != device->index) { - // TODO - ORBIS_LOG_ERROR("dmem directMemoryQuery: device mismatch", device->index, - args->devIndex, args->unk, args->flags, args->offset, - args->info, args->infoSize); - - return orbis::ErrorCode::INVAL; - } - - if (args->infoSize != sizeof(DirectMemoryQueryInfo)) { - return orbis::ErrorCode::INVAL; - } - - auto it = device->allocations.lowerBound(args->offset); - - if (it == device->allocations.end()) { - return orbis::ErrorCode::ACCES; - } - - auto queryInfo = *it; - - if (it->memoryType == -1u) { - return orbis::ErrorCode::ACCES; - } - - if ((args->flags & 1) == 0) { - if (it.endAddress() <= args->offset) { - return orbis::ErrorCode::ACCES; - } - } else { - if (it.beginAddress() > args->offset || it.endAddress() <= args->offset) { - return orbis::ErrorCode::ACCES; - } - } - - DirectMemoryQueryInfo info{ - .start = it.beginAddress(), - .end = it.endAddress(), - .memoryType = it->memoryType, - }; - - ORBIS_LOG_WARNING("dmem directMemoryQuery", device->index, args->devIndex, - args->unk, args->flags, args->offset, args->info, - args->infoSize, info.start, info.end, info.memoryType); - return orbis::uwrite(args->info, info); - } - } - - ORBIS_LOG_FATAL("Unhandled dmem ioctl", device->index, request); - thread->where(); - return {}; + return errc; } -static orbis::ErrorCode dmem_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - auto device = file->device.rawStaticCast(); - return device->mmap(address, size, prot, flags, offset); +DmemDevice::DmemDevice(int index) : index(index) { + blockFlags = orbis::vmem::BlockFlags::DirectMemory; + + addIoctl(dmem_ioctl_allocate); + addIoctl(dmem_ioctl_release); + addIoctl(dmem_ioctl_set_type); + addIoctl(dmem_ioctl_get_type); + addIoctl(dmem_ioctl_get_total_size); + addIoctl(dmem_ioctl_clear); + addIoctl(dmem_ioctl_transfer_budget); + addIoctl(dmem_ioctl_control_release); + addIoctl(dmem_ioctl_set_pid_and_protect); + addIoctl(dmem_ioctl_allocate_for_mini_app); + addIoctl(dmem_ioctl_allocate_main); + addIoctl(dmem_ioctl_query); + addIoctl(dmem_ioctl_checked_release); + addIoctl(dmem_ioctl_get_avail_size); + addIoctl(dmem_ioctl_reserve); } -static const orbis::FileOps ops = { - .ioctl = dmem_ioctl, - .mmap = dmem_mmap, -}; +orbis::ErrorCode +DmemDevice::map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *, orbis::Process *process) { + auto result = orbis::dmem::map(index, range, offset, protection); -orbis::ErrorCode DmemDevice::allocate(std::uint64_t *start, - std::uint64_t searchEnd, - std::uint64_t len, - std::uint64_t alignment, - std::uint32_t memoryType) { - std::size_t offset = *start; - if (alignment == 0) { - alignment = 1; - } - if (searchEnd == 0) { - searchEnd = dmemTotalSize; - } + if (result == orbis::ErrorCode{}) { + if (auto dmemType = orbis::dmem::query(0, offset)) { + auto [pmemOffset, errc] = orbis::dmem::getPmemOffset(0, offset); + rx::dieIf(errc != orbis::ErrorCode{}, "failed to query dmem type {}", + errc); - while (offset < searchEnd) { - offset += alignment - 1; - offset &= ~(alignment - 1); - - if (offset + len > dmemTotalSize) { - ORBIS_LOG_ERROR("dmem: failed to allocate direct memory: out of memory", - *start, searchEnd, len, alignment, memoryType, offset); - return orbis::ErrorCode::AGAIN; + amdgpu::mapMemory(process->pid, range, dmemType->memoryType, protection, + pmemOffset); } - - auto it = allocations.lowerBound(offset); - - if (it != allocations.end()) { - if (it->memoryType == -1u) { - if (offset < it.beginAddress()) { - offset = it.beginAddress() + alignment - 1; - offset &= ~(alignment - 1); - } - - if (offset + len >= it.endAddress()) { - offset = it.endAddress(); - continue; - } - } else { - if (offset + len > it.beginAddress()) { - offset = it.endAddress(); - continue; - } - } - } - - allocations.map(rx::AddressRange::fromBeginSize(offset, len), - { - .memoryType = memoryType, - }); - ORBIS_LOG_WARNING("dmem: allocated direct memory", *start, searchEnd, len, - alignment, memoryType, offset); - *start = offset; - return {}; } - ORBIS_LOG_ERROR("dmem: failed to allocate direct memory", *start, searchEnd, - len, alignment, memoryType, offset); - return orbis::ErrorCode::AGAIN; + return result; } -orbis::ErrorCode DmemDevice::release(std::uint64_t start, std::uint64_t size) { - allocations.unmap(rx::AddressRange::fromBeginSize(start, size)); - return {}; -} - -orbis::ErrorCode DmemDevice::queryMaxFreeChunkSize(std::uint64_t *start, - std::uint64_t searchEnd, - std::uint64_t alignment, - std::uint64_t *size) { - std::size_t offset = *start; - std::size_t resultSize = 0; - std::size_t resultOffset = 0; - - alignment = std::max(alignment, vm::kPageSize); - alignment = rx::alignUp(alignment, vm::kPageSize); - - while (offset < searchEnd) { - offset += alignment - 1; - offset &= ~(alignment - 1); - - if (offset >= dmemTotalSize) { - break; - } - - auto it = allocations.lowerBound(offset); - - if (it == allocations.end()) { - if (resultSize < dmemTotalSize - offset) { - resultSize = dmemTotalSize - offset; - resultOffset = offset; - } - - break; - } - - if (it->memoryType == -1u) { - if (offset < it.beginAddress()) { - offset = it.beginAddress() + alignment - 1; - offset &= ~(alignment - 1); - } - - if (it.endAddress() > offset && resultSize < it.endAddress() - offset) { - resultSize = it.endAddress() - offset; - resultOffset = offset; - } - } else if (offset > it.beginAddress() && - resultSize < offset - it.beginAddress()) { - resultSize = offset - it.beginAddress(); - resultOffset = offset; - } - - offset = it.endAddress(); - } - - resultSize /= 0x20; - - *start = resultOffset; - *size = resultSize; - - ORBIS_LOG_WARNING("dmem queryMaxFreeChunkSize", resultOffset, resultSize); - - if (resultSize == 0) { - return orbis::ErrorCode::NOMEM; - } - return {}; -} +static const orbis::FileOps ops = {}; orbis::ErrorCode DmemDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, @@ -397,18 +360,6 @@ orbis::ErrorCode DmemDevice::open(rx::Ref *file, const char *path, } orbis::IoDevice *createDmemCharacterDevice(int index) { - auto *newDevice = orbis::knew(); - newDevice->index = index; - newDevice->dmemTotalSize = dmemSize; - - auto path = rx::format("{}/dmem-{}", rx::getShmPath(), index); - auto shmFd = ::open(path.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); - - if (ftruncate(shmFd, dmemSize) < 0) { - ::close(shmFd); - std::abort(); - } - - newDevice->shmFd = shmFd; + auto *newDevice = orbis::knew(index); return newDevice; } diff --git a/rpcsx/iodev/dmem.hpp b/rpcsx/iodev/dmem.hpp deleted file mode 100644 index 34bc997f0..000000000 --- a/rpcsx/iodev/dmem.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include "orbis/IoDevice.hpp" -#include "orbis/KernelAllocator.hpp" -#include "orbis/error/ErrorCode.hpp" -#include "orbis/file.hpp" -#include "rx/Rc.hpp" -#include "rx/SharedMutex.hpp" -#include -#include -#include - -struct DmemDevice : public orbis::IoDevice { - rx::shared_mutex mtx; - int index; - int shmFd = -1; - std::size_t dmemTotalSize; - - ~DmemDevice(); - - struct AllocationInfo { - std::uint32_t memoryType; - - bool operator==(const AllocationInfo &) const = default; - auto operator<=>(const AllocationInfo &) const = default; - }; - - rx::MemoryTableWithPayload allocations; - - orbis::ErrorCode allocate(std::uint64_t *start, std::uint64_t searchEnd, - std::uint64_t len, std::uint64_t alignment, - std::uint32_t memoryType); - - orbis::ErrorCode queryMaxFreeChunkSize(std::uint64_t *start, - std::uint64_t searchEnd, - std::uint64_t alignment, - std::uint64_t *size); - - orbis::ErrorCode release(std::uint64_t start, std::uint64_t size); - - orbis::ErrorCode open(rx::Ref *file, const char *path, - std::uint32_t flags, std::uint32_t mode, - orbis::Thread *thread) override; - - orbis::ErrorCode mmap(void **address, std::uint64_t len, std::int32_t prot, - std::int32_t flags, std::int64_t directMemoryStart); -}; diff --git a/rpcsx/iodev/gc.cpp b/rpcsx/iodev/gc.cpp index 6647f25e5..b3caf1b49 100644 --- a/rpcsx/iodev/gc.cpp +++ b/rpcsx/iodev/gc.cpp @@ -1,17 +1,20 @@ +#include "dce.hpp" #include "gpu/DeviceCtl.hpp" -#include "iodev/dce.hpp" -#include "iodev/dmem.hpp" #include "orbis/IoDevice.hpp" #include "orbis/KernelAllocator.hpp" #include "orbis/KernelContext.hpp" +#include "orbis/dmem.hpp" #include "orbis/file.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" #include "rx/SharedMutex.hpp" #include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/print.hpp" -#include "vm.hpp" #include #include #include @@ -26,15 +29,44 @@ struct ComputeQueue { struct GcDevice : public orbis::IoDevice { rx::shared_mutex mtx; + rx::AddressRange dmemRange; orbis::kmap clients; orbis::kmap computeQueues; - void *submitArea = nullptr; + orbis::uintptr_t submitArea = 0; orbis::ErrorCode open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, orbis::Thread *thread) override; void addClient(orbis::Process *process); void removeClient(orbis::Process *process); + + GcDevice() { blockFlags = orbis::vmem::BlockFlags::DirectMemory; } + + ~GcDevice() { orbis::pmem::deallocate(dmemRange); } + + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *process) override { + if (offset + range.size() > dmemRange.size()) { + return orbis::ErrorCode::INVAL; + } + + rx::println(stderr, "map gc {:x}-{:x} {:04x} {}", range.beginAddress(), + range.endAddress(), offset, protection); + + auto result = + orbis::pmem::map(range.beginAddress(), + rx::AddressRange::fromBeginSize( + dmemRange.beginAddress() + offset, range.size()), + orbis::vmem::toCpuProtection(protection)); + + if (result == orbis::ErrorCode{}) { + amdgpu::mapMemory(process->pid, range, orbis::MemoryType::WbOnion, + protection, dmemRange.beginAddress() + offset); + } + + return result; + } }; struct GcFile : public orbis::File { @@ -55,26 +87,35 @@ static orbis::ErrorCode gc_ioctl(orbis::File *file, std::uint64_t request, switch (request) { case 0xc008811b: // get submit done flag ptr? - if (device->submitArea == nullptr) { - auto dmem = orbis::g_context->dmemDevice.staticCast(); - std::uint64_t start = 0; - auto err = dmem->allocate(&start, ~0, vm::kPageSize, 0, 0); - if (err != orbis::ErrorCode{}) { - return err; + if (device->submitArea == 0) { + auto [dmemOffset, dmemErrc] = orbis::dmem::allocate( + 0, rx::AddressRange::fromBeginEnd(0, 0), orbis::dmem::kPageSize, + orbis::MemoryType::WbGarlic); + + if (dmemErrc != orbis::ErrorCode{}) { + return dmemErrc; } - auto address = reinterpret_cast(0xfe0100000); - err = dmem->mmap(&address, vm::kPageSize, - vm::kMapProtCpuReadWrite | vm::kMapProtGpuAll, - vm::kMapFlagShared, start); - if (err != orbis::ErrorCode{}) { - dmem->release(start, vm::kPageSize); - return err; + + auto directRange = + rx::AddressRange::fromBeginSize(dmemOffset, orbis::dmem::kPageSize); + + auto [vmemRange, vmemErrc] = orbis::vmem::mapDirect( + thread->tproc, 0xfe0100000, directRange, + orbis::vmem::Protection::CpuRead | orbis::vmem::Protection::CpuWrite | + orbis::vmem::Protection::GpuRead | + orbis::vmem::Protection::GpuWrite, + {}); + + if (vmemErrc != orbis::ErrorCode{}) { + orbis::dmem::release(0, directRange); + return dmemErrc; } - device->submitArea = address; + + device->submitArea = vmemRange.beginAddress(); } ORBIS_LOG_ERROR("gc ioctl 0xc008811b", *(std::uint64_t *)argp); - *reinterpret_cast(argp) = device->submitArea; + *reinterpret_cast(argp) = device->submitArea; break; case 0xc004812e: { @@ -438,25 +479,7 @@ static orbis::ErrorCode gc_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode gc_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("gc mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - -static const orbis::FileOps ops = { - .ioctl = gc_ioctl, - .mmap = gc_mmap, -}; +static const orbis::FileOps ops = {.ioctl = gc_ioctl}; orbis::ErrorCode GcDevice::open(rx::Ref *file, const char *path, std::uint32_t flags, std::uint32_t mode, @@ -490,4 +513,18 @@ void GcDevice::removeClient(orbis::Process *process) { } } -orbis::IoDevice *createGcCharacterDevice() { return orbis::knew(); } +orbis::IoDevice *createGcCharacterDevice() { + auto result = orbis::knew(); + auto dmemSize = orbis::dmem::getSize(0); + auto [dmemOffset, errc] = orbis::dmem::allocate( + 0, + rx::AddressRange::fromBeginEnd(dmemSize - orbis::dmem::kPageSize * 2, + dmemSize), + orbis::dmem::kPageSize, orbis::MemoryType::WbGarlic); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to allocate GC memory, error {}", errc); + result->dmemRange = + rx::AddressRange::fromBeginSize(dmemOffset, orbis::dmem::kPageSize); + return result; +} diff --git a/rpcsx/iodev/hmd_mmap.cpp b/rpcsx/iodev/hmd_mmap.cpp index dc9909e95..c9d6e0157 100644 --- a/rpcsx/iodev/hmd_mmap.cpp +++ b/rpcsx/iodev/hmd_mmap.cpp @@ -2,7 +2,6 @@ #include "orbis/KernelAllocator.hpp" #include "orbis/file.hpp" #include "orbis/utils/Logs.hpp" -#include "vm.hpp" struct HmdMmapDevice : public orbis::IoDevice { orbis::ErrorCode open(rx::Ref *file, const char *path, @@ -19,24 +18,8 @@ static orbis::ErrorCode hmd_mmap_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode hmd_mmap_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("hmd_mmap mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - static const orbis::FileOps ops = { .ioctl = hmd_mmap_ioctl, - .mmap = hmd_mmap_mmap, }; orbis::ErrorCode HmdMmapDevice::open(rx::Ref *file, diff --git a/rpcsx/iodev/rng.cpp b/rpcsx/iodev/rng.cpp index e089f8a7b..8f3041440 100644 --- a/rpcsx/iodev/rng.cpp +++ b/rpcsx/iodev/rng.cpp @@ -2,7 +2,6 @@ #include "orbis/KernelAllocator.hpp" #include "orbis/file.hpp" #include "orbis/utils/Logs.hpp" -#include "vm.hpp" struct RngDevice : public orbis::IoDevice { orbis::ErrorCode open(rx::Ref *file, const char *path, @@ -19,24 +18,8 @@ static orbis::ErrorCode rng_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode rng_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("rng mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - static const orbis::FileOps ops = { - .ioctl = rng_ioctl, - .mmap = rng_mmap, + .ioctl = rng_ioctl }; orbis::ErrorCode RngDevice::open(rx::Ref *file, const char *path, diff --git a/rpcsx/iodev/sbl_srv.cpp b/rpcsx/iodev/sbl_srv.cpp index e08269a57..cb6b565aa 100644 --- a/rpcsx/iodev/sbl_srv.cpp +++ b/rpcsx/iodev/sbl_srv.cpp @@ -5,7 +5,6 @@ #include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" #include "rx/SharedMutex.hpp" -#include "vm.hpp" struct SblSrvFile : public orbis::File {}; @@ -23,24 +22,8 @@ static orbis::ErrorCode sbl_srv_ioctl(orbis::File *file, std::uint64_t request, return {}; } -static orbis::ErrorCode sbl_srv_mmap(orbis::File *file, void **address, - std::uint64_t size, std::int32_t prot, - std::int32_t flags, std::int64_t offset, - orbis::Thread *thread) { - ORBIS_LOG_FATAL("sbl_srv mmap", address, size, offset); - auto result = vm::map(*address, size, prot, flags); - - if (result == (void *)-1) { - return orbis::ErrorCode::INVAL; // TODO - } - - *address = result; - return {}; -} - static const orbis::FileOps ops = { .ioctl = sbl_srv_ioctl, - .mmap = sbl_srv_mmap, }; orbis::ErrorCode SblSrvDevice::open(rx::Ref *file, diff --git a/rpcsx/ipmi.cpp b/rpcsx/ipmi.cpp index 0f981df68..3b109e14f 100644 --- a/rpcsx/ipmi.cpp +++ b/rpcsx/ipmi.cpp @@ -4,15 +4,20 @@ #include "io-device.hpp" #include "orbis/KernelContext.hpp" #include "orbis/osem.hpp" +#include "orbis/pmem.hpp" #include "orbis/thread/Process.hpp" +#include "orbis/thread/Thread.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/align.hpp" +#include "rx/die.hpp" #include "rx/format.hpp" #include "rx/hexdump.hpp" -#include "rx/mem.hpp" #include "rx/print.hpp" #include "rx/watchdog.hpp" #include "vfs.hpp" -#include "vm.hpp" +#include #include #include #include @@ -21,45 +26,65 @@ ipmi::IpmiClient ipmi::audioIpmiClient; +orbis::Process *g_workerProcess; + template struct GuestAlloc { orbis::ptr guestAddress; + std::size_t size; - GuestAlloc(std::size_t size) { + GuestAlloc(std::size_t size) : size(size) { if (size == 0) { guestAddress = nullptr; } else { - guestAddress = orbis::ptr( - vm::map(nullptr, size, vm::kMapProtCpuRead | vm::kMapProtCpuWrite, - vm::kMapFlagPrivate | vm::kMapFlagAnonymous)); + size = rx::alignUp(size, orbis::vmem::kPageSize); + + auto [range, vmemErrc] = orbis::vmem::mapFlex( + g_workerProcess ? g_workerProcess : orbis::g_currentThread->tproc, + size, + orbis::vmem::Protection::CpuRead | orbis::vmem::Protection::CpuWrite); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "impi: failed to map memory, error {}", + static_cast(vmemErrc)); + + guestAddress = std::bit_cast>(range.beginAddress()); } } GuestAlloc() : GuestAlloc(sizeof(T)) {} GuestAlloc(const T &data) : GuestAlloc() { - if (orbis::uwrite(guestAddress, data) != orbis::ErrorCode{}) { - std::abort(); + if (auto errc = orbis::uwrite(guestAddress, data); + errc != orbis::ErrorCode{}) { + rx::die("ipmi: failed to write data to allocated page {}, error {}", + (void *)guestAddress, errc); } } GuestAlloc(const void *data, std::size_t size) : GuestAlloc(size) { - if (orbis::uwriteRaw(guestAddress, data, size) != orbis::ErrorCode{}) { - std::abort(); + if (auto errc = orbis::uwriteRaw(guestAddress, data, size); + errc != orbis::ErrorCode{}) { + rx::die("ipmi: failed to write data to allocated page {}, error {}, data " + "{}, size {}", + (void *)guestAddress, errc, data, size); } } GuestAlloc(const GuestAlloc &) = delete; GuestAlloc(GuestAlloc &&other) noexcept : guestAddress(other.guestAddress) { - other.guestAddress = 0; + other.guestAddress = nullptr; } GuestAlloc &operator=(GuestAlloc &&other) noexcept { std::swap(guestAddress, other.guestAddress); } ~GuestAlloc() { - if (guestAddress != 0) { - vm::unmap(guestAddress, sizeof(T)); + if (guestAddress != nullptr) { + orbis::vmem::unmap( + g_workerProcess ? g_workerProcess : orbis::g_currentThread->tproc, + rx::AddressRange::fromBeginSize( + std::bit_cast(guestAddress), size)); } } @@ -117,6 +142,10 @@ orbis::sint ipmi::IpmiClient::sendSyncMessageRaw( return serverResult; } +void ipmi::setWorkerProcess(orbis::Process *process) { + g_workerProcess = process; +} + ipmi::IpmiClient ipmi::createIpmiClient(orbis::Thread *thread, const char *name) { rx::Ref client; diff --git a/rpcsx/ipmi.hpp b/rpcsx/ipmi.hpp index c66b5e8b3..85afe2ea5 100644 --- a/rpcsx/ipmi.hpp +++ b/rpcsx/ipmi.hpp @@ -258,6 +258,8 @@ struct IpmiServer { extern ipmi::IpmiClient audioIpmiClient; +void setWorkerProcess(orbis::Process *process); + IpmiClient createIpmiClient(orbis::Thread *thread, const char *name); IpmiServer &createIpmiServer(orbis::Process *process, const char *name); orbis::EventFlag *createEventFlag(std::string_view name, uint32_t attrs, diff --git a/rpcsx/linker.cpp b/rpcsx/linker.cpp index fec2d8bd6..0808afd48 100644 --- a/rpcsx/linker.cpp +++ b/rpcsx/linker.cpp @@ -1,11 +1,23 @@ #include "linker.hpp" -#include "io-device.hpp" +#include "kernel/KernelObject.hpp" +#include "orbis/IoDevice.hpp" #include "orbis/KernelAllocator.hpp" +#include "orbis/KernelObject.hpp" +#include "orbis/fmem.hpp" #include "orbis/module/Module.hpp" +#include "orbis/pmem.hpp" #include "orbis/stat.hpp" #include "orbis/uio.hpp" +#include "orbis/vmem.hpp" +#include "rx/AddressRange.hpp" +#include "rx/SharedMutex.hpp" +#include "rx/StrUtil.hpp" +#include "rx/debug.hpp" +#include "rx/die.hpp" +#include "rx/format.hpp" +#include "rx/mem.hpp" +#include "rx/print.hpp" #include "vfs.hpp" -#include "vm.hpp" #include #include #include @@ -16,8 +28,6 @@ #include #include -std::uint64_t monoPimpAddress; - static std::vector unself(const std::byte *image, std::size_t size) { struct [[gnu::packed]] Header { std::uint32_t magic; @@ -70,8 +80,7 @@ static std::vector unself(const std::byte *image, std::size_t size) { auto &segment = segments[i]; if ((segment.flags & 0x7fb) != 0 || segment.decryptedSize != segment.encryptedSize) { - std::fprintf(stderr, "Unsupported self segment (%lx)\n", segment.flags); - std::abort(); + rx::die("Unsupported self segment ({:x})", segment.flags); } if (~segment.flags & 0x800) { @@ -92,6 +101,8 @@ static std::vector unself(const std::byte *image, std::size_t size) { return result; } +std::uint64_t monoPimpAddress; + std::uint64_t rx::linker::encodeFid(std::string_view fid) { static const char suffix[] = "\x51\x8D\x64\xA6\x35\xDE\xD8\xC1\xE6\xB0\x39\xB1\xC3\xE5\x52\x30"; @@ -359,12 +370,159 @@ void rx::linker::override(std::string originalModuleName, std::move(replacedModulePath); } -rx::Ref rx::linker::loadModule(std::span image, - orbis::Process *process) { +struct ElfFile : orbis::File { + rx::AddressRange physicalMemory; + orbis::kvector image; + bool initialized = false; +}; + +struct ElfDevice : orbis::IoDevice { + rx::shared_mutex mtx; + orbis::kmap, rx::StringLess> files; + + orbis::ErrorCode open(rx::Ref *file, const char *path, + std::uint32_t flags, std::uint32_t mode, + orbis::Thread *thread) override { + std::lock_guard lock(mtx); + auto it = files.lower_bound(path); + + if (it != files.end() && it->first == path) { + *file = it->second; + return {}; + } + + rx::Ref rawFile; + if (auto result = + vfs::open(path, orbis::kOpenFlagReadOnly, 0, &rawFile, thread)) { + return result.errc(); + } + + orbis::Stat fileStat; + if (auto error = rawFile->ops->stat(rawFile.get(), &fileStat, nullptr); + error != orbis::ErrorCode{}) { + return error; + } + + auto len = fileStat.size; + + std::vector image(len); + auto ptr = image.data(); + orbis::IoVec ioVec{ + .base = ptr, + .len = static_cast(len), + }; + + orbis::Uio io{ + .offset = 0, + .iov = &ioVec, + .iovcnt = 1, + .resid = 0, + .segflg = orbis::UioSeg::SysSpace, + .rw = orbis::UioRw::Read, + .td = thread, + }; + + while (io.offset < image.size()) { + ioVec = { + .base = ptr + io.offset, + .len = image.size() - io.offset, + }; + + auto result = rawFile->ops->read(rawFile.get(), &io, thread); + rx::dieIf(result != orbis::ErrorCode{}, "elf: '{}' read failed: {}", path, + static_cast(result)); + } + + rawFile = {}; + + if (image[0] != std::byte{'\x7f'} || image[1] != std::byte{'E'} || + image[2] != std::byte{'L'} || image[3] != std::byte{'F'}) { + image = unself(image.data(), image.size()); + } + + Elf64_Ehdr header; + std::memcpy(&header, image.data(), sizeof(Elf64_Ehdr)); + + Elf64_Phdr phdrsStorage[16]; + if (header.e_phnum > std::size(phdrsStorage)) { + rx::die("elf: unexpected count of segments {}, {}", header.e_phnum, path); + } + + std::memcpy(phdrsStorage, image.data() + header.e_phoff, + header.e_phnum * sizeof(Elf64_Phdr)); + auto phdrs = std::span(phdrsStorage, header.e_phnum); + + std::uint64_t endAddress = 0; + std::uint64_t baseAddress = ~static_cast(0); + + for (auto &phdr : phdrs) { + switch (phdr.p_type) { + case kElfProgramTypeLoad: + baseAddress = + std::min(baseAddress, rx::alignDown(phdr.p_vaddr, phdr.p_align)); + + if ((phdr.p_flags & PF_W) == 0) { + endAddress = + std::max(endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, + orbis::vmem::kPageSize)); + } + break; + } + } + + auto imageSize = endAddress - baseAddress; + auto alignedImageSize = rx::alignUp(imageSize, orbis::vmem::kPageSize); + + auto [allocationRange, errc] = orbis::pmem::allocate( + orbis::pmem::getSize() - 1, alignedImageSize, + orbis::AllocationFlags::Stack, orbis::vmem::kPageSize); + + if (errc != orbis::ErrorCode{}) { + return errc; + } + + auto newFile = orbis::knew(); + newFile->physicalMemory = allocationRange; + newFile->image = orbis::kvector(image.begin(), image.end()); + newFile->device = this; + *file = newFile; + + files.emplace_hint(it, orbis::kstring(path), newFile); + return {}; + } + + orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset, + rx::EnumBitSet protection, + orbis::File *file, orbis::Process *) override { + auto elf = static_cast(file); + auto physicalRange = rx::AddressRange::fromBeginSize( + elf->physicalMemory.beginAddress() + offset, range.size()); + if (!physicalRange.isValid() || + physicalRange.endAddress() > elf->physicalMemory.endAddress()) { + return orbis::ErrorCode::INVAL; + } + + return orbis::pmem::map(range.beginAddress(), physicalRange, + orbis::vmem::toCpuProtection(protection)); + } + + void serialize(rx::Serializer &s) const { + // FIXME: implement + } + + void deserialize(rx::Deserializer &d) { + // FIXME: implement + } +}; + +static auto g_elfDevice = orbis::createGlobalObject(); + +static rx::Ref loadModule(ElfFile *elf, orbis::Process *process, + std::string_view name) { rx::Ref result{orbis::knew()}; Elf64_Ehdr header; - std::memcpy(&header, image.data(), sizeof(Elf64_Ehdr)); + std::memcpy(&header, elf->image.data(), sizeof(Elf64_Ehdr)); result->type = header.e_type; Elf64_Phdr phdrsStorage[16]; @@ -372,7 +530,7 @@ rx::Ref rx::linker::loadModule(std::span image, std::abort(); } - std::memcpy(phdrsStorage, image.data() + header.e_phoff, + std::memcpy(phdrsStorage, elf->image.data() + header.e_phoff, header.e_phnum * sizeof(Elf64_Phdr)); auto phdrs = std::span(phdrsStorage, header.e_phnum); @@ -402,8 +560,8 @@ rx::Ref rx::linker::loadModule(std::span image, case kElfProgramTypeLoad: baseAddress = std::min(baseAddress, rx::alignDown(phdr.p_vaddr, phdr.p_align)); - endAddress = std::max( - endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, vm::kPageSize)); + endAddress = std::max(endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, + orbis::vmem::kPageSize)); break; case kElfProgramTypeDynamic: dynamicPhdrIndex = index; @@ -436,8 +594,8 @@ rx::Ref rx::linker::loadModule(std::span image, sceRelRoPhdrIndex = index; baseAddress = std::min(baseAddress, rx::alignDown(phdr.p_vaddr, phdr.p_align)); - endAddress = std::max( - endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, vm::kPageSize)); + endAddress = std::max(endAddress, rx::alignUp(phdr.p_vaddr + phdr.p_memsz, + orbis::vmem::kPageSize)); break; case kElfProgramTypeGnuEhFrame: gnuEhFramePhdrIndex = index; @@ -455,17 +613,27 @@ rx::Ref rx::linker::loadModule(std::span image, } auto imageSize = endAddress - baseAddress; + auto alignedImageSize = rx::alignUp(imageSize, orbis::vmem::kPageSize); - auto imageBase = reinterpret_cast( - vm::map(reinterpret_cast(baseAddress), - rx::alignUp(imageSize, vm::kPageSize), 0, - vm::kMapFlagPrivate | vm::kMapFlagAnonymous | - (baseAddress ? vm::kMapFlagFixed : 0))); + rx::EnumBitSet allocationFlags = {}; - if (imageBase == MAP_FAILED) { - std::abort(); + auto mapHitAddress = baseAddress; + if (baseAddress != 0) { + allocationFlags |= orbis::AllocationFlags::Fixed; + } else if (header.e_type == rx::linker::kElfTypeDyn || + header.e_type == rx::linker::kElfTypeSceDynamic) { + mapHitAddress = 0x800000000; } + auto [imageRange, errc] = orbis::vmem::reserve( + process, mapHitAddress, alignedImageSize, allocationFlags); + + rx::dieIf(errc != orbis::ErrorCode{}, + "failed to map image memory {}, errno {}", imageSize, + static_cast(errc)); + + auto imageBase = reinterpret_cast(imageRange.beginAddress()); + result->entryPoint = header.e_entry ? reinterpret_cast( imageBase - baseAddress + header.e_entry) @@ -473,7 +641,7 @@ rx::Ref rx::linker::loadModule(std::span image, if (interpPhdrIndex >= 0) { result->interp = reinterpret_cast( - image.data() + phdrs[interpPhdrIndex].p_offset); + elf->image.data() + phdrs[interpPhdrIndex].p_offset); } if (sceProcParamIndex >= 0) { @@ -518,19 +686,12 @@ rx::Ref rx::linker::loadModule(std::span image, }; auto *exinfo = reinterpret_cast( - image.data() + phdrs[gnuEhFramePhdrIndex].p_offset); + elf->image.data() + phdrs[gnuEhFramePhdrIndex].p_offset); - if (exinfo->version != 1) { - std::abort(); - } - - if (exinfo->fdeCount != 0x03) { - std::abort(); - } - - if (exinfo->encodingTable != 0x3b) { - std::abort(); - } + rx::dieIf(exinfo->version != 1, "Unexpected gnu ehframe version"); + rx::dieIf(exinfo->fdeCount != 0x03, "Unexpected gnu ehframe fde count"); + rx::dieIf(exinfo->encodingTable != 0x3b, + "Unexpected gnu ehframe encoding table"); std::byte *dataBuffer = nullptr; @@ -541,7 +702,7 @@ rx::Ref rx::linker::loadModule(std::span image, auto offset = *reinterpret_cast(&exinfo->first); dataBuffer = &exinfo->first + sizeof(std::int32_t) + offset; } else { - std::abort(); + rx::die("unexpected gnu ehframe encoding {:x}", exinfo->encoding); } auto *dataBufferIt = dataBuffer; @@ -563,7 +724,7 @@ rx::Ref rx::linker::loadModule(std::span image, result->ehFrame = imageBase - baseAddress + phdrs[gnuEhFramePhdrIndex].p_vaddr + - (dataBuffer - image.data() - phdrs[gnuEhFramePhdrIndex].p_offset); + (dataBuffer - elf->image.data() - phdrs[gnuEhFramePhdrIndex].p_offset); result->ehFrameSize = dataBufferIt - dataBuffer; } @@ -575,7 +736,7 @@ rx::Ref rx::linker::loadModule(std::span image, if (dynamicPhdrIndex >= 0 && phdrs[dynamicPhdrIndex].p_filesz > 0) { auto &dynPhdr = phdrs[dynamicPhdrIndex]; std::vector dyns(dynPhdr.p_filesz / sizeof(Elf64_Dyn)); - std::memcpy(dyns.data(), image.data() + dynPhdr.p_offset, + std::memcpy(dyns.data(), elf->image.data() + dynPhdr.p_offset, dyns.size() * sizeof(Elf64_Dyn)); int sceStrtabIndex = -1; @@ -616,7 +777,7 @@ rx::Ref rx::linker::loadModule(std::span image, auto sceStrtab = sceStrtabIndex >= 0 && sceDynlibDataPhdrIndex >= 0 ? reinterpret_cast( - image.data() + + elf->image.data() + dynTabOffsetGet(dyns[sceStrtabIndex].d_un.d_val) + phdrs[sceDynlibDataPhdrIndex].p_offset) : nullptr; @@ -625,12 +786,12 @@ rx::Ref rx::linker::loadModule(std::span image, if (strtab == nullptr && strtabIndex >= 0) { strtab = reinterpret_cast( - image.data() + dynTabOffsetGet(dyns[strtabIndex].d_un.d_val)); + elf->image.data() + dynTabOffsetGet(dyns[strtabIndex].d_un.d_val)); } auto sceDynlibData = sceDynlibDataPhdrIndex >= 0 - ? image.data() + phdrs[sceDynlibDataPhdrIndex].p_offset + ? elf->image.data() + phdrs[sceDynlibDataPhdrIndex].p_offset : nullptr; auto sceSymtabData = @@ -646,15 +807,13 @@ rx::Ref rx::linker::loadModule(std::span image, : 0; if (symtab == nullptr && symtabIndex >= 0) { - if (hashIndex < 0) { - std::fprintf(stderr, "SYMTAB without HASH!\n"); - std::abort(); - } + rx::dieIf(hashIndex < 0, "elf: SYMTAB without HASH! {}", + result->moduleName); symtab = reinterpret_cast( - image.data() + dynTabOffsetGet(dyns[symtabIndex].d_un.d_val)); + elf->image.data() + dynTabOffsetGet(dyns[symtabIndex].d_un.d_val)); symtabSize = *reinterpret_cast( - image.data() + dynTabOffsetGet(dyns[hashIndex].d_un.d_val) + + elf->image.data() + dynTabOffsetGet(dyns[hashIndex].d_un.d_val) + sizeof(std::uint32_t)); } @@ -760,7 +919,7 @@ rx::Ref rx::linker::loadModule(std::span image, sceDynlibData + dyn.d_un.d_ptr); } else { pltRelocations = reinterpret_cast( - image.data() + dyn.d_un.d_ptr); + elf->image.data() + dyn.d_un.d_ptr); } break; case kElfDynamicTypeScePltRel: @@ -781,7 +940,7 @@ rx::Ref rx::linker::loadModule(std::span image, sceDynlibData + dyn.d_un.d_ptr); } else { nonPltRelocations = reinterpret_cast( - image.data() + dyn.d_un.d_ptr); + elf->image.data() + dyn.d_un.d_ptr); } break; case kElfDynamicTypeRelaSize: @@ -800,10 +959,7 @@ rx::Ref rx::linker::loadModule(std::span image, } } - if (hasPs4Dyn && hasPs5Dyn) { - std::fprintf(stderr, "unexpected import type\n"); - std::abort(); - } + rx::dieIf(hasPs4Dyn && hasPs5Dyn, "unexpected import type"); if (!hasPs4Dyn && !hasPs5Dyn && interpPhdrIndex >= 0) { result->dynType = orbis::DynType::FreeBsd; @@ -854,24 +1010,23 @@ rx::Ref rx::linker::loadModule(std::span image, library = moduleLibary.substr(0, hashPos); module = moduleLibary.substr(hashPos + 1); - auto libaryNid = *decodeNid(library); - auto moduleNid = *decodeNid(module); + auto libaryNid = *rx::linker::decodeNid(library); + auto moduleNid = *rx::linker::decodeNid(module); symbol.libraryIndex = idToLibraryIndex.at(libaryNid); symbol.moduleIndex = idToModuleIndex.at(moduleNid); - symbol.id = *decodeNid(name); + symbol.id = *rx::linker::decodeNid(name); if (name == "5JrIq4tzVIo") { monoPimpAddress = symbol.address + (std::uint64_t)imageBase; - std::fprintf(stderr, "mono_pimp address = %lx\n", - monoPimpAddress); + rx::println(stderr, "mono_pimp address = {:x}", monoPimpAddress); } - } else if (auto nid = decodeNid(fullName)) { + } else if (auto nid = rx::linker::decodeNid(fullName)) { symbol.id = *nid; symbol.libraryIndex = -1; symbol.moduleIndex = -1; } else { - symbol.id = - encodeFid(strtab + static_cast(sym.st_name)); + symbol.id = rx::linker::encodeFid( + strtab + static_cast(sym.st_name)); symbol.libraryIndex = -1; symbol.moduleIndex = -1; } @@ -882,50 +1037,126 @@ rx::Ref rx::linker::loadModule(std::span image, } } + std::string_view mapName = result->moduleName; + + if (header.e_type == rx::linker::kElfTypeExec || + header.e_type == rx::linker::kElfTypeSceExec || + header.e_type == rx::linker::kElfTypeSceDynExec) { + mapName = "executable"; + } + for (auto phdr : phdrs) { if (phdr.p_type == kElfProgramTypeLoad || phdr.p_type == kElfProgramTypeSceRelRo || phdr.p_type == kElfProgramTypeGnuRelRo) { - auto segmentEnd = rx::alignUp(phdr.p_vaddr + phdr.p_memsz, vm::kPageSize); - auto segmentBegin = - rx::alignDown(phdr.p_vaddr - baseAddress, phdr.p_align); - auto segmentSize = segmentEnd - segmentBegin; - ::mprotect(imageBase + segmentBegin, segmentSize, PROT_WRITE); - std::memcpy(imageBase + phdr.p_vaddr - baseAddress, - image.data() + phdr.p_offset, phdr.p_filesz); - - if (phdr.p_type == kElfProgramTypeSceRelRo || - phdr.p_type == kElfProgramTypeGnuRelRo) { - phdr.p_flags |= vm::kMapProtCpuWrite; // TODO: reprotect on relocations - } - - int mapFlags = 0; + rx::EnumBitSet protFlags = {}; if (phdr.p_flags & PF_X) { - mapFlags |= vm::kMapProtCpuExec; + protFlags |= orbis::vmem::Protection::CpuExec; } if (phdr.p_flags & PF_W) { - mapFlags |= vm::kMapProtCpuWrite; + protFlags |= orbis::vmem::Protection::CpuWrite; } if (phdr.p_flags & PF_R) { - mapFlags |= vm::kMapProtCpuRead; + protFlags |= orbis::vmem::Protection::CpuRead; } - if (mapFlags == 0) { - mapFlags = vm::kMapProtCpuWrite; + if (!protFlags) { + protFlags = orbis::vmem::Protection::CpuRead | + orbis::vmem::Protection::CpuWrite; } - vm::protect(imageBase + segmentBegin, segmentSize, mapFlags); + phdr.p_memsz = rx::alignUp(phdr.p_memsz, orbis::vmem::kPageSize); - if (phdr.p_type == kElfProgramTypeLoad) { - if (result->segmentCount >= std::size(result->segments)) { - std::abort(); + auto segmentEnd = + rx::alignUp(phdr.p_vaddr + phdr.p_memsz, orbis::vmem::kPageSize); + auto segmentBegin = rx::alignDown(phdr.p_vaddr, phdr.p_align); + + auto segmentRange = + rx::AddressRange::fromBeginEnd(segmentBegin, segmentEnd); + + if ((phdr.p_flags & PF_W) || phdr.p_type == kElfProgramTypeSceRelRo || + phdr.p_type == kElfProgramTypeGnuRelRo) { + // map anonymous memory, copy segment data + + auto [vmem, vmemErrc] = + orbis::vmem::mapFlex(process, segmentRange.size(), protFlags, + imageRange.beginAddress() + segmentBegin, + orbis::AllocationFlags::Fixed, {}, mapName); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "elf: failed to map flexible to virtual memory {}", + (int)vmemErrc); + + if ((phdr.p_flags & PF_W) == 0) { + rx::mem::protect(vmem, + rx::mem::Protection::R | rx::mem::Protection::W); } - auto &segment = result->segments[result->segmentCount++]; + std::memcpy(imageBase + phdr.p_vaddr, elf->image.data() + phdr.p_offset, + phdr.p_filesz); + + rx::println(stderr, "{}: RW segment {:x}-{:x}, {}", result->moduleName, + segmentRange.beginAddress(), segmentRange.endAddress(), + protFlags.raw()); + } else { + // map elf device directly + rx::dieIf(rx::alignUp(phdr.p_filesz, orbis::vmem::kPageSize) != + segmentRange.size(), + "unexpected read only segment size, {:x} vs {:x}", + phdr.p_filesz, segmentRange.size()); + + rx::println(stderr, "{}: RX segment {:x}-{:x}, {}", result->moduleName, + segmentRange.beginAddress(), segmentRange.endAddress(), + protFlags.raw()); + + { + std::lock_guard lock(elf->mtx); + if (!elf->initialized) { + auto [vmem, vmemErrc] = orbis::vmem::mapFile( + process, imageRange.beginAddress() + segmentBegin, + segmentRange.size(), orbis::AllocationFlags::Fixed, + protFlags | orbis::vmem::Protection::CpuWrite, {}, {}, elf, + segmentBegin, mapName); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "elf: failed to map elf to virtual memory {}", vmemErrc); + + std::memset(imageBase + phdr.p_vaddr + phdr.p_filesz, 0, + phdr.p_memsz - phdr.p_filesz); + std::memcpy(imageBase + phdr.p_vaddr, + elf->image.data() + phdr.p_offset, phdr.p_filesz); + elf->initialized = true; + } + } + + auto [vmem, vmemErrc] = orbis::vmem::mapFile( + process, imageRange.beginAddress() + segmentBegin, + segmentRange.size(), orbis::AllocationFlags::Fixed, protFlags, + orbis::vmem::BlockFlags::FlexibleMemory | + orbis::vmem::BlockFlags::Commited, + orbis::vmem::BlockFlagsEx::Shared, elf, segmentBegin, mapName); + + rx::dieIf(vmemErrc != orbis::ErrorCode{}, + "elf: failed to map elf to virtual memory {}", (int)vmemErrc); + } + + std::uint32_t segmentIndex; + if (phdr.p_type == kElfProgramTypeLoad) { + segmentIndex = protFlags & orbis::vmem::Protection::CpuExec ? 0 : 1; + } else { + segmentIndex = 2; + } + + auto &segment = result->segments[segmentIndex]; + if (segment.addr != nullptr) { + rx::println(stderr, "elf: corrupted, segment {} overriding. {}", + segmentIndex, name); + } else { segment.addr = imageBase + segmentBegin; segment.size = phdr.p_memsz; - segment.prot = phdr.p_flags; + segment.prot = protFlags.toUnderlying(); + result->segmentCount = std::max(segmentIndex + 1, result->segmentCount); } } } @@ -951,16 +1182,19 @@ rx::Ref rx::linker::loadModule(std::span image, result->phNum = header.e_phnum; result->proc = process; - std::printf("Loaded module '%s' (%lx) from object '%s', address: %p - %p\n", + std::strncpy(result->soName, name.data(), sizeof(result->soName)); + rx::println("Loaded module '{}' ({:x}) from object '{}', " + "address: {} - {}", result->moduleName, (unsigned long)result->attributes, - result->soName, imageBase, (char *)imageBase + result->size); + result->soName, (void *)imageBase, + (void *)((char *)imageBase + result->size)); for (const auto &mod : result->neededModules) { - std::printf(" needed module '%s' (%lx)\n", mod.name.c_str(), + rx::println(" needed module '{}' ({:x})", mod.name.c_str(), (unsigned long)mod.attr); } for (const auto &lib : result->neededLibraries) { - std::printf(" needed library '%s' (%lx), kind %s\n", lib.name.c_str(), + rx::println(" needed library '{}' ({:x}), kind {}", lib.name.c_str(), (unsigned long)lib.attr, lib.isExport ? "export" : "import"); } @@ -973,59 +1207,17 @@ rx::Ref rx::linker::loadModule(std::span image, static rx::Ref loadModuleFileImpl(std::string_view path, orbis::Thread *thread) { - rx::Ref instance; - if (vfs::open(path, orbis::kOpenFlagReadOnly, 0, &instance, thread) - .isError()) { + rx::Ref elf; + if (auto errc = g_elfDevice->open(&elf, path.data(), 0, 0, thread); + errc != orbis::ErrorCode{}) { return {}; } - orbis::Stat fileStat; - if (instance->ops->stat(instance.get(), &fileStat, nullptr) != - orbis::ErrorCode{}) { - return {}; + if (auto sepPos = path.rfind('/'); sepPos != std::string_view::npos) { + path.remove_prefix(sepPos + 1); } - auto len = fileStat.size; - - std::vector image(len); - auto ptr = image.data(); - orbis::IoVec ioVec{ - .base = ptr, - .len = static_cast(len), - }; - orbis::Uio io{ - .offset = 0, - .iov = &ioVec, - .iovcnt = 1, - .resid = 0, - .segflg = orbis::UioSeg::SysSpace, - .rw = orbis::UioRw::Read, - .td = thread, - }; - - while (io.offset < image.size()) { - ioVec = { - .base = ptr + io.offset, - .len = image.size() - io.offset, - }; - auto result = instance->ops->read(instance.get(), &io, thread); - if (result != orbis::ErrorCode{}) { - std::fprintf(stderr, "Module file reading error\n"); - std::abort(); - } - } - - if (image[0] != std::byte{'\x7f'} || image[1] != std::byte{'E'} || - image[2] != std::byte{'L'} || image[3] != std::byte{'F'}) { - image = unself(image.data(), image.size()); - - // std::ofstream( - // std::filesystem::path(path).filename().replace_extension("elf"), - // std::ios::binary) - // .write((const char *)image.data(), image.size()); - } - - return rx::linker::loadModule(image, thread->tproc); + return loadModule(elf.rawStaticCast(), thread->tproc, path); } rx::Ref rx::linker::loadModuleFile(std::string_view path, diff --git a/rpcsx/linker.hpp b/rpcsx/linker.hpp index 22d48cfe4..9f8e1b53a 100644 --- a/rpcsx/linker.hpp +++ b/rpcsx/linker.hpp @@ -75,8 +75,6 @@ enum OrbisElfType_t { void override(std::string originalModuleName, std::filesystem::path replacedModulePath); -rx::Ref loadModule(std::span image, - orbis::Process *process); rx::Ref loadModuleFile(std::string_view path, orbis::Thread *thread); } // namespace rx::linker diff --git a/rpcsx/main.cpp b/rpcsx/main.cpp index a999183ab..cbe11b8a0 100644 --- a/rpcsx/main.cpp +++ b/rpcsx/main.cpp @@ -8,16 +8,22 @@ #include "ipmi.hpp" #include "linker.hpp" #include "ops.hpp" +#include "orbis/dmem.hpp" +#include "orbis/fmem.hpp" +#include "orbis/pmem.hpp" #include "orbis/ucontext.hpp" #include "orbis/utils/Logs.hpp" +#include "orbis/vmem.hpp" #include "rx/Config.hpp" +#include "rx/die.hpp" +#include "rx/format.hpp" #include "rx/mem.hpp" #include "rx/print.hpp" #include "rx/watchdog.hpp" #include "thread.hpp" #include "vfs.hpp" -#include "vm.hpp" #include "xbyak/xbyak.h" +#include #include #include #include @@ -42,6 +48,7 @@ #include #include #include +#include #include #include @@ -65,26 +72,19 @@ handle_signal(int sig, siginfo_t *info, void *ucontext) { orbis::g_currentThread->tproc->vmId >= 0 && sig == SIGSEGV && signalAddress >= orbis::kMinAddress && signalAddress < orbis::kMaxAddress) { - auto vmid = orbis::g_currentThread->tproc->vmId; + auto process = orbis::g_currentThread->tproc; + auto vmid = process->vmId; auto ctx = reinterpret_cast(ucontext); bool isWrite = (ctx->uc_mcontext.gregs[REG_ERR] & 0x2) != 0; - auto origVmProt = vm::getPageProtection(signalAddress); - int prot = 0; + auto origVmProt = orbis::vmem::queryProtection(process, signalAddress); + auto page = signalAddress / rx::mem::pageSize; - - if (origVmProt & vm::kMapProtCpuRead) { - prot |= PROT_READ; - } - if (origVmProt & vm::kMapProtCpuWrite) { - prot |= PROT_WRITE; - } - if (origVmProt & vm::kMapProtCpuExec) { - prot |= PROT_EXEC; - } - auto gpuDevice = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}; - if (gpuDevice && (prot & (isWrite ? PROT_WRITE : PROT_READ)) != 0) { + if (gpuDevice && origVmProt && + (origVmProt->prot & (isWrite ? orbis::vmem::Protection::CpuWrite + : orbis::vmem::Protection::CpuRead))) { + auto prot = toCpuProtection(origVmProt->prot); auto &gpuContext = gpuDevice.getContext(); while (true) { auto flags = @@ -115,7 +115,7 @@ handle_signal(int sig, siginfo_t *info, void *ucontext) { } if (!isWrite) { - prot &= ~PROT_WRITE; + prot = prot & ~rx::mem::Protection::W; break; } @@ -125,19 +125,20 @@ handle_signal(int sig, siginfo_t *info, void *ucontext) { } } - if (::mprotect((void *)(page * rx::mem::pageSize), rx::mem::pageSize, - prot)) { - std::perror("cache reprotection error"); - std::abort(); - } + auto range = rx::AddressRange::fromBeginSize(page * rx::mem::pageSize, + rx::mem::pageSize); + auto errc = rx::mem::protect(range, prot); + rx::dieIf(errc != std::errc{}, + "cache: virtual memory protection failed, address {}, error {}", + range.beginAddress(), static_cast(errc)); _writefsbase_u64(orbis::g_currentThread->fsBase); return; } - std::fprintf(stderr, "SIGSEGV, address %lx, access %s, prot %s\n", + std::fprintf(stderr, "SIGSEGV, address %lx, access %s, prot %u\n", signalAddress, isWrite ? "write" : "read", - vm::mapProtToString(origVmProt).c_str()); + origVmProt ? origVmProt->prot.toUnderlying() : -1); } if (orbis::g_currentThread != nullptr) { @@ -290,14 +291,14 @@ struct StackWriter { }; static bool g_traceSyscalls = false; -static const char *getSyscallName(orbis::Thread *thread, int sysno) { +static const orbis::sysent *getSyscallEnt(orbis::Thread *thread, int sysno) { auto sysvec = thread->tproc->sysent; if (sysno >= sysvec->size) { return nullptr; } - return orbis::getSysentName(sysvec->table[sysno].call); + return sysvec->table + sysno; } static void onSysEnter(orbis::Thread *thread, int id, uint64_t *args, int argsCount) { @@ -307,12 +308,13 @@ static void onSysEnter(orbis::Thread *thread, int id, uint64_t *args, flockfile(stderr); std::fprintf(stderr, " [%u] ", thread->tid); - if (auto name = getSyscallName(thread, id)) { - std::fprintf(stderr, "%s(", name); - } else { - std::fprintf(stderr, "sys_%u(", id); + if (auto ent = getSyscallEnt(thread, id)) { + std::fprintf(stderr, "%s\n", ent->format(args).c_str()); + return; } + std::fprintf(stderr, "sys_%u(", id); + for (int i = 0; i < argsCount; ++i) { if (i != 0) { std::fprintf(stderr, ", "); @@ -332,24 +334,26 @@ static void onSysExit(orbis::Thread *thread, int id, uint64_t *args, } flockfile(stderr); - std::fprintf(stderr, "%c: [%u] ", result.isError() ? 'E' : 'S', thread->tid); + rx::print(stderr, "{}: [{}] ", result.isError() ? 'E' : 'S', thread->tid); - if (auto name = getSyscallName(thread, id)) { - std::fprintf(stderr, "%s(", name); + if (auto ent = getSyscallEnt(thread, id)) { + rx::print(stderr, "{}", ent->format(args)); } else { - std::fprintf(stderr, "sys_%u(", id); - } + rx::print(stderr, "sys_{}(", id); - for (int i = 0; i < argsCount; ++i) { - if (i != 0) { - std::fprintf(stderr, ", "); + for (int i = 0; i < argsCount; ++i) { + if (i != 0) { + rx::print(stderr, ", "); + } + + rx::print(stderr, "{:#x}", args[i]); } - std::fprintf(stderr, "%#lx", args[i]); + rx::print(stderr, ")"); } - std::fprintf(stderr, ") -> Status %d, Value %lx:%lx\n", result.value(), - thread->retval[0], thread->retval[1]); + rx::println(stderr, " -> {}, Value {:x}:{:x}", result.errc(), + thread->retval[0], thread->retval[1]); if (result.isError()) { thread->where(); @@ -357,11 +361,11 @@ static void onSysExit(orbis::Thread *thread, int id, uint64_t *args, funlockfile(stderr); } -static void guestInitDev() { - auto dmem1 = createDmemCharacterDevice(1); - orbis::g_context->dmemDevice = dmem1; +static void guestInitDev(orbis::Thread *thread) { + auto dmem0 = createDmemCharacterDevice(0); + dmem0->open(&orbis::g_context->dmem, "", 0, 0, thread); - auto dce = createDceCharacterDevice(); + auto dce = createDceCharacterDevice(thread->tproc); orbis::g_context->dceDevice = dce; auto ttyFd = ::open("tty.txt", O_CREAT | O_TRUNC | O_WRONLY, 0666); @@ -377,12 +381,12 @@ static void guestInitDev() { auto analogAudioDevice = nullAudioDevice; auto spdifAudioDevice = nullAudioDevice; - vfs::addDevice("dmem0", createDmemCharacterDevice(0)); vfs::addDevice("npdrm", createNpdrmCharacterDevice()); vfs::addDevice("icc_configuration", createIccConfigurationCharacterDevice()); vfs::addDevice("console", consoleDev); vfs::addDevice("camera", createCameraCharacterDevice()); - vfs::addDevice("dmem1", dmem1); + vfs::addDevice("dmem0", dmem0); + vfs::addDevice("dmem1", createDmemCharacterDevice(1)); vfs::addDevice("dmem2", createDmemCharacterDevice(2)); vfs::addDevice("stdout", consoleDev); vfs::addDevice("stderr", consoleDev); @@ -518,7 +522,7 @@ static void guestInitDev() { auto shm = createShmDevice(); orbis::g_context->shmDevice = shm; - orbis::g_context->blockpoolDevice = createBlockPoolDevice(); + createBlockPoolDevice()->open(&orbis::g_context->blockpool, "", 0, 0, thread); } static void guestInitFd(orbis::Thread *mainThread) { @@ -542,22 +546,29 @@ struct ExecEnv { int guestExec(orbis::Thread *mainThread, ExecEnv execEnv, rx::Ref executableModule, std::span argv, std::span envp) { - const auto stackEndAddress = 0x7'ffff'c000ull; - const auto stackSize = 0x40000 * 32; - auto stackStartAddress = stackEndAddress - stackSize; - mainThread->stackStart = - vm::map(reinterpret_cast(stackStartAddress), stackSize, - vm::kMapProtCpuWrite | vm::kMapProtCpuRead, - vm::kMapFlagAnonymous | vm::kMapFlagFixed | vm::kMapFlagPrivate | - vm::kMapFlagStack); + const auto stackEndAddress = 0x7'eeff'c000ull; + const auto stackSize = 0x200000; - mainThread->stackEnd = - reinterpret_cast(mainThread->stackStart) + stackSize; + auto stackStartAddress = stackEndAddress - stackSize; + + auto [stackVmRange, vmErrc] = orbis::vmem::mapFlex( + mainThread->tproc, stackSize, + orbis::vmem::Protection::CpuRead | orbis::vmem::Protection::CpuWrite, + stackStartAddress, + orbis::AllocationFlags::Stack | orbis::AllocationFlags::Fixed, + orbis::vmem::BlockFlags::Stack, "main stack"); + + rx::dieIf(vmErrc != orbis::ErrorCode{}, + "failed to map main thread stack, error {}", + static_cast(vmErrc)); + + mainThread->stackStart = stackVmRange.beginAddress(); + mainThread->stackEnd = stackVmRange.endAddress(); std::vector argvOffsets; std::vector envpOffsets; - StackWriter stack{reinterpret_cast(mainThread->stackEnd)}; + StackWriter stack{mainThread->stackEnd}; for (auto &elem : argv) { argvOffsets.push_back(stack.pushString(elem.data())); @@ -680,26 +691,24 @@ ExecEnv guestCreateExecEnv(orbis::Thread *mainThread, return {.entryPoint = entryPoint, .interpBase = interpBase}; } - auto libSceLibcInternal = rx::linker::loadModuleFile( - "/system/common/lib/libSceLibcInternal.sprx", mainThread); - - if (libSceLibcInternal == nullptr) { - std::println(stderr, "libSceLibcInternal not found"); - std::abort(); - } - - libSceLibcInternal->id = - mainThread->tproc->modulesMap.insert(libSceLibcInternal); - auto libkernel = rx::linker::loadModuleFile( (isSystem ? "/system/common/lib/libkernel_sys.sprx" : "/system/common/lib/libkernel.sprx"), mainThread); - if (libkernel == nullptr) { - rx::println(stderr, "libkernel not found"); - std::abort(); - } + rx::dieIf(libkernel == nullptr, "libkernel not found"); + libkernel->id = mainThread->tproc->modulesMap.insert(libkernel); + + mainThread->tproc->libkernelRange = rx::AddressRange::fromBeginSize( + std::bit_cast(libkernel->base), libkernel->size); + + auto libSceLibcInternal = rx::linker::loadModuleFile( + "/system/common/lib/libSceLibcInternal.sprx", mainThread); + + rx::dieIf(libSceLibcInternal == nullptr, "libSceLibcInternal not found"); + + libSceLibcInternal->id = + mainThread->tproc->modulesMap.insert(libSceLibcInternal); if (orbis::g_context->fwType == orbis::FwType::Ps4) { for (auto sym : libkernel->symbols) { @@ -707,7 +716,7 @@ ExecEnv guestCreateExecEnv(orbis::Thread *mainThread, auto address = (uint64_t)libkernel->base + sym.address; ::mprotect((void *)rx::alignDown(address, 0x1000), rx::alignUp(sym.size + sym.address, 0x1000), PROT_WRITE); - std::println("patching sceKernelGetMainSocId"); + rx::println("patching sceKernelGetMainSocId"); struct GetMainSocId : Xbyak::CodeGenerator { GetMainSocId(std::uint64_t address, std::uint64_t size) : Xbyak::CodeGenerator(size, (void *)address) { @@ -741,7 +750,6 @@ ExecEnv guestCreateExecEnv(orbis::Thread *mainThread, } } - libkernel->id = mainThread->tproc->modulesMap.insert(libkernel); interpBase = reinterpret_cast(libkernel->base); entryPoint = libkernel->entryPoint; @@ -757,18 +765,18 @@ int guestExec(orbis::Thread *mainThread, } static void usage(const char *argv0) { - std::println("{} [...] [args...]", argv0); - std::println(" options:"); - std::println(" --version, -v - print version"); - std::println(" -m, --mount "); - std::println(" -o, --override "); - std::println(" --fw "); - std::println( + rx::println("{} [...] [args...]", argv0); + rx::println(" options:"); + rx::println(" --version, -v - print version"); + rx::println(" -m, --mount "); + rx::println(" -o, --override "); + rx::println(" --fw "); + rx::println( " --gpu - specify physical gpu index to use, default is 0"); - std::println(" --disable-cache - disable cache of gpu resources"); - // std::println(" --presenter "); - std::println(" --trace"); + rx::println(" --disable-cache - disable cache of gpu resources"); + // rx::println(" --presenter "); + rx::println(" --trace"); } static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, @@ -792,6 +800,7 @@ static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, } auto process = orbis::createProcess(thread->tproc, childPid); + orbis::vmem::initialize(process, true); // override init process mappings auto logFd = ::open(("log-" + std::to_string(childPid) + ".txt").c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666); @@ -822,10 +831,9 @@ static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, 0xF0000000FFFF4000, }, }; - process->budgetId = 0; + process->budgetId = thread->tproc->budgetId; process->isInSandbox = false; - vm::fork(childPid); vfs::fork(); *flag = true; @@ -859,8 +867,6 @@ static orbis::SysResult launchDaemon(orbis::Thread *thread, std::string path, } } - vm::reset(); - thread->tproc->nextTlsSlot = 1; auto executableModule = rx::linker::loadModuleFile(path, thread); @@ -920,10 +926,10 @@ int main(int argc, const char *argv[]) { return 1; } - std::println("mounting '{}' to virtual '{}'", argv[argIndex + 1], - argv[argIndex + 2]); + rx::println("mounting '{}' to virtual '{}'", argv[argIndex + 1], + argv[argIndex + 2]); if (!std::filesystem::is_directory(argv[argIndex + 1])) { - std::println(stderr, "Directory '{}' not exists", argv[argIndex + 1]); + rx::println(stderr, "Directory '{}' not exists", argv[argIndex + 1]); return 1; } @@ -939,7 +945,7 @@ int main(int argc, const char *argv[]) { return 1; } - std::println("mounting firmware '{}'", argv[argIndex + 1]); + rx::println("mounting firmware '{}'", argv[argIndex + 1]); vfs::mount("/", createHostIoDevice(argv[argIndex + 1], "/")); @@ -1030,10 +1036,19 @@ int main(int argc, const char *argv[]) { orbis::constructAllGlobals(); orbis::g_context->deviceEventEmitter = orbis::knew(); + // FIXME: determine mode by reading elf file + orbis::pmem::initialize(10ull * 1024 * 1024 * 1024); + orbis::dmem::initialize(); + orbis::fmem::initialize(2ull * 1024 * 1024 * 1024); + rx::startWatchdog(); rx::createGpuDevice(); vfs::initialize(); + while (orbis::g_context->gpuDevice == nullptr) { + std::this_thread::yield(); + } + std::vector guestArgv(argv + argIndex, argv + argc); if (guestArgv.empty()) { guestArgv.emplace_back("/mini-syscore.elf"); @@ -1046,6 +1061,8 @@ int main(int argc, const char *argv[]) { // vm::printHostStats(); orbis::allocatePid(); auto initProcess = orbis::createProcess(nullptr, asRoot ? 1 : 10); + orbis::vmem::initialize(initProcess); + // pthread_setname_np(pthread_self(), "10.MAINTHREAD"); int status = 0; @@ -1080,8 +1097,7 @@ int main(int argc, const char *argv[]) { .flags = 0, .item = { - // vmem - reserved space for stack - .total = 2ul * 1024 * 1024 * 1024 - (64 * 1024 * 1024), + .total = 0x1C000000, }, }, { @@ -1134,8 +1150,6 @@ int main(int argc, const char *argv[]) { }, }; - vm::initialize(initProcess->pid); - auto bigAppBudget = orbis::g_context->createProcessTypeBudget( orbis::Budget::ProcessType::BigApp, "big app budget", bigAppBudgetInfo); @@ -1219,10 +1233,7 @@ int main(int argc, const char *argv[]) { auto executableModule = rx::linker::loadModuleFile(guestArgv[0], mainThread); - if (executableModule == nullptr) { - std::println(stderr, "Failed to open '{}'", guestArgv[0]); - std::abort(); - } + rx::dieIf(executableModule == nullptr, "Failed to open '{}'", guestArgv[0]); executableModule->id = initProcess->modulesMap.insert(executableModule); initProcess->processParam = executableModule->processParam; @@ -1249,7 +1260,7 @@ int main(int argc, const char *argv[]) { executableModule->dynType = orbis::DynType::Ps5; } - guestInitDev(); + guestInitDev(mainThread); guestInitFd(mainThread); // data transfer mode @@ -1294,6 +1305,9 @@ int main(int argc, const char *argv[]) { orbis::g_context->regMgrInt[0x9010000] = 0; // video out color effect if (!isSystem) { + // do IPMI allocations in init process + ipmi::setWorkerProcess(initProcess); + ipmi::createMiniSysCoreObjects(initProcess); ipmi::createSysAvControlObjects(initProcess); ipmi::createSysCoreObjects(initProcess); @@ -1380,7 +1394,6 @@ int main(int argc, const char *argv[]) { status = guestExec(mainThread, execEnv, std::move(executableModule), guestArgv, {}); - vm::deinitialize(); rx::thread::deinitialize(); return status; diff --git a/rpcsx/ops.cpp b/rpcsx/ops.cpp index 29b29c965..b9a1d4a7e 100644 --- a/rpcsx/ops.cpp +++ b/rpcsx/ops.cpp @@ -2,8 +2,6 @@ #include "backtrace.hpp" #include "io-device.hpp" #include "io-devices.hpp" -#include "iodev/blockpool.hpp" -#include "iodev/dmem.hpp" #include "linker.hpp" #include "orbis-config.hpp" #include "orbis/KernelContext.hpp" @@ -14,14 +12,12 @@ #include "orbis/uio.hpp" #include "orbis/umtx.hpp" #include "orbis/utils/Logs.hpp" -#include "orbis/vm.hpp" +#include "orbis/vmem.hpp" #include "rx/Rc.hpp" #include "rx/watchdog.hpp" #include "thread.hpp" #include "vfs.hpp" -#include "vm.hpp" #include -#include #include #include #include @@ -31,10 +27,15 @@ #include #include #include + +#ifdef __linux #include #include -#include #include +#include +#endif + +#include using namespace orbis; @@ -89,7 +90,7 @@ loadPrx(orbis::Thread *thread, std::string_view name, bool relocate, loadedObjects[module->soName] = module.get(); if (loadedModules.try_emplace(module->moduleName, module.get()).second) { - std::printf("Setting '%s' as '%s' module\n", module->soName, + std::printf("Registering '%s' as '%s' module\n", module->soName, module->moduleName); } @@ -142,17 +143,7 @@ loadPrx(orbis::Thread *thread, std::string_view path, bool relocate) { std::string expectedName; if (auto sep = path.rfind('/'); sep != std::string_view::npos) { - auto tmpExpectedName = path.substr(sep + 1); - - if (tmpExpectedName.ends_with(".sprx")) { - tmpExpectedName.remove_suffix(5); - } - - expectedName += tmpExpectedName; - - if (!expectedName.ends_with(".prx")) { - expectedName += ".prx"; - } + expectedName = path.substr(sep + 1); } return loadPrx(thread, path, relocate, loadedObjects, loadedModules, @@ -174,126 +165,6 @@ std::string getAbsolutePath(std::string path, Thread *thread) { return std::filesystem::path(path).lexically_normal().string(); } -orbis::SysResult mmap(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len, orbis::sint prot, orbis::sint flags, - orbis::sint fd, orbis::off_t pos) { - if (fd == -1) { - auto result = vm::map(addr, len, prot, flags); - if (result == (void *)-1) { - return ErrorCode::NOMEM; - } - - thread->retval[0] = reinterpret_cast(result); - return {}; - } - - auto file = thread->tproc->fileDescriptors.get(fd); - if (file == nullptr) { - return ErrorCode::BADF; - } - - if (file->ops->mmap == nullptr) { - ORBIS_LOG_FATAL("unimplemented mmap", fd, (void *)addr, len, prot, flags, - pos); - return mmap(thread, addr, len, prot, flags, -1, 0); - } - - void *maddr = addr; - auto result = - file->ops->mmap(file.get(), &maddr, len, prot, flags, pos, thread); - - if (result != ErrorCode{}) { - return result; - } - - thread->retval[0] = reinterpret_cast(maddr); - return {}; -} - -orbis::SysResult dmem_mmap(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len, orbis::sint memoryType, - orbis::sint prot, sint flags, - orbis::off_t directMemoryStart) { - auto dmem = static_cast(orbis::g_context->dmemDevice.get()); - void *address = addr; - auto result = dmem->mmap(&address, len, prot, flags, directMemoryStart); - if (result != ErrorCode{}) { - return result; - } - - thread->retval[0] = reinterpret_cast(address); - return {}; -} - -orbis::SysResult munmap(orbis::Thread *, orbis::ptr addr, - orbis::size_t len) { - if (vm::unmap(addr, len)) { - return {}; - } - return ErrorCode::INVAL; -} - -orbis::SysResult msync(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint flags) { - return {}; -} - -orbis::SysResult mprotect(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint prot) { - if (!vm::protect((void *)addr, len, prot)) { - return ErrorCode::INVAL; - } - return {}; -} - -orbis::SysResult minherit(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint inherit) { - return ErrorCode::INVAL; -} - -orbis::SysResult madvise(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::sint behav) { - return {}; -} - -orbis::SysResult mincore(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len, orbis::ptr vec) { - return ErrorCode::INVAL; -} - -orbis::SysResult mlock(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len) { - return {}; -} -orbis::SysResult mlockall(orbis::Thread *thread, orbis::sint how) { return {}; } -orbis::SysResult munlockall(orbis::Thread *thread) { return {}; } -orbis::SysResult munlock(orbis::Thread *thread, orbis::ptr addr, - orbis::size_t len) { - return {}; -} -orbis::SysResult virtual_query(orbis::Thread *thread, - orbis::ptr addr, orbis::sint flags, - orbis::ptr info, orbis::ulong infoSize) { - if (infoSize != sizeof(vm::VirtualQueryInfo)) { - return ErrorCode::INVAL; - } - - if (!vm::virtualQuery(addr, flags, (vm::VirtualQueryInfo *)info)) { - return ErrorCode::ACCES; - } - return {}; -} - -orbis::SysResult -query_memory_protection(orbis::Thread *thread, orbis::ptr address, - orbis::ptr protection) { - if (vm::queryProtection(address, &protection->startAddress, - &protection->endAddress, &protection->prot)) { - return {}; - } - return ErrorCode::INVAL; -} - orbis::SysResult open(orbis::Thread *thread, orbis::ptr path, orbis::sint flags, orbis::sint mode, rx::Ref *file) { @@ -324,33 +195,6 @@ orbis::SysResult rename(Thread *thread, ptr from, thread); } -orbis::SysResult blockpool_open(orbis::Thread *thread, - rx::Ref *file) { - auto dev = static_cast(orbis::g_context->blockpoolDevice.get()); - return dev->open(file, nullptr, 0, 0, thread); -} - -orbis::SysResult blockpool_map(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len, orbis::sint prot, - orbis::sint flags) { - auto blockpool = - static_cast(orbis::g_context->blockpoolDevice.get()); - void *address = addr; - auto result = blockpool->map(&address, len, prot, flags, thread); - if (result != ErrorCode{}) { - return result; - } - - thread->retval[0] = reinterpret_cast(address); - return {}; -} -orbis::SysResult blockpool_unmap(orbis::Thread *thread, orbis::caddr_t addr, - orbis::size_t len) { - auto blockpool = - static_cast(orbis::g_context->blockpoolDevice.get()); - return blockpool->unmap(addr, len, thread); -} - orbis::SysResult socket(orbis::Thread *thread, orbis::ptr name, orbis::sint domain, orbis::sint type, orbis::sint protocol, rx::Ref *file) { @@ -780,6 +624,7 @@ SysResult fork(Thread *thread, slong flags) { } auto process = orbis::createProcess(thread->tproc, childPid); + orbis::vmem::fork(process, thread->tproc); process->hostPid = ::getpid(); process->sysent = thread->tproc->sysent; process->onSysEnter = thread->tproc->onSysEnter; @@ -803,7 +648,6 @@ SysResult fork(Thread *thread, slong flags) { } } - vm::fork(childPid); vfs::fork(); *flag = true; @@ -892,7 +736,7 @@ SysResult execve(Thread *thread, ptr fname, ptr> argv, } } - vm::reset(); + orbis::vmem::initialize(thread->tproc, true); thread->tproc->nextTlsSlot = 1; for (auto [id, mod] : thread->tproc->modulesMap) { @@ -940,6 +784,7 @@ void block(Thread *thread) { if (--thread->unblocked != 0) { return; } +#ifdef __linux sigset_t set; sigemptyset(&set); sigaddset(&set, SIGUSR1); @@ -954,6 +799,7 @@ void block(Thread *thread) { std::free(thread->altStack.back()); thread->altStack.pop_back(); thread->sigReturns.pop_back(); +#endif } void unblock(Thread *thread) { @@ -982,29 +828,12 @@ void unblock(Thread *thread) { } // namespace ProcessOps rx::procOpsTable = { - .mmap = mmap, - .dmem_mmap = dmem_mmap, - .munmap = munmap, - .msync = msync, - .mprotect = mprotect, - .minherit = minherit, - .madvise = madvise, - .mincore = mincore, - .mlock = mlock, - .mlockall = mlockall, - .munlockall = munlockall, - .munlock = munlock, - .virtual_query = virtual_query, - .query_memory_protection = query_memory_protection, .open = open, .shm_open = shm_open, .unlink = unlink, .mkdir = mkdir, .rmdir = rmdir, .rename = rename, - .blockpool_open = blockpool_open, - .blockpool_map = blockpool_map, - .blockpool_unmap = blockpool_unmap, .socket = socket, .socketpair = socketPair, .shm_unlink = shm_unlink, diff --git a/rpcsx/thread.cpp b/rpcsx/thread.cpp index efea0816d..9ed38b556 100644 --- a/rpcsx/thread.cpp +++ b/rpcsx/thread.cpp @@ -276,7 +276,7 @@ void rx::thread::copyContext(orbis::Thread *thread, orbis::UContext &dst, void rx::thread::setContext(orbis::Thread *thread, const orbis::UContext &src) { auto &context = *std::bit_cast(thread->context); thread->stackStart = src.stack.sp; - thread->stackEnd = (char *)thread->stackStart + src.stack.size; + thread->stackEnd = thread->stackStart + src.stack.size; thread->setSigMask(src.sigmask); // dst.onstack = src.gregs[REG_ONSTACK]; diff --git a/rpcsx/vm.cpp b/rpcsx/vm.cpp deleted file mode 100644 index 2f6d8f28d..000000000 --- a/rpcsx/vm.cpp +++ /dev/null @@ -1,1221 +0,0 @@ -#include "vm.hpp" -#include "gpu/DeviceCtl.hpp" -#include "io-device.hpp" -#include "iodev/dmem.hpp" -#include "orbis-config.hpp" -#include "orbis/KernelContext.hpp" -#include "orbis/thread/Process.hpp" -#include "orbis/thread/Thread.hpp" -#include "orbis/utils/Logs.hpp" -#include "rx/Rc.hpp" -#include "rx/format.hpp" -#include "rx/print.hpp" -#include "rx/watchdog.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static std::mutex g_mtx; - -std::string vm::mapFlagsToString(std::int32_t flags) { - std::string result; - - if ((flags & kMapFlagShared) == kMapFlagShared) { - if (!result.empty()) { - result += " | "; - } - - result += "Shared"; - flags &= ~kMapFlagShared; - } - if ((flags & kMapFlagPrivate) == kMapFlagPrivate) { - if (!result.empty()) { - result += " | "; - } - - result += "Private"; - flags &= ~kMapFlagPrivate; - } - if ((flags & kMapFlagFixed) == kMapFlagFixed) { - if (!result.empty()) { - result += " | "; - } - - result += "Fixed"; - flags &= ~kMapFlagFixed; - } - if ((flags & kMapFlagRename) == kMapFlagRename) { - if (!result.empty()) { - result += " | "; - } - - result += "Rename"; - flags &= ~kMapFlagRename; - } - if ((flags & kMapFlagNoReserve) == kMapFlagNoReserve) { - if (!result.empty()) { - result += " | "; - } - - result += "NoReserve"; - flags &= ~kMapFlagNoReserve; - } - if ((flags & kMapFlagNoOverwrite) == kMapFlagNoOverwrite) { - if (!result.empty()) { - result += " | "; - } - - result += "NoOverwrite"; - flags &= ~kMapFlagNoOverwrite; - } - if ((flags & kMapFlagVoid) == kMapFlagVoid) { - if (!result.empty()) { - result += " | "; - } - - result += "Void"; - flags &= ~kMapFlagVoid; - } - if ((flags & kMapFlagHasSemaphore) == kMapFlagHasSemaphore) { - if (!result.empty()) { - result += " | "; - } - - result += "HasSemaphore"; - flags &= ~kMapFlagHasSemaphore; - } - if ((flags & kMapFlagStack) == kMapFlagStack) { - if (!result.empty()) { - result += " | "; - } - - result += "Stack"; - flags &= ~kMapFlagStack; - } - if ((flags & kMapFlagNoSync) == kMapFlagNoSync) { - if (!result.empty()) { - result += " | "; - } - - result += "NoSync"; - flags &= ~kMapFlagNoSync; - } - if ((flags & kMapFlagAnonymous) == kMapFlagAnonymous) { - if (!result.empty()) { - result += " | "; - } - - result += "Anonymous"; - flags &= ~kMapFlagAnonymous; - } - if ((flags & kMapFlagSystem) == kMapFlagSystem) { - if (!result.empty()) { - result += " | "; - } - - result += "System"; - flags &= ~kMapFlagSystem; - } - if ((flags & kMapFlagAllAvailable) == kMapFlagAllAvailable) { - if (!result.empty()) { - result += " | "; - } - - result += "AllAvailable"; - flags &= ~kMapFlagAllAvailable; - } - if ((flags & kMapFlagNoCore) == kMapFlagNoCore) { - if (!result.empty()) { - result += " | "; - } - - result += "NoCore"; - flags &= ~kMapFlagNoCore; - } - if ((flags & kMapFlagPrefaultRead) == kMapFlagPrefaultRead) { - if (!result.empty()) { - result += " | "; - } - - result += "PrefaultRead"; - flags &= ~kMapFlagPrefaultRead; - } - if ((flags & kMapFlagSelf) == kMapFlagSelf) { - if (!result.empty()) { - result += " | "; - } - - result += "Self"; - flags &= ~kMapFlagSelf; - } - - auto alignment = (flags & kMapFlagsAlignMask) >> kMapFlagsAlignShift; - flags &= ~kMapFlagsAlignMask; - - if (alignment != 0) { - if (!result.empty()) { - result += " | "; - } - - result += "Alignment(" + std::to_string(alignment) + ")"; - } - - if (flags != 0) { - if (!result.empty()) { - result += " | "; - } - - result += std::to_string(flags); - } - - return result; -} - -std::string vm::mapProtToString(std::int32_t prot) { - std::string result; - - if ((prot & kMapProtCpuRead) == kMapProtCpuRead) { - if (!result.empty()) { - result += " | "; - } - result += "CpuRead"; - prot &= ~kMapProtCpuRead; - } - if ((prot & kMapProtCpuWrite) == kMapProtCpuWrite) { - if (!result.empty()) { - result += " | "; - } - result += "CpuWrite"; - prot &= ~kMapProtCpuWrite; - } - if ((prot & kMapProtCpuExec) == kMapProtCpuExec) { - if (!result.empty()) { - result += " | "; - } - result += "CpuExec"; - prot &= ~kMapProtCpuExec; - } - if ((prot & kMapProtGpuRead) == kMapProtGpuRead) { - if (!result.empty()) { - result += " | "; - } - result += "GpuRead"; - prot &= ~kMapProtGpuRead; - } - if ((prot & kMapProtGpuWrite) == kMapProtGpuWrite) { - if (!result.empty()) { - result += " | "; - } - result += "GpuWrite"; - prot &= ~kMapProtGpuWrite; - } - - if (prot != 0) { - if (!result.empty()) { - result += " | "; - } - - result += std::to_string(prot); - } - - return result; -} - -static constexpr std::uint64_t kPageMask = vm::kPageSize - 1; -static constexpr std::uint64_t kBlockShift = 32; -static constexpr std::uint64_t kBlockSize = static_cast(1) - << kBlockShift; -static constexpr std::uint64_t kBlockMask = kBlockSize - 1; -static constexpr std::uint64_t kPagesInBlock = kBlockSize / vm::kPageSize; -static constexpr std::uint64_t kFirstBlock = 0x00; -static constexpr std::uint64_t kLastBlock = 0xff; -static constexpr std::uint64_t kBlockCount = kLastBlock - kFirstBlock + 1; -static constexpr std::uint64_t kGroupSize = 64; -static constexpr std::uint64_t kGroupMask = kGroupSize - 1; -static constexpr std::uint64_t kGroupsInBlock = kPagesInBlock / kGroupSize; -static constexpr std::uint64_t kMinAddress = orbis::kMinAddress; -static constexpr std::uint64_t kMaxAddress = orbis::kMaxAddress; -static constexpr std::uint64_t kMemorySize = kBlockCount * kBlockSize; - -static_assert((kLastBlock + 1) * kBlockSize == orbis::kMaxAddress); - -static int gMemoryShm = -1; - -struct Group { - std::uint64_t allocated; - std::uint64_t shared; - std::uint64_t readable; - std::uint64_t writable; - std::uint64_t executable; - std::uint64_t gpuReadable; - std::uint64_t gpuWritable; -}; - -enum { - kReadable = vm::kMapProtCpuRead, - kWritable = vm::kMapProtCpuWrite, - kExecutable = vm::kMapProtCpuExec, - kGpuReadable = vm::kMapProtGpuRead, - kGpuWritable = vm::kMapProtGpuWrite, - - kAllocated = 1 << 3, - kShared = 1 << 6, -}; - -inline constexpr std::uint64_t makePagesMask(std::uint64_t page, - std::uint64_t count) { - if (count == 64) { - return ~0ull << page; - } - - return ((1ull << count) - 1ull) << page; -} -struct Block { - Group groups[kGroupsInBlock]; - - void setFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t flags, bool noOverwrite) { - modifyFlags(firstPage, pagesCount, flags, ~static_cast(0), - noOverwrite); - } - - void addFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t flags) { - modifyFlags(firstPage, pagesCount, flags, 0); - } - - void removeFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t flags) { - modifyFlags(firstPage, pagesCount, 0, flags); - } - - unsigned getProtection(std::uint64_t page) const { - std::uint64_t groupIndex = page / kGroupSize; - auto mask = makePagesMask(page & kGroupMask, 1); - auto &group = groups[groupIndex]; - - if ((group.allocated & mask) == 0) { - return 0; - } - - unsigned result = 0; - - result |= (group.readable & mask) == mask ? kReadable : 0; - result |= (group.writable & mask) == mask ? kReadable | kWritable : 0; - - result |= (group.executable & mask) == mask ? kReadable | kExecutable : 0; - - result |= (group.gpuReadable & mask) == mask ? kGpuReadable : 0; - result |= (group.gpuWritable & mask) == mask ? kGpuWritable : 0; - result |= (group.shared & mask) == mask ? kShared : 0; - - return result; - } - - void modifyFlags(std::uint64_t firstPage, std::uint64_t pagesCount, - std::uint32_t addFlags, std::uint32_t removeFlags, - bool noOverwrite = false) { - std::uint64_t groupIndex = firstPage / kGroupSize; - - std::uint64_t addAllocatedFlags = - (addFlags & kAllocated) ? ~static_cast(0) : 0; - std::uint64_t addSharedFlags = - (addFlags & kShared) ? ~static_cast(0) : 0; - std::uint64_t addReadableFlags = - (addFlags & kReadable) ? ~static_cast(0) : 0; - std::uint64_t addWritableFlags = - (addFlags & kWritable) ? ~static_cast(0) : 0; - std::uint64_t addExecutableFlags = - (addFlags & kExecutable) ? ~static_cast(0) : 0; - std::uint64_t addGpuReadableFlags = - (addFlags & kGpuReadable) ? ~static_cast(0) : 0; - std::uint64_t addGpuWritableFlags = - (addFlags & kGpuWritable) ? ~static_cast(0) : 0; - - std::uint64_t removeAllocatedFlags = - (removeFlags & kAllocated) ? ~static_cast(0) : 0; - std::uint64_t removeSharedFlags = - (removeFlags & kShared) ? ~static_cast(0) : 0; - std::uint64_t removeReadableFlags = - (removeFlags & kReadable) ? ~static_cast(0) : 0; - std::uint64_t removeWritableFlags = - (removeFlags & kWritable) ? ~static_cast(0) : 0; - std::uint64_t removeExecutableFlags = - (removeFlags & kExecutable) ? ~static_cast(0) : 0; - std::uint64_t removeGpuReadableFlags = - (removeFlags & kGpuReadable) ? ~static_cast(0) : 0; - std::uint64_t removeGpuWritableFlags = - (removeFlags & kGpuWritable) ? ~static_cast(0) : 0; - - if ((firstPage & kGroupMask) != 0) { - auto count = kGroupSize - (firstPage & kGroupMask); - - if (count > pagesCount) { - count = pagesCount; - } - - auto mask = makePagesMask(firstPage & kGroupMask, count); - pagesCount -= count; - - auto &group = groups[groupIndex++]; - - if (noOverwrite) { - mask &= ~group.allocated; - } - - group.allocated = (group.allocated & ~(removeAllocatedFlags & mask)) | - (addAllocatedFlags & mask); - group.shared = (group.shared & ~(removeSharedFlags & mask)) | - (addSharedFlags & mask); - group.readable = (group.readable & ~(removeReadableFlags & mask)) | - (addReadableFlags & mask); - group.writable = (group.writable & ~(removeWritableFlags & mask)) | - (addWritableFlags & mask); - group.executable = (group.executable & ~(removeExecutableFlags & mask)) | - (addExecutableFlags & mask); - group.gpuReadable = - (group.gpuReadable & ~(removeGpuReadableFlags & mask)) | - (addGpuReadableFlags & mask); - group.gpuWritable = - (group.gpuWritable & ~(removeGpuWritableFlags & mask)) | - (addGpuWritableFlags & mask); - } - - if (noOverwrite) { - while (pagesCount >= kGroupSize) { - pagesCount -= kGroupSize; - - auto &group = groups[groupIndex++]; - auto mask = ~group.allocated; - - group.allocated = (group.allocated & ~(removeAllocatedFlags & mask)) | - (addAllocatedFlags & mask); - group.shared = (group.shared & ~(removeSharedFlags & mask)) | - (addSharedFlags & mask); - group.readable = (group.readable & ~(removeReadableFlags & mask)) | - (addReadableFlags & mask); - group.writable = (group.writable & ~(removeWritableFlags & mask)) | - (addWritableFlags & mask); - group.executable = - (group.executable & ~(removeExecutableFlags & mask)) | - (addExecutableFlags & mask); - group.gpuReadable = - (group.gpuReadable & ~(removeGpuReadableFlags & mask)) | - (addGpuReadableFlags & mask); - group.gpuWritable = - (group.gpuWritable & ~(removeGpuWritableFlags & mask)) | - (addGpuWritableFlags & mask); - } - } else { - while (pagesCount >= kGroupSize) { - pagesCount -= kGroupSize; - - auto &group = groups[groupIndex++]; - - group.allocated = - (group.allocated & ~removeAllocatedFlags) | addAllocatedFlags; - group.shared = (group.shared & ~removeSharedFlags) | addSharedFlags; - group.readable = - (group.readable & ~removeReadableFlags) | addReadableFlags; - group.writable = - (group.writable & ~removeWritableFlags) | addWritableFlags; - group.executable = - (group.executable & ~removeExecutableFlags) | addExecutableFlags; - group.gpuReadable = - (group.gpuReadable & ~removeGpuReadableFlags) | addGpuReadableFlags; - group.gpuWritable = - (group.gpuWritable & ~removeGpuWritableFlags) | addGpuWritableFlags; - } - } - - if (pagesCount > 0) { - auto mask = makePagesMask(0, pagesCount); - auto &group = groups[groupIndex++]; - - if (noOverwrite) { - mask &= ~group.allocated; - } - - group.allocated = (group.allocated & ~(removeAllocatedFlags & mask)) | - (addAllocatedFlags & mask); - group.shared = (group.shared & ~(removeSharedFlags & mask)) | - (addSharedFlags & mask); - group.readable = (group.readable & ~(removeReadableFlags & mask)) | - (addReadableFlags & mask); - group.writable = (group.writable & ~(removeWritableFlags & mask)) | - (addWritableFlags & mask); - group.executable = (group.executable & ~(removeExecutableFlags & mask)) | - (addExecutableFlags & mask); - group.gpuReadable = - (group.gpuReadable & ~(removeGpuReadableFlags & mask)) | - (addGpuReadableFlags & mask); - group.gpuWritable = - (group.gpuWritable & ~(removeGpuWritableFlags & mask)) | - (addGpuWritableFlags & mask); - } - } - - bool isFreePages(std::uint64_t page, std::uint64_t count) { - auto groupIndex = page / kGroupSize; - - std::uint64_t foundCount = 0; - - { - auto pageInGroup = page % kGroupSize; - auto allocatedBits = groups[groupIndex].allocated; - auto freePages = std::countr_zero(allocatedBits >> pageInGroup); - - if (freePages < count && freePages + pageInGroup < kGroupSize) { - return false; - } - - foundCount += freePages; - } - - for (++groupIndex; groupIndex < kGroupsInBlock && foundCount < count; - ++groupIndex) { - auto allocatedBits = groups[groupIndex].allocated; - auto freePages = std::countr_zero(allocatedBits); - foundCount += freePages; - - if (freePages != kGroupSize) { - break; - } - } - - return foundCount >= count; - } - - [[nodiscard]] bool isFree() const { - for (auto &group : groups) { - if (group.allocated) { - return false; - } - } - - return true; - } - - std::uint64_t findFreePages(std::uint64_t count, std::uint64_t alignment) { - std::uint64_t foundCount = 0; - std::uint64_t foundPage = 0; - - if (alignment < kGroupSize * vm::kPageSize) { - std::uint64_t groupAlignment = alignment >> vm::kPageShift; - - for (std::uint64_t groupIndex = 0; - groupIndex < kGroupsInBlock && foundCount < count; ++groupIndex) { - auto allocatedBits = groups[groupIndex].allocated; - - if (foundCount != 0) { - // we already found block with free pages at the end - if (count - foundCount >= kGroupSize) { - // we need whole group. if it not empty, we need to try next range - if (allocatedBits != 0) { - foundCount = 0; - } else { - foundCount += kGroupSize; - } - } else { - if (allocatedBits == 0) { - // whole group is clear, fast path - foundCount += kGroupSize; - break; - } else { - // add free pages from beginning of the current group - foundCount += std::countr_zero(allocatedBits); - - if (foundCount >= count) { - break; - } - - // not enough free pages, need to try next range - foundCount = 0; - } - } - } - - if (foundCount == 0) { - if (~allocatedBits == 0) { - continue; - } - - if (count < kGroupSize) { - // For small allocations try to find free room from beggining of - // group - auto tmpAllocatedBits = allocatedBits; - std::uint64_t processedPages = 0; - - while (processedPages < kGroupSize) { - auto freeCount = std::countr_zero(tmpAllocatedBits); - if (freeCount + processedPages > kGroupSize) { - freeCount = kGroupSize - processedPages; - } - - processedPages += freeCount; - if (freeCount >= 64) { - tmpAllocatedBits = 0; - } else { - tmpAllocatedBits >>= freeCount; - } - - if (freeCount >= count || - (freeCount > 0 && processedPages >= kGroupSize)) { - foundPage = - groupIndex * kGroupSize + processedPages - freeCount; - foundCount = freeCount; - break; - } - - while (auto usedCount = std::countr_one(tmpAllocatedBits)) { - auto nextProcessedPages = - rx::alignUp(processedPages + usedCount, groupAlignment); - if (nextProcessedPages - processedPages >= 64) { - tmpAllocatedBits = 0; - } else { - tmpAllocatedBits >>= nextProcessedPages - processedPages; - } - processedPages = nextProcessedPages; - } - } - } else { - // this is big allocation, count free last pages in block, continue - // searching on next iterations - auto freeCount = std::countl_zero(allocatedBits); - auto alignedPageIndex = - rx::alignUp(kGroupSize - freeCount, groupAlignment); - freeCount = - kGroupSize - alignedPageIndex; // calc aligned free pages - - foundCount = freeCount; - foundPage = groupIndex * kGroupSize + alignedPageIndex; - } - } - } - } else { - std::uint64_t blockAlignment = alignment / (kGroupSize * vm::kPageSize); - - for (std::uint64_t groupIndex = 0; - groupIndex < kGroupsInBlock && foundCount < count; ++groupIndex) { - if (foundCount == 0) { - groupIndex = rx::alignUp(groupIndex, blockAlignment); - - if (groupIndex >= kGroupsInBlock) { - break; - } - } - - auto allocatedBits = groups[groupIndex].allocated; - - if (allocatedBits == 0) { - if (foundCount == 0) { - foundPage = groupIndex * kGroupSize; - } - - foundCount += kGroupSize; - } else { - if (foundCount == 0 && count < kGroupSize) { - auto freeCount = std::countr_zero(allocatedBits); - - if (freeCount >= count) { - foundPage = groupIndex * kGroupSize; - foundCount = freeCount; - break; - } - } - - foundCount = 0; - } - } - } - - if (foundCount >= count) { - assert(((foundPage << vm::kPageShift) & (alignment - 1)) == 0); - return foundPage; - } - - return ~static_cast(0); - } -}; - -static Block gBlocks[kBlockCount]; - -struct MapInfo { - rx::Ref device; - std::uint64_t offset; - std::uint32_t flags; - char name[32]; - - bool operator==(const MapInfo &) const = default; -}; - -static rx::MemoryTableWithPayload gMapInfo; - -static void reserve(std::uint64_t startAddress, std::uint64_t endAddress) { - auto blockIndex = startAddress >> kBlockShift; - - assert(endAddress > startAddress); - assert(blockIndex == (endAddress >> kBlockShift)); - - auto firstPage = (startAddress & kBlockMask) >> vm::kPageShift; - auto pagesCount = - (endAddress - startAddress + (vm::kPageSize - 1)) >> vm::kPageShift; - - gBlocks[blockIndex - kFirstBlock].setFlags(firstPage, pagesCount, kAllocated, - false); -} - -void vm::fork(std::uint64_t pid) { - auto shmPath = rx::format("{}/memory-{}", rx::getShmPath(), pid); - gMemoryShm = - ::open(shmPath.c_str(), O_CREAT | O_RDWR | O_TRUNC, S_IRUSR | S_IWUSR); - - (void)g_mtx.try_lock(); - g_mtx.unlock(); // release mutex - - if (gMemoryShm == -1) { - rx::println(stderr, "Memory: failed to open {}", shmPath); - std::abort(); - } - - if (::ftruncate64(gMemoryShm, kMemorySize) < 0) { - std::println(stderr, "Memory: failed to allocate {}", shmPath); - std::abort(); - } - - for (auto address = kMinAddress; address < kMaxAddress; - address += kPageSize) { - auto prot = gBlocks[(address >> kBlockShift) - kFirstBlock].getProtection( - (address & kBlockMask) >> vm::kPageShift); - - if (prot & kShared) { - continue; - } - - if (prot & kMapProtCpuAll) { - auto mapping = ::mmap(nullptr, kPageSize, PROT_WRITE, MAP_SHARED, - gMemoryShm, address - kMinAddress); - assert(mapping != MAP_FAILED); - - rx::mem::protect(rx::AddressRange::fromBeginSize(address, kPageSize), - rx::mem::Protection::R); - std::memcpy(mapping, reinterpret_cast(address), kPageSize); - ::munmap(mapping, kPageSize); - ::munmap(reinterpret_cast(address), kPageSize); - - mapping = ::mmap(reinterpret_cast(address), kPageSize, - prot & kMapProtCpuAll, MAP_FIXED | MAP_SHARED, - gMemoryShm, address - kMinAddress); - assert(mapping != MAP_FAILED); - } - - // TODO: copy gpu memory? - } -} - -void vm::reset() { - std::memset(gBlocks, 0, sizeof(gBlocks)); - - ::munmap(reinterpret_cast(kMinAddress), kMaxAddress - kMinAddress); - if (::ftruncate64(gMemoryShm, 0) < 0) { - std::abort(); - } - if (::ftruncate64(gMemoryShm, kMemorySize) < 0) { - std::abort(); - } - - reserve(0, kMinAddress); - rx::mem::reserve( - rx::AddressRange::fromBeginSize(kMinAddress, kMaxAddress - kMinAddress)); -} - -void vm::initialize(std::uint64_t pid) { - std::println("Memory: initialization"); - auto shmPath = rx::format("{}/memory-{}", rx::getShmPath(), pid); - - gMemoryShm = - ::open(shmPath.c_str(), O_CREAT | O_RDWR | O_TRUNC, S_IRUSR | S_IWUSR); - - if (gMemoryShm == -1) { - std::println(stderr, "Memory: failed to open {}", shmPath); - std::abort(); - } - - if (::ftruncate64(gMemoryShm, kMemorySize) < 0) { - std::println(stderr, "Memory: failed to allocate {}", shmPath); - std::abort(); - } - - reserve(0, kMinAddress); // unmapped area - - rx::mem::reserve(rx::AddressRange::fromBeginEnd(kMinAddress, kMaxAddress)); -} - -void vm::deinitialize() { - std::println("Memory: shutdown"); - ::close(gMemoryShm); - gMemoryShm = -1; - - for (auto &block : gBlocks) { - block = {}; - } -} - -constexpr auto kPhysicalMemorySize = 5568ull * 1024 * 1024; -constexpr auto kFlexibleMemorySize = 448ull * 1024 * 1024; -constexpr auto kMainDirectMemorySize = - kPhysicalMemorySize - kFlexibleMemorySize; - -void *vm::map(void *addr, std::uint64_t len, std::int32_t prot, - std::int32_t flags, std::int32_t internalFlags, - orbis::IoDevice *device, std::uint64_t offset) { - std::println(stderr, "vm::map(addr = {}, len = {}, prot = {}, flags = {})", - addr, len, mapProtToString(prot), mapFlagsToString(flags)); - - len = rx::alignUp(len, kPageSize); - auto pagesCount = (len + (kPageSize - 1)) >> kPageShift; - auto hitAddress = reinterpret_cast(addr); - - std::uint64_t alignment = (flags & kMapFlagsAlignMask) >> kMapFlagsAlignShift; - if (alignment == 0) { - alignment = kPageSize; - } else { - alignment = static_cast(1) << alignment; - } - - if (alignment < kPageSize) { - std::println(stderr, "Memory error: wrong alignment {}", alignment); - alignment = kPageSize; - } - - flags &= ~kMapFlagsAlignMask; - - bool noOverwrite = (flags & (kMapFlagNoOverwrite | kMapFlagFixed)) == - (kMapFlagNoOverwrite | kMapFlagFixed); - - if (hitAddress & (alignment - 1)) { - if (flags & kMapFlagStack) { - hitAddress = rx::alignDown(hitAddress - 1, alignment); - flags |= kMapFlagFixed; - flags &= ~kMapFlagStack; - } else { - hitAddress = rx::alignUp(hitAddress, alignment); - } - } - - std::lock_guard lock(g_mtx); - - std::uint64_t address = 0; - if ((flags & kMapFlagFixed) == kMapFlagFixed) { - address = hitAddress; - - auto blockIndex = address >> kBlockShift; - - if (blockIndex < kFirstBlock || blockIndex > kLastBlock) { - std::println(stderr, - "Memory error: fixed mapping with wrong address {:x} pages", - address); - return MAP_FAILED; - } - } else if (hitAddress != 0) { - auto blockIndex = hitAddress >> kBlockShift; - auto page = (hitAddress & kBlockMask) >> kPageShift; - - if (blockIndex < kFirstBlock || blockIndex > kLastBlock) { - std::println(stderr, "Memory error: wrong hit address {:x} pages", - hitAddress); - hitAddress = 0; - } else { - blockIndex -= kFirstBlock; - - if (gBlocks[blockIndex].isFreePages(page, pagesCount)) { - address = hitAddress; - } - } - } - - static constexpr auto kBadAddress = ~static_cast(0); - - std::size_t mapBlockCount = (len + kBlockSize - 1) / kBlockSize; - - auto isFreeBlockRange = [&](std::uint64_t firstBlock, - std::size_t blockCount) { - for (auto blockIndex = firstBlock; blockIndex < firstBlock + blockCount; - ++blockIndex) { - if (!gBlocks[blockIndex].isFree()) { - return false; - } - } - - return true; - }; - - if (address == 0 && hitAddress != 0) { - auto hitBlockIndex = hitAddress >> kBlockShift; - if (mapBlockCount > 1) { - for (auto blockIndex = hitBlockIndex; - blockIndex <= kLastBlock - mapBlockCount + 1; ++blockIndex) { - if (isFreeBlockRange(blockIndex, mapBlockCount)) { - address = blockIndex * kBlockSize; - break; - } - } - } else { - for (auto blockIndex = hitBlockIndex; blockIndex <= kLastBlock; - ++blockIndex) { - auto pageAddress = gBlocks[blockIndex - kFirstBlock].findFreePages( - pagesCount, alignment); - - if (pageAddress != kBadAddress) { - address = (pageAddress << kPageShift) | (blockIndex * kBlockSize); - break; - } - } - } - } - - if (address == 0) { - if (mapBlockCount > 1) { - for (auto blockIndex = kFirstBlock; - blockIndex <= kLastBlock - mapBlockCount + 1; ++blockIndex) { - if (isFreeBlockRange(blockIndex, mapBlockCount)) { - address = blockIndex * kBlockSize; - break; - } - } - } else { - for (auto blockIndex = kFirstBlock; blockIndex <= kLastBlock; - ++blockIndex) { - auto pageAddress = gBlocks[blockIndex - kFirstBlock].findFreePages( - pagesCount, alignment); - - if (pageAddress != kBadAddress) { - address = (pageAddress << kPageShift) | (blockIndex * kBlockSize); - break; - } - } - } - } - - if (address == 0) { - std::println(stderr, - "Memory error: no free memory left for mapping of {} pages", - pagesCount); - return MAP_FAILED; - } - - if (address & (alignment - 1)) { - std::println(stderr, "Memory error: failed to map aligned address"); - std::abort(); - } - - if (address >= kMaxAddress || address > kMaxAddress - len) { - std::println(stderr, "Memory error: out of memory"); - std::abort(); - } - - int realFlags = MAP_FIXED | MAP_SHARED; - bool isAnon = (flags & kMapFlagAnonymous) == kMapFlagAnonymous; - bool isShared = (flags & kMapFlagShared) == kMapFlagShared; - flags &= ~(kMapFlagFixed | kMapFlagAnonymous | kMapFlagShared); - - auto &block = gBlocks[(address >> kBlockShift) - kFirstBlock]; - if (noOverwrite) { - auto firstPage = (address & kBlockMask) >> kPageShift; - for (std::size_t i = 0; i < pagesCount; ++i) { - if (block.getProtection(firstPage + i)) { - return (void *)-1; - } - } - } - - block.setFlags((address & kBlockMask) >> kPageShift, pagesCount, - (prot & (kMapProtCpuAll | kMapProtGpuAll)) | kAllocated | - (isShared ? kShared : 0), - false); - - /* - if (flags & kMapFlagStack) { - realFlags |= MAP_GROWSDOWN | MAP_STACK | MAP_ANONYMOUS | MAP_PRIVATE; - offset = 0; - fd = -1; - flags &= ~kMapFlagStack; - } else { - realFlags |= MAP_SHARED; - } - */ - if (flags) { - std::println(stderr, " unhandled flags 0x{:x}", flags); - } - - { - MapInfo info; - if (auto it = gMapInfo.queryArea(address); it != gMapInfo.end()) { - info = it.get(); - } - info.device = device; - info.flags = flags; - info.offset = offset; - - gMapInfo.map(rx::AddressRange::fromBeginSize(address, len), info); - } - - if (auto thr = orbis::g_currentThread) { - std::lock_guard lock(orbis::g_context->gpuDeviceMtx); - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitMapMemory(thr->tproc->pid, address, len, -1, -1, prot, - address - kMinAddress); - } - } - - if (internalFlags & kMapInternalReserveOnly) { - return reinterpret_cast(address); - } - - auto result = - ::mmap(reinterpret_cast(address), len, prot & kMapProtCpuAll, - realFlags, gMemoryShm, address - kMinAddress); - - if (result != MAP_FAILED && isAnon) { - bool needReprotect = (prot & PROT_WRITE) == 0; - if (needReprotect) { - ::mprotect(result, len, PROT_WRITE); - } - std::memset(result, 0, len); - if (needReprotect) { - ::mprotect(result, len, prot & kMapProtCpuAll); - } - } - - return result; -} - -bool vm::unmap(void *addr, std::uint64_t size) { - size = rx::alignUp(size, kPageSize); - auto pages = (size + (kPageSize - 1)) >> kPageShift; - auto address = reinterpret_cast(addr); - - if (address < kMinAddress || address >= kMaxAddress || size >= kMaxAddress || - address > kMaxAddress - size) { - std::println(stderr, "Memory error: unmap out of memory"); - return false; - } - - if ((address & kPageMask) != 0) { - std::println(stderr, "Memory error: unmap unaligned address"); - return false; - } - - if ((address >> kBlockShift) != ((address + size - 1) >> kBlockShift)) { - std::println( - stderr, - "Memory error: unmap cross block range. address 0x{:x}, size=0x{:x}", - address, size); - __builtin_trap(); - } - - std::lock_guard lock(g_mtx); - gBlocks[(address >> kBlockShift) - kFirstBlock].removeFlags( - (address & kBlockMask) >> kPageShift, pages, ~0); - if (auto thr = orbis::g_currentThread) { - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitUnmapMemory(thr->tproc->pid, address, size); - } - } else { - std::println(stderr, "ignoring unmapping {:x}-{:x}", address, - address + size); - } - gMapInfo.unmap(rx::AddressRange::fromBeginSize(address, size)); - return ::munmap(addr, size); -} - -bool vm::protect(void *addr, std::uint64_t size, std::int32_t prot) { - std::println("vm::protect(addr = {}, len = {}, prot = {})", addr, size, - mapProtToString(prot)); - - auto address = reinterpret_cast(addr); - auto endAddress = address + size; - address = rx::alignDown(address, kPageSize); - endAddress = rx::alignUp(endAddress, kPageSize); - size = endAddress - address; - auto pages = size >> kPageShift; - if (address < kMinAddress || address >= kMaxAddress || size >= kMaxAddress || - address > kMaxAddress - size) { - std::println(stderr, "Memory error: protect out of memory"); - return false; - } - - if ((address & kPageMask) != 0) { - std::println(stderr, "Memory error: protect unaligned address"); - return false; - } - - if ((address >> kBlockShift) != ((address + size - 1) >> kBlockShift)) { - std::println(stderr, "Memory error: protect cross block range"); - std::abort(); - } - - std::lock_guard lock(g_mtx); - - gBlocks[(address >> kBlockShift) - kFirstBlock].setFlags( - (address & kBlockMask) >> kPageShift, pages, - kAllocated | (prot & (kMapProtCpuAll | kMapProtGpuAll)), false); - - if (auto thr = orbis::g_currentThread) { - std::println("memory prot: {:x}", prot); - if (auto gpu = amdgpu::DeviceCtl{orbis::g_context->gpuDevice}) { - gpu.submitProtectMemory(thr->tproc->pid, address, size, prot); - } - } else if (prot >> 4) { - std::println(stderr, "ignoring mapping {:x}-{:x}", address, address + size); - } - return ::mprotect(std::bit_cast(address), size, - prot & kMapProtCpuAll) == 0; -} - -static std::int32_t getPageProtectionImpl(std::uint64_t address) { - return gBlocks[(address >> kBlockShift) - kFirstBlock].getProtection( - (address & kBlockMask) >> vm::kPageShift) & - ~kShared; -} - -bool vm::queryProtection(const void *addr, std::uint64_t *startAddress, - std::uint64_t *endAddress, std::int32_t *prot) { - auto address = reinterpret_cast(addr); - if (address < kMinAddress || address >= kMaxAddress) { - return false; - } - - std::uint64_t start = address & ~kPageMask; - std::uint64_t end = start + kPageSize; - - std::lock_guard lock(g_mtx); - - auto resultProt = getPageProtectionImpl(address); - - while (true) { - auto testAddr = start - kPageSize; - if (testAddr < kMinAddress) { - break; - } - - auto testProt = getPageProtectionImpl(testAddr); - - if (resultProt != testProt) { - break; - } - - start = testAddr; - } - - while (true) { - auto testAddr = end; - if (testAddr >= kMaxAddress) { - break; - } - - auto testProt = getPageProtectionImpl(testAddr); - - if (resultProt != testProt) { - break; - } - - end = testAddr + kPageSize; - } - - *startAddress = start; - *endAddress = end; - *prot = resultProt; - - return true; -} - -unsigned vm::getPageProtection(std::uint64_t address) { - if (address < kMinAddress || address >= kMaxAddress) { - std::println(stderr, "Memory error: getPageProtection out of memory"); - return 0; - } - - std::lock_guard lock(g_mtx); - - return getPageProtectionImpl(address); -} - -bool vm::virtualQuery(const void *addr, std::int32_t flags, - VirtualQueryInfo *info) { - std::lock_guard lock(g_mtx); - - auto address = reinterpret_cast(addr); - if (address < kMinAddress || address >= kMaxAddress) { - return false; - } - - auto it = gMapInfo.lowerBound(address); - - if (it == gMapInfo.end()) { - return false; - } - - if ((flags & 1) == 0) { - if (it.endAddress() <= address) { - return false; - } - } else { - if (it.beginAddress() > address || it.endAddress() <= address) { - return false; - } - } - - std::int32_t memoryType = 0; - std::uint32_t blockFlags = 0; - if (it->device != nullptr) { - if (auto dmem = dynamic_cast(it->device.get())) { - auto dmemIt = dmem->allocations.queryArea(it->offset); - if (dmemIt == dmem->allocations.end()) { - return false; - } - memoryType = dmemIt->memoryType; - blockFlags = kBlockFlagDirectMemory; - rx::print(stderr, "virtual query {}", addr); - rx::println(stderr, "memory type: {}", memoryType); - } - // TODO - } - - std::int32_t prot = getPageProtectionImpl(it.beginAddress()); - - *info = { - .start = it.beginAddress(), - .end = it.endAddress(), - .protection = prot, - .memoryType = memoryType, - .flags = blockFlags, - }; - - ORBIS_LOG_ERROR("virtualQuery", addr, flags, info->start, info->end, - info->protection, info->memoryType, info->flags); - - std::memcpy(info->name, it->name, sizeof(info->name)); - return true; -} - -void vm::setName(std::uint64_t start, std::uint64_t size, const char *name) { - std::lock_guard lock(g_mtx); - - MapInfo info; - if (auto it = gMapInfo.queryArea(start); it != gMapInfo.end()) { - info = it.get(); - } - - std::strncpy(info.name, name, sizeof(info.name)); - - gMapInfo.map(rx::AddressRange::fromBeginSize(start, size), info); -} diff --git a/rpcsx/vm.hpp b/rpcsx/vm.hpp deleted file mode 100644 index c2e93cf34..000000000 --- a/rpcsx/vm.hpp +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once -#include "orbis/IoDevice.hpp" -#include -#include - -namespace vm { -static constexpr std::uint64_t kPageShift = 14; -static constexpr std::uint64_t kPageSize = static_cast(1) - << kPageShift; - -enum BlockFlags { - kBlockFlagFlexibleMemory = 1 << 0, - kBlockFlagDirectMemory = 1 << 1, - kBlockFlagStack = 1 << 2, - kBlockFlagPooledMemory = 1 << 3, - kBlockFlagCommited = 1 << 4, -}; - -enum MapFlags { - kMapFlagShared = 0x1, - kMapFlagPrivate = 0x2, - kMapFlagFixed = 0x10, - kMapFlagRename = 0x20, - kMapFlagNoReserve = 0x40, - kMapFlagNoOverwrite = 0x80, - kMapFlagVoid = 0x100, - kMapFlagHasSemaphore = 0x200, - kMapFlagStack = 0x400, - kMapFlagNoSync = 0x800, - kMapFlagAnonymous = 0x1000, - kMapFlagSystem = 0x2000, - kMapFlagAllAvailable = 0x4000, - kMapFlagNoCore = 0x20000, - kMapFlagPrefaultRead = 0x40000, - kMapFlagSelf = 0x80000, -}; - -enum MapProt { - kMapProtCpuRead = 1, - kMapProtCpuWrite = 2, - kMapProtCpuReadWrite = kMapProtCpuRead | kMapProtCpuWrite, - kMapProtCpuExec = 4, - kMapProtCpuAll = 0x7, - kMapProtGpuRead = 0x10, - kMapProtGpuWrite = 0x20, - kMapProtGpuAll = 0x30, -}; - -enum MapInternalFlags { - kMapInternalReserveOnly = 1 << 0, -}; - -struct VirtualQueryInfo { - std::uint64_t start; - std::uint64_t end; - std::uint64_t offset; - std::int32_t protection; - std::int32_t memoryType; - std::uint32_t flags; - char name[32]; -}; - -static constexpr std::uint32_t kMapFlagsAlignShift = 24; -static constexpr std::uint32_t kMapFlagsAlignMask = 0x1f << kMapFlagsAlignShift; - -std::string mapFlagsToString(std::int32_t flags); -std::string mapProtToString(std::int32_t prot); - -void fork(std::uint64_t pid); -void reset(); -void initialize(std::uint64_t pid); -void deinitialize(); -void *map(void *addr, std::uint64_t len, std::int32_t prot, std::int32_t flags, - std::int32_t internalFlags = 0, orbis::IoDevice *device = nullptr, - std::uint64_t offset = 0); -bool unmap(void *addr, std::uint64_t size); -bool protect(void *addr, std::uint64_t size, std::int32_t prot); - -void setName(std::uint64_t start, std::uint64_t size, const char *name); -bool virtualQuery(const void *addr, std::int32_t flags, VirtualQueryInfo *info); -bool queryProtection(const void *addr, std::uint64_t *startAddress, - std::uint64_t *endAddress, std::int32_t *prot); -unsigned getPageProtection(std::uint64_t address); -} // namespace vm diff --git a/rx/include/rx/EnumBitSet.hpp b/rx/include/rx/EnumBitSet.hpp index 6a42eebb2..62bcbe7c5 100644 --- a/rx/include/rx/EnumBitSet.hpp +++ b/rx/include/rx/EnumBitSet.hpp @@ -20,7 +20,9 @@ Examples: Intersection (&) and symmetric difference (^) is also available. */ -#include "types.hpp" +#include "format-base.hpp" +#include "rx/refl.hpp" +#include namespace rx { template @@ -52,8 +54,8 @@ private: : m_data(data) {} public: - static constexpr usz bitmax = sizeof(T) * 8; - static constexpr usz bitsize = + static constexpr std::size_t bitmax = sizeof(T) * 8; + static constexpr std::size_t bitsize = static_cast(T::bitset_last) + 1; static_assert(std::is_enum_v, @@ -69,7 +71,7 @@ public: << static_cast(value); } - EnumBitSet() = default; + constexpr EnumBitSet() = default; // Construct from a single bit constexpr EnumBitSet(T bit) noexcept : m_data(shift(bit)) {} @@ -91,7 +93,9 @@ public: return m_data; } - constexpr detail::InvertedEnumBitSet operator~() const { return {m_data}; } + constexpr detail::InvertedEnumBitSet operator~() const { + return static_cast(~m_data); + } [[deprecated("Use operator|=")]] constexpr EnumBitSet & operator+=(EnumBitSet rhs) { @@ -126,6 +130,11 @@ public: return *this; } + constexpr EnumBitSet &operator&=(detail::InvertedEnumBitSet rhs) { + m_data &= rhs.m_data; + return *this; + } + constexpr EnumBitSet &operator^=(EnumBitSet rhs) { m_data ^= static_cast(rhs); return *this; @@ -191,7 +200,9 @@ public: return (m_data & arg.m_data) == 0; } - underlying_type &raw() { return m_data; } + constexpr underlying_type &raw() { return m_data; } + + constexpr auto operator<=>(const EnumBitSet &) const = default; }; template constexpr EnumBitSet toBitSet(T bit) { @@ -288,4 +299,50 @@ constexpr EnumBitSet operator^(const U &lhs, T rhs) { } // namespace bitset } // namespace rx +template + requires requires(rx::format_parse_context &ctx) { + rx::formatter().parse(ctx); + } +struct rx::formatter> { + constexpr rx::format_parse_context::iterator + parse(rx::format_parse_context &ctx) { + return ctx.begin(); + } + + rx::format_context::iterator format(rx::EnumBitSet bitSet, + rx::format_context &ctx) const { + auto raw = bitSet.toUnderlying(); + if (raw != 0) { + bool first = true; + for (std::size_t i = 0; i <= static_cast(T::bitset_last); + ++i) { + auto mask = 1ull << i; + if (!(raw & mask)) { + continue; + } + + if (first) { + first = false; + } else { + rx::format_to(ctx.out(), " | "); + } + + rx::format_to(ctx.out(), "{}", T(i)); + raw &= ~mask; + } + + if (raw) { + if (!first) { + rx::format_to(ctx.out(), " | "); + } + + rx::format_to(ctx.out(), "{:#x}", raw); + } + } else { + rx::format_to(ctx.out(), "{}::None", rx::getNameOf()); + } + return ctx.out(); + } +}; + using namespace rx::bitset; diff --git a/rx/include/rx/FunctionRef.hpp b/rx/include/rx/FunctionRef.hpp index 630eb1cb1..2f030f243 100644 --- a/rx/include/rx/FunctionRef.hpp +++ b/rx/include/rx/FunctionRef.hpp @@ -20,7 +20,7 @@ public: : context( const_cast> *>(&object)), invoke(+[](void *context, ArgsT... args) -> RT { - return (*reinterpret_cast(context))(std::move(args)...); + return (*reinterpret_cast(context))(args...); }) {} template diff --git a/rx/include/rx/Mappable.hpp b/rx/include/rx/Mappable.hpp index 08d02b17d..61761a723 100644 --- a/rx/include/rx/Mappable.hpp +++ b/rx/include/rx/Mappable.hpp @@ -8,6 +8,7 @@ namespace rx { class Mappable { +public: #ifdef _WIN32 using NativeHandle = void *; static constexpr NativeHandle kInvalidHandle = nullptr; @@ -16,9 +17,6 @@ class Mappable { static constexpr auto kInvalidHandle = NativeHandle(-1); #endif - NativeHandle m_handle = kInvalidHandle; - -public: Mappable() = default; Mappable(Mappable &&other) noexcept { *this = std::move(other); } Mappable(const Mappable &) = delete; @@ -33,6 +31,12 @@ public: } } + static Mappable CreateFromNativeHandle(NativeHandle handle) { + Mappable result; + result.m_handle = handle; + return result; + } + static std::pair CreateMemory(std::size_t size); static std::pair CreateSwap(std::size_t size); std::errc map(rx::AddressRange virtualRange, std::size_t offset, @@ -47,5 +51,7 @@ public: private: void destroy(); + + NativeHandle m_handle = kInvalidHandle; }; } // namespace rx diff --git a/rx/include/rx/MemoryTable.hpp b/rx/include/rx/MemoryTable.hpp index 7fefbf137..7b8a2fcb2 100644 --- a/rx/include/rx/MemoryTable.hpp +++ b/rx/include/rx/MemoryTable.hpp @@ -1,7 +1,8 @@ #pragma once -#include "rx/AddressRange.hpp" -#include "rx/Rc.hpp" +#include "AddressRange.hpp" +#include "Rc.hpp" +#include "Serializer.hpp" #include #include #include @@ -84,7 +85,7 @@ public: return {startAddress, endAddress}; } - void map(rx::AddressRange range) { + void map(AddressRange range) { auto [beginIt, beginInserted] = mAreas.emplace(range.beginAddress(), Kind::O); auto [endIt, endInserted] = mAreas.emplace(range.endAddress(), Kind::X); @@ -311,6 +312,23 @@ public: assert(kind != Kind::X); kind = Kind::O; } + + void serialize(Serializer &s) const + requires Serializable + { + s.serialize(kind); + if (kind != Kind::X) { + s.serialize(storage.data); + } + } + void deserialize(Deserializer &d) + requires Serializable + { + d.deserialize(kind); + if (kind != Kind::X && !d.failure()) { + d.deserialize(storage.data); + } + } }; template class Payload { @@ -365,6 +383,9 @@ public: assert(!isClose()); value &= ~kCloseOpenBit; } + + void serialize(Serializer &s) const { s.serialize(value); } + void deserialize(Deserializer &d) { d.deserialize(value); } }; template class Payload> { @@ -452,6 +473,9 @@ public: assert(!isClose()); value &= ~kCloseOpenBit; } + + void serialize(Serializer &s) const { s.serialize(value); } + void deserialize(Deserializer &d) { d.deserialize(value); } }; template &payload; + template class AreaInfo : public AddressRange { + T &payload; public: - AreaInfo(Payload &payload, rx::AddressRange range) + AreaInfo(T &payload, AddressRange range) : payload(payload), AddressRange(range) {} decltype(auto) operator->() { return &payload.get(); } decltype(auto) get() { return payload.get(); } }; - class iterator { - using map_iterator = - typename std::map::iterator; - map_iterator it; + template class Iterator { + MapIterator it; public: - iterator() = default; - iterator(map_iterator it) : it(it) {} + Iterator() = default; + Iterator(MapIterator it) : it(it) {} AreaInfo operator*() const { return {it->second, range()}; } - rx::AddressRange range() const { - return rx::AddressRange::fromBeginEnd(beginAddress(), endAddress()); + AddressRange range() const { + return AddressRange::fromBeginEnd(beginAddress(), endAddress()); } std::uint64_t beginAddress() const { return it->first; } @@ -495,7 +517,8 @@ public: decltype(auto) get() const { return it->second.get(); } decltype(auto) operator->() const { return &it->second.get(); } - iterator &operator++() { + + Iterator &operator++() { ++it; if (!it->second.isCloseOpen()) { @@ -505,7 +528,7 @@ public: return *this; } - iterator &operator--() { + Iterator &operator--() { --it; if (it->second.isClose()) { @@ -515,18 +538,28 @@ public: return *this; } - bool operator==(iterator other) const { return it == other.it; } - bool operator!=(iterator other) const { return it != other.it; } + bool operator==(Iterator other) const { return it == other.it; } friend MemoryTableWithPayload; }; + using iterator = + Iterator::iterator, + AreaInfo>; + using const_iterator = + Iterator::const_iterator, + AreaInfo>; + MemoryTableWithPayload() = default; MemoryTableWithPayload(MemoryTableWithPayload &&) = default; MemoryTableWithPayload &operator=(MemoryTableWithPayload &&) = default; MemoryTableWithPayload(const MemoryTableWithPayload &) = delete; MemoryTableWithPayload &operator=(const MemoryTableWithPayload &) = delete; + const_iterator cbegin() const { return const_iterator(mAreas.begin()); } + const_iterator cend() const { return const_iterator(mAreas.end()); } + const_iterator begin() const { return const_iterator(mAreas.cbegin()); } + const_iterator end() const { return const_iterator(mAreas.cend()); } iterator begin() { return iterator(mAreas.begin()); } iterator end() { return iterator(mAreas.end()); } @@ -579,9 +612,9 @@ public: return endAddress < address ? mAreas.end() : it; } - iterator map(rx::AddressRange range, PayloadT payload, bool merge = true, + iterator map(AddressRange range, PayloadT payload, bool merge = true, bool noOverride = false) { - assert(range.beginAddress() < range.endAddress()); + assert(range.isValid()); auto [beginIt, beginInserted] = mAreas.emplace(range.beginAddress(), payload_type::createOpen(payload)); auto [endIt, endInserted] = @@ -676,7 +709,7 @@ public: return origBegin; } - void unmap(iterator it) { + iterator unmap(iterator it) { auto openIt = it.it; auto closeIt = openIt; ++closeIt; @@ -690,13 +723,49 @@ public: if (closeIt->second.isCloseOpen()) { closeIt->second.setOpen(); } else { - mAreas.erase(closeIt); + closeIt = mAreas.erase(closeIt); } + + return iterator(closeIt); } - void unmap(rx::AddressRange range) { + iterator unmap(AddressRange range) { // FIXME: can be optimized - unmap(map(range, PayloadT{}, false)); + return unmap(map(range, PayloadT{}, false)); + } + + void serialize(Serializer &s) const + requires Serializable + { + for (auto block : *this) { + s.serialize(block.beginAddress()); + s.serialize(block.endAddress()); + s.serialize(block.get()); + } + + s.serialize(-1); + s.serialize(-1); + } + + void deserialize(Deserializer &d) + requires Serializable + { + clear(); + + while (!d.failure()) { + auto beginAddress = d.deserialize(); + auto endAddress = d.deserialize(); + + if (beginAddress == static_cast(-1) && + endAddress == static_cast(-1)) { + break; + } + + auto value = d.deserialize(); + + map(AddressRange::fromBeginEnd(beginAddress, endAddress), + std::move(value), false); + } } }; } // namespace rx diff --git a/rx/include/rx/Rc.hpp b/rx/include/rx/Rc.hpp index 3c791efd9..a5533469b 100644 --- a/rx/include/rx/Rc.hpp +++ b/rx/include/rx/Rc.hpp @@ -12,7 +12,7 @@ struct RcBase { virtual ~RcBase() = default; void incRef() { - if (references.fetch_add(1, std::memory_order::relaxed) > 4096) { + if (references.fetch_add(1, std::memory_order::relaxed) > 100 * 1024 * 1024) { assert(!"too many references"); } } diff --git a/rx/include/rx/Serializer.hpp b/rx/include/rx/Serializer.hpp index b9742162c..09e419679 100644 --- a/rx/include/rx/Serializer.hpp +++ b/rx/include/rx/Serializer.hpp @@ -169,19 +169,17 @@ struct Deserializer { if constexpr (requires { { type::deserialize(*this) } -> std::convertible_to; }) { - return T::deserialize(*this); + return type::deserialize(*this); } else if constexpr (requires(type &result) { type::deserialize(*this, result); }) { type result; - T::deserialize(*this, result); + type::deserialize(*this, result); return result; } else if constexpr (requires(type &result) { - { - result.deserialize(*this) - } -> std::convertible_to; + result.deserialize(*this); }) { - T result; + type result; result.deserialize(*this); return result; } else if constexpr (requires(type &result) { diff --git a/rx/include/rx/refl.hpp b/rx/include/rx/refl.hpp index 60ff2123a..9662ce79f 100644 --- a/rx/include/rx/refl.hpp +++ b/rx/include/rx/refl.hpp @@ -174,13 +174,29 @@ constexpr auto calcFieldCount() { } else if constexpr (requires { EnumT::count; }) { return static_cast(EnumT::count); } else if constexpr (!requires { getNameOf()[0]; }) { - return N; + if constexpr (requires { getNameOf()[0]; }) { + if constexpr (constexpr auto c = getNameOf()[0]; + c >= '0' && c <= '9') { + return N; + } else { + return calcFieldCount(); + } + } else { + return N; + } } else { - constexpr auto c = getNameOf()[0]; - if constexpr (!requires { getNameOf()[0]; }) { - return N; - } else if constexpr (c >= '0' && c <= '9') { - return N; + if constexpr (constexpr auto c = getNameOf()[0]; + c >= '0' && c <= '9') { + if constexpr (requires { getNameOf()[0]; }) { + if constexpr (constexpr auto c = getNameOf()[0]; + c >= '0' && c <= '9') { + return N; + } else { + return calcFieldCount(); + } + } else { + return N; + } } else { return calcFieldCount(); }