mirror of
https://github.com/RPCSX/rpcsx.git
synced 2026-04-05 14:37:08 +00:00
orbis: implement protect dmem for pid
fixed void mappings fixed fmem commit add map flags validation redirect stdout/stderr to log-init.txt remove tty.txt, redirect to stdin/stdout ipmi: handle SceLncService::loadExec(status, "EXIT") fill SceShellCoreUtil shm magic
This commit is contained in:
DH
parent
b9d36bc0b8
commit
3f6fad89c2
21 changed files with 1088 additions and 299 deletions
|
|
@ -317,8 +317,7 @@ struct AllocableResource : Resource {
|
|||
} else {
|
||||
if (flags & AllocationFlags::Stack) {
|
||||
fixedRange = rx::AddressRange::fromBeginSize(
|
||||
rx::alignDown(it.endAddress() - size, alignment),
|
||||
size);
|
||||
rx::alignDown(it.endAddress() - size, alignment), size);
|
||||
} else {
|
||||
fixedRange = rx::AddressRange::fromBeginSize(
|
||||
rx::alignUp(it.beginAddress(), alignment), size);
|
||||
|
|
@ -353,6 +352,10 @@ struct AllocableResource : Resource {
|
|||
return {it, {}, fixedRange};
|
||||
}
|
||||
|
||||
return {merge(it), {}, fixedRange};
|
||||
}
|
||||
|
||||
iterator merge(iterator it) {
|
||||
if (it != begin()) {
|
||||
// try to merge with previous node
|
||||
iterator prevIt = it;
|
||||
|
|
@ -393,7 +396,7 @@ struct AllocableResource : Resource {
|
|||
}
|
||||
}
|
||||
|
||||
return {it, {}, fixedRange};
|
||||
return it;
|
||||
}
|
||||
|
||||
void destroy() {
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ add_library(obj.orbis-kernel OBJECT
|
|||
)
|
||||
|
||||
target_link_libraries(obj.orbis-kernel PUBLIC orbis::kernel::config rx kernel)
|
||||
target_compile_options(obj.orbis-kernel PRIVATE "-mfsgsbase")
|
||||
|
||||
target_include_directories(obj.orbis-kernel
|
||||
PUBLIC
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ namespace orbis {
|
|||
enum class MemoryType : std::uint32_t {
|
||||
Invalid = -1u,
|
||||
WbOnion = 0, // write back, CPU bus
|
||||
WCGarlic = 3, // combining, GPU bus
|
||||
WcGarlic = 3, // combining, GPU bus
|
||||
WbGarlic = 10, // write back, GPU bus
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ allocate(unsigned dmemIndex, rx::AddressRange searchRange, std::uint64_t len,
|
|||
MemoryType memoryType, std::uint64_t alignment = kPageSize,
|
||||
bool pooled = false);
|
||||
|
||||
std::pair<std::uint64_t, ErrorCode>
|
||||
allocateSystem(unsigned dmemIndex, std::uint64_t len, MemoryType memoryType,
|
||||
std::uint64_t alignment = kPageSize);
|
||||
|
||||
ErrorCode release(unsigned dmemIndex, rx::AddressRange range,
|
||||
bool pooled = false);
|
||||
|
||||
|
|
@ -54,9 +58,17 @@ std::pair<rx::AddressRange, ErrorCode>
|
|||
getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange,
|
||||
std::uint64_t alignment);
|
||||
|
||||
ErrorCode map(unsigned dmemIndex, rx::AddressRange range, std::uint64_t offset,
|
||||
ErrorCode map(orbis::Process *process, unsigned dmemIndex,
|
||||
rx::AddressRange range, std::uint64_t offset,
|
||||
rx::EnumBitSet<vmem::Protection> protection);
|
||||
|
||||
ErrorCode notifyUnmap(orbis::Process *process, unsigned dmemIndex,
|
||||
std::uint64_t offset, rx::AddressRange range);
|
||||
|
||||
ErrorCode protect(orbis::Process *process, unsigned dmemIndex,
|
||||
rx::AddressRange range,
|
||||
rx::EnumBitSet<vmem::Protection> prot);
|
||||
|
||||
std::pair<std::uint64_t, ErrorCode> getPmemOffset(unsigned dmemIndex,
|
||||
std::uint64_t dmemOffset);
|
||||
} // namespace orbis::dmem
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ struct Module final {
|
|||
|
||||
DynType dynType = DynType::None;
|
||||
|
||||
uint32_t refCount{};
|
||||
uint32_t phNum{};
|
||||
uint64_t phdrAddress{};
|
||||
|
||||
|
|
|
|||
|
|
@ -786,7 +786,7 @@ SysResult sys_localtime_to_utc(Thread *thread, int64_t time, uint unk,
|
|||
SysResult sys_set_uevt(Thread *thread /* TODO */);
|
||||
SysResult sys_get_cpu_usage_proc(Thread *thread /* TODO */);
|
||||
SysResult sys_get_map_statistics(Thread *thread /* TODO */);
|
||||
SysResult sys_set_chicken_switches(Thread *thread /* TODO */);
|
||||
SysResult sys_set_chicken_switches(Thread *thread, sint flags);
|
||||
SysResult sys_extend_page_table_pool(Thread *thread);
|
||||
SysResult sys_extend_page_table_pool2(Thread *thread);
|
||||
SysResult sys_get_kernel_mem_statistics(Thread *thread /* TODO */);
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include "rx/Serializer.hpp"
|
||||
#include "rx/SharedMutex.hpp"
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
|
||||
namespace orbis {
|
||||
class KernelContext;
|
||||
|
|
@ -93,6 +94,7 @@ struct Process final {
|
|||
std::optional<sint> exitStatus;
|
||||
|
||||
std::uint32_t sdkVersion = 0;
|
||||
bool allowDmemAliasing = false;
|
||||
std::uint64_t nextTlsSlot = 1;
|
||||
std::uint64_t lastTlsOffset = 0;
|
||||
|
||||
|
|
@ -126,6 +128,50 @@ struct Process final {
|
|||
}
|
||||
|
||||
Budget *getBudget() const;
|
||||
|
||||
template <typename Cb>
|
||||
requires(alignof(Cb) <= 8 && sizeof(Cb) <= 64) &&
|
||||
(std::is_same_v<std::invoke_result_t<Cb>, void> ||
|
||||
(alignof(std::invoke_result_t<Cb>) <= 8 &&
|
||||
sizeof(std::invoke_result_t<Cb>) <= 64))
|
||||
std::invoke_result_t<Cb> invoke(Cb &&fn) {
|
||||
auto constructObject = [](void *to, void *from) {
|
||||
new (to) Cb(std::move(*reinterpret_cast<Cb *>(from)));
|
||||
};
|
||||
|
||||
auto destroyObject = [](void *object) {
|
||||
reinterpret_cast<Cb *>(object)->~Cb();
|
||||
};
|
||||
|
||||
if constexpr (std::is_same_v<std::invoke_result_t<Cb>, void>) {
|
||||
invokeImpl(
|
||||
nullptr, nullptr, &fn, constructObject, destroyObject,
|
||||
[](void *, void *fnPtr) { (*reinterpret_cast<Cb *>(fnPtr))(); });
|
||||
} else {
|
||||
alignas(std::invoke_result_t<Cb>) char
|
||||
result[sizeof(std::invoke_result_t<Cb>)];
|
||||
invokeImpl(
|
||||
&result,
|
||||
[](void *to, void *from) {
|
||||
new (to) std::invoke_result_t<Cb>(
|
||||
std::move(*reinterpret_cast<std::invoke_result_t<Cb> *>(from)));
|
||||
},
|
||||
&fn, constructObject, destroyObject,
|
||||
[](void *result, void *fnPtr) {
|
||||
new (result)
|
||||
std::invoke_result_t<Cb>((*reinterpret_cast<Cb *>(fnPtr))());
|
||||
});
|
||||
return std::move(*reinterpret_cast<std::invoke_result_t<Cb> *>(result));
|
||||
}
|
||||
}
|
||||
|
||||
void invokeAsync(void (*fn)());
|
||||
|
||||
private:
|
||||
void invokeImpl(void *returnValue, void (*copyResult)(void *to, void *from),
|
||||
void *fnPtr, void (*constructObject)(void *to, void *from),
|
||||
void (*destroyObject)(void *to),
|
||||
void (*invokeImpl)(void *returnValue, void *fnPtr));
|
||||
};
|
||||
|
||||
pid_t allocatePid();
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ enum class BlockFlagsEx : std::uint8_t {
|
|||
Private,
|
||||
Shared,
|
||||
PoolControl,
|
||||
Void,
|
||||
Reserved,
|
||||
|
||||
bitset_last = Reserved
|
||||
|
|
@ -149,6 +150,31 @@ toGpuProtection(rx::EnumBitSet<Protection> prot) {
|
|||
return result;
|
||||
}
|
||||
|
||||
inline bool validateProtection(rx::EnumBitSet<Protection> &prot) {
|
||||
prot = rx::EnumBitSet<Protection>::fromUnderlying(prot.toUnderlying() & 0xff);
|
||||
|
||||
if (prot & ~(kProtCpuAll | kProtGpuAll)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (prot & Protection::CpuWrite) {
|
||||
prot |= Protection::CpuRead;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool validateMemoryType(MemoryType type,
|
||||
rx::EnumBitSet<Protection> prot) {
|
||||
if (type == MemoryType::WbGarlic) {
|
||||
if (prot & (Protection::CpuWrite | Protection::GpuWrite)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void initialize(Process *process, bool force = false);
|
||||
void fork(Process *process, Process *parentThread);
|
||||
|
||||
|
|
@ -164,22 +190,27 @@ mapFile(Process *process, std::uint64_t addressHint, std::uint64_t size,
|
|||
rx::EnumBitSet<Protection> prot, rx::EnumBitSet<BlockFlags> blockFlags,
|
||||
rx::EnumBitSet<BlockFlagsEx> blockFlagsEx, File *file,
|
||||
std::uint64_t fileOffset, std::string_view name = {},
|
||||
std::uint64_t alignment = kPageSize,
|
||||
std::uint64_t alignment = kPageSize, std::uint64_t callerAddress = 0,
|
||||
MemoryType type = MemoryType::Invalid);
|
||||
|
||||
std::pair<rx::AddressRange, ErrorCode>
|
||||
mapDirect(Process *process, std::uint64_t addressHint,
|
||||
rx::AddressRange directRange, rx::EnumBitSet<Protection> prot,
|
||||
rx::EnumBitSet<AllocationFlags> allocFlags,
|
||||
std::string_view name = {}, std::uint64_t alignment = kPageSize,
|
||||
MemoryType type = MemoryType::Invalid);
|
||||
std::pair<rx::AddressRange, ErrorCode> mapDirect(
|
||||
Process *process, std::uint64_t addressHint, rx::AddressRange directRange,
|
||||
rx::EnumBitSet<Protection> prot, rx::EnumBitSet<AllocationFlags> allocFlags,
|
||||
std::string_view name = {}, std::uint64_t alignment = kPageSize,
|
||||
std::uint64_t callerAddress = 0, MemoryType type = MemoryType::Invalid);
|
||||
|
||||
std::pair<rx::AddressRange, ErrorCode>
|
||||
mapFlex(Process *process, std::uint64_t size, rx::EnumBitSet<Protection> prot,
|
||||
std::uint64_t addressHint = 0,
|
||||
rx::EnumBitSet<AllocationFlags> allocFlags = {},
|
||||
rx::EnumBitSet<BlockFlags> blockFlags = {}, std::string_view name = {},
|
||||
std::uint64_t alignment = kPageSize);
|
||||
std::uint64_t alignment = kPageSize, std::uint64_t callerAddress = 0);
|
||||
|
||||
std::pair<rx::AddressRange, ErrorCode>
|
||||
mapVoid(Process *process, std::uint64_t size, std::uint64_t addressHint = 0,
|
||||
rx::EnumBitSet<AllocationFlags> allocFlags = {},
|
||||
std::string_view name = {}, std::uint64_t alignment = kPageSize,
|
||||
std::uint64_t callerAddress = 0);
|
||||
|
||||
std::pair<rx::AddressRange, ErrorCode>
|
||||
commitPooled(Process *process, rx::AddressRange addressRange, MemoryType type,
|
||||
|
|
@ -191,7 +222,6 @@ ErrorCode protect(Process *process, rx::AddressRange range,
|
|||
ErrorCode unmap(Process *process, rx::AddressRange range);
|
||||
ErrorCode setName(Process *process, rx::AddressRange range,
|
||||
std::string_view name);
|
||||
ErrorCode setType(Process *process, rx::AddressRange range, MemoryType type);
|
||||
ErrorCode setTypeAndProtect(Process *process, rx::AddressRange range,
|
||||
MemoryType type, rx::EnumBitSet<Protection> prot);
|
||||
std::optional<QueryResult> query(Process *process, std::uint64_t address,
|
||||
|
|
|
|||
|
|
@ -76,8 +76,6 @@ void initializeAllocator() {
|
|||
sMemoryResource->m_heap_next = ptr + sizeof(KernelMemoryResource);
|
||||
sMemoryResource->m_heap = std::move(heap);
|
||||
|
||||
rx::print(stderr, "global: size {}, alignment {}\n", GlobalStorage::GetSize(),
|
||||
GlobalStorage::GetAlignment());
|
||||
// allocate whole global storage
|
||||
g_globalStorage = (std::byte *)sMemoryResource->kalloc(
|
||||
GlobalStorage::GetSize(), GlobalStorage::GetAlignment());
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "dmem.hpp"
|
||||
#include "KernelAllocator.hpp"
|
||||
#include "KernelContext.hpp"
|
||||
#include "KernelObject.hpp"
|
||||
#include "error.hpp"
|
||||
#include "kernel/KernelObject.hpp"
|
||||
|
|
@ -14,6 +15,7 @@
|
|||
#include <array>
|
||||
#include <rx/format.hpp>
|
||||
#include <string_view>
|
||||
#include <thread/Process.hpp>
|
||||
#include <utility>
|
||||
|
||||
struct DirectMemoryAllocation {
|
||||
|
|
@ -21,6 +23,35 @@ struct DirectMemoryAllocation {
|
|||
static constexpr std::uint32_t kPooledBit = 1 << 30;
|
||||
std::uint32_t type = 0;
|
||||
|
||||
struct Mapping {
|
||||
orbis::Process *process;
|
||||
rx::AddressRange vmRange;
|
||||
|
||||
void serialize(rx::Serializer &s) const {
|
||||
s.serialize(process->pid);
|
||||
s.serialize(vmRange);
|
||||
}
|
||||
|
||||
void deserialize(rx::Deserializer &d) {
|
||||
auto pid = d.deserialize<orbis::pid_t>();
|
||||
if (d.failure()) {
|
||||
return;
|
||||
}
|
||||
auto foundProcess = orbis::findProcessById(pid);
|
||||
if (foundProcess == nullptr) {
|
||||
d.setFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
process = foundProcess;
|
||||
d.deserialize(vmRange);
|
||||
}
|
||||
|
||||
bool operator==(const Mapping &) const = default;
|
||||
};
|
||||
|
||||
orbis::kvector<Mapping> mappings;
|
||||
|
||||
[[nodiscard]] bool isPooled() const { return type & kPooledBit; }
|
||||
void markAsPooled() { type |= kPooledBit | kAllocatedBit; }
|
||||
|
||||
|
|
@ -42,7 +73,9 @@ struct DirectMemoryAllocation {
|
|||
[[nodiscard]] DirectMemoryAllocation
|
||||
merge(const DirectMemoryAllocation &other, rx::AddressRange,
|
||||
rx::AddressRange) const {
|
||||
return other;
|
||||
auto result = *this;
|
||||
result.mappings.insert_range(result.mappings.end(), other.mappings);
|
||||
return result;
|
||||
}
|
||||
|
||||
bool operator==(const DirectMemoryAllocation &) const = default;
|
||||
|
|
@ -60,6 +93,8 @@ struct DirectMemoryResource
|
|||
using BaseResource =
|
||||
kernel::AllocableResource<DirectMemoryAllocation, orbis::kallocator>;
|
||||
|
||||
BaseResource systemResource;
|
||||
|
||||
void create(std::size_t size) {
|
||||
auto [pmemRange, errc] = orbis::pmem::allocate(0, size, {}, 64 * 1024);
|
||||
|
||||
|
|
@ -86,6 +121,8 @@ struct DirectMemoryResource
|
|||
if (result != std::errc{}) {
|
||||
return orbis::toErrorCode(result);
|
||||
}
|
||||
|
||||
systemResource.destroy();
|
||||
dmemReservedSize = 0;
|
||||
return {};
|
||||
}
|
||||
|
|
@ -99,6 +136,22 @@ struct DirectMemoryResource
|
|||
d.deserialize(static_cast<DirectMemoryResourceState &>(*this));
|
||||
BaseResource::deserialize(d);
|
||||
}
|
||||
|
||||
std::pair<std::uint64_t, orbis::ErrorCode> reserveSystem(std::uint64_t size) {
|
||||
DirectMemoryAllocation alloc;
|
||||
alloc.setMemoryType(orbis::MemoryType::WbOnion);
|
||||
auto result = map(0, size, alloc, orbis::AllocationFlags::Stack,
|
||||
orbis::dmem::kPageSize);
|
||||
|
||||
if (result.errc != std::errc{}) {
|
||||
return {{}, orbis::toErrorCode(result.errc)};
|
||||
}
|
||||
|
||||
allocations.unmap(result.range);
|
||||
dmemReservedSize += size;
|
||||
systemResource.allocations.map(result.range, {});
|
||||
return {result.range.beginAddress(), {}};
|
||||
}
|
||||
};
|
||||
|
||||
static std::array g_dmemPools = {
|
||||
|
|
@ -111,10 +164,12 @@ static std::array g_dmemPools = {
|
|||
};
|
||||
|
||||
orbis::ErrorCode orbis::dmem::initialize() {
|
||||
g_dmemPools[0]->create(0x120000000);
|
||||
g_dmemPools[0]->create(0x180000000);
|
||||
g_dmemPools[1]->create(0x1000000);
|
||||
g_dmemPools[2]->create(0x1000000);
|
||||
|
||||
g_dmemPools[0]->reserveSystem(0x60000000);
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
@ -222,6 +277,43 @@ orbis::dmem::allocate(unsigned dmemIndex, rx::AddressRange searchRange,
|
|||
return {result, {}};
|
||||
}
|
||||
|
||||
std::pair<std::uint64_t, orbis::ErrorCode>
|
||||
orbis::dmem::allocateSystem(unsigned dmemIndex, std::uint64_t len,
|
||||
MemoryType memoryType, std::uint64_t alignment) {
|
||||
if (dmemIndex >= std::size(g_dmemPools)) {
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
auto dmem = g_dmemPools[dmemIndex];
|
||||
std::lock_guard lock(*dmem);
|
||||
|
||||
alignment = alignment == 0 ? kPageSize : alignment;
|
||||
len = rx::alignUp(len, dmem::kPageSize);
|
||||
|
||||
DirectMemoryAllocation allocation;
|
||||
allocation.setMemoryType(memoryType);
|
||||
|
||||
auto allocResult = dmem->systemResource.map(0, len, allocation,
|
||||
AllocationFlags::Dry, alignment);
|
||||
if (allocResult.errc != std::errc{}) {
|
||||
return {{}, ErrorCode::AGAIN};
|
||||
}
|
||||
|
||||
auto result = allocResult.range.beginAddress();
|
||||
|
||||
auto commitResult =
|
||||
dmem->map(result, len, allocation, AllocationFlags::Fixed, alignment);
|
||||
|
||||
rx::dieIf(commitResult.errc != std::errc{},
|
||||
"dmem: failed to commit main memory, error {}", commitResult.errc);
|
||||
|
||||
dmemDump(dmemIndex, rx::format("allocated main {:x}-{:x}",
|
||||
allocResult.range.beginAddress(),
|
||||
allocResult.range.endAddress()));
|
||||
|
||||
return {allocResult.range.beginAddress() | 0x4000000000, {}};
|
||||
}
|
||||
|
||||
orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex,
|
||||
rx::AddressRange range, bool pooled) {
|
||||
if (dmemIndex >= std::size(g_dmemPools)) {
|
||||
|
|
@ -253,6 +345,13 @@ orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex,
|
|||
return ErrorCode::NOENT;
|
||||
}
|
||||
|
||||
for (auto mapping : it->mappings) {
|
||||
mapping.process->invoke(
|
||||
[=] { vmem::unmap(mapping.process, mapping.vmRange); });
|
||||
}
|
||||
|
||||
it->mappings.clear();
|
||||
|
||||
DirectMemoryAllocation allocation{};
|
||||
auto result = dmem->map(range.beginAddress(), range.size(), allocation,
|
||||
AllocationFlags::Fixed, vmem::kPageSize);
|
||||
|
|
@ -261,6 +360,8 @@ orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex,
|
|||
return toErrorCode(result.errc);
|
||||
}
|
||||
|
||||
dmemDump(dmemIndex, rx::format("released {:x}-{:x}", range.beginAddress(),
|
||||
range.endAddress()));
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
@ -281,16 +382,7 @@ orbis::dmem::reserveSystem(unsigned dmemIndex, std::uint64_t size) {
|
|||
return {{}, ErrorCode::NOMEM};
|
||||
}
|
||||
|
||||
DirectMemoryAllocation alloc;
|
||||
alloc.setMemoryType(MemoryType::WbOnion);
|
||||
auto result = dmem->map(0, size, alloc, AllocationFlags::Stack, kPageSize);
|
||||
|
||||
if (result.errc != std::errc{}) {
|
||||
return {{}, toErrorCode(result.errc)};
|
||||
}
|
||||
|
||||
dmem->dmemReservedSize += size;
|
||||
return {result.range.beginAddress() | 0x4000000000, {}};
|
||||
return dmem->reserveSystem(size);
|
||||
}
|
||||
|
||||
std::pair<std::uint64_t, orbis::ErrorCode>
|
||||
|
|
@ -465,7 +557,7 @@ orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange,
|
|||
|
||||
alignment = alignment == 0 ? kPageSize : alignment;
|
||||
|
||||
if (alignment % kPageSize) {
|
||||
if (alignment % vmem::kPageSize) {
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
|
|
@ -474,7 +566,9 @@ orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange,
|
|||
if (searchRange.endAddress() > dmem->dmemTotalSize) {
|
||||
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
|
||||
searchRange.endAddress(), dmem->dmemTotalSize);
|
||||
return {{}, orbis::ErrorCode::INVAL};
|
||||
// return {{}, orbis::ErrorCode::INVAL};
|
||||
searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(),
|
||||
dmem->dmemTotalSize);
|
||||
}
|
||||
|
||||
if (!searchRange.isValid() &&
|
||||
|
|
@ -502,7 +596,11 @@ orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange,
|
|||
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
|
||||
searchRange.endAddress(), dmem->dmemTotalSize,
|
||||
dmem->dmemReservedSize);
|
||||
return {{}, orbis::ErrorCode::INVAL};
|
||||
searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(),
|
||||
dmem->dmemTotalSize -
|
||||
dmem->dmemReservedSize);
|
||||
|
||||
// return {{}, orbis::ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
auto it = dmem->lowerBound(searchRange.beginAddress());
|
||||
|
|
@ -529,8 +627,8 @@ orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange,
|
|||
return {result, {}};
|
||||
}
|
||||
|
||||
orbis::ErrorCode orbis::dmem::map(unsigned dmemIndex, rx::AddressRange range,
|
||||
std::uint64_t offset,
|
||||
orbis::ErrorCode orbis::dmem::map(orbis::Process *process, unsigned dmemIndex,
|
||||
rx::AddressRange range, std::uint64_t offset,
|
||||
rx::EnumBitSet<vmem::Protection> protection) {
|
||||
if (dmemIndex >= std::size(g_dmemPools)) {
|
||||
return ErrorCode::INVAL;
|
||||
|
|
@ -569,6 +667,15 @@ orbis::ErrorCode orbis::dmem::map(unsigned dmemIndex, rx::AddressRange range,
|
|||
return orbis::ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
if (!vmem::validateMemoryType(allocationInfoIt->getMemoryType(),
|
||||
protection)) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
if (!allocationInfoIt->mappings.empty() && !process->allowDmemAliasing) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
auto directRange = rx::AddressRange::fromBeginSize(offset, range.size())
|
||||
.intersection(allocationInfoIt.range());
|
||||
|
||||
|
|
@ -579,8 +686,96 @@ orbis::ErrorCode orbis::dmem::map(unsigned dmemIndex, rx::AddressRange range,
|
|||
auto physicalRange =
|
||||
rx::AddressRange::fromBeginSize(dmem->pmemOffset + offset, range.size());
|
||||
|
||||
return orbis::pmem::map(range.beginAddress(), physicalRange,
|
||||
vmem::toCpuProtection(protection));
|
||||
auto result = process->invoke([=] {
|
||||
return orbis::pmem::map(range.beginAddress(), physicalRange,
|
||||
vmem::toCpuProtection(protection));
|
||||
});
|
||||
|
||||
if (result == ErrorCode{}) {
|
||||
allocationInfoIt->mappings.push_back({
|
||||
.process = process,
|
||||
.vmRange = range,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
orbis::ErrorCode orbis::dmem::notifyUnmap(orbis::Process *process,
|
||||
unsigned dmemIndex,
|
||||
std::uint64_t offset,
|
||||
rx::AddressRange range) {
|
||||
if (dmemIndex >= std::size(g_dmemPools)) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
auto dmem = g_dmemPools[dmemIndex];
|
||||
std::lock_guard lock(*dmem);
|
||||
|
||||
auto it = dmem->query(offset);
|
||||
if (it == dmem->end()) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
for (auto mapIt = it->mappings.begin(); mapIt != it->mappings.end();) {
|
||||
if (mapIt->process == process && mapIt->vmRange.intersects(range)) {
|
||||
if (mapIt->vmRange == range) {
|
||||
mapIt = it->mappings.erase(mapIt);
|
||||
break;
|
||||
}
|
||||
|
||||
if (mapIt->vmRange.beginAddress() == range.beginAddress()) {
|
||||
mapIt->vmRange = rx::AddressRange::fromBeginEnd(
|
||||
range.endAddress(), mapIt->vmRange.endAddress());
|
||||
break;
|
||||
}
|
||||
|
||||
if (mapIt->vmRange.endAddress() == range.endAddress()) {
|
||||
mapIt->vmRange = rx::AddressRange::fromBeginEnd(
|
||||
mapIt->vmRange.beginAddress(), range.beginAddress());
|
||||
break;
|
||||
}
|
||||
|
||||
auto leftAllocation = rx::AddressRange::fromBeginEnd(
|
||||
mapIt->vmRange.beginAddress(), range.beginAddress());
|
||||
|
||||
auto rightAllocation = rx::AddressRange::fromBeginEnd(
|
||||
range.endAddress(), mapIt->vmRange.endAddress());
|
||||
|
||||
mapIt->vmRange = leftAllocation;
|
||||
it->mappings.push_back({.process = process, .vmRange = rightAllocation});
|
||||
break;
|
||||
}
|
||||
|
||||
++mapIt;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
orbis::ErrorCode orbis::dmem::protect(orbis::Process *process,
|
||||
unsigned dmemIndex,
|
||||
rx::AddressRange range,
|
||||
rx::EnumBitSet<vmem::Protection> prot) {
|
||||
auto dmem = g_dmemPools[dmemIndex];
|
||||
std::lock_guard lock(*dmem);
|
||||
|
||||
auto it = dmem->query(range.beginAddress());
|
||||
if (it == dmem->end()) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
if (!it.range().contains(range)) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
for (auto mapping : it->mappings) {
|
||||
if (process == nullptr || process == mapping.process) {
|
||||
vmem::protect(mapping.process, mapping.vmRange, prot);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
std::pair<std::uint64_t, orbis::ErrorCode>
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ static orbis::SysResult doPltRelocation(orbis::Process *process,
|
|||
foundInLibs.emplace_back(std::string_view(defLib.name));
|
||||
}
|
||||
|
||||
for (auto nsDefModule : defModule->namespaceModules) {
|
||||
for (auto &nsDefModule : defModule->namespaceModules) {
|
||||
for (auto defSym : nsDefModule->symbols) {
|
||||
if (defSym.id != symbol.id || defSym.bind == orbis::SymbolBind::Local) {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -78,6 +78,10 @@ orbis::sys_mtypeprotect(Thread *thread, uintptr_t addr, size_t len,
|
|||
MemoryType type,
|
||||
rx::EnumBitSet<vmem::Protection> prot) {
|
||||
auto range = rx::AddressRange::fromBeginSize(addr, len);
|
||||
if (!range.isValid() || static_cast<unsigned>(type) > 10) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
return vmem::setTypeAndProtect(thread->tproc, range, type, prot);
|
||||
}
|
||||
orbis::SysResult orbis::sys_regmgr_call(Thread *thread, uint32_t op,
|
||||
|
|
@ -1289,7 +1293,7 @@ orbis::sys_dynlib_get_info_ex(Thread *thread, SceKernelModule handle,
|
|||
result.ehFrameSize = module->ehFrameSize;
|
||||
std::memcpy(result.segments, module->segments, sizeof(ModuleSegment) * 2);
|
||||
result.segmentCount = 2;
|
||||
result.refCount = 1;
|
||||
result.refCount = module->refCount;
|
||||
ORBIS_LOG_WARNING(__FUNCTION__, result.id, result.name, result.tlsIndex,
|
||||
result.tlsInit, result.tlsInitSize, result.tlsSize,
|
||||
result.tlsOffset, result.tlsAlign, result.initProc,
|
||||
|
|
@ -1512,6 +1516,10 @@ orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr,
|
|||
auto callerAddress = getCallerAddress(thread);
|
||||
auto alignment = dmem::kPageSize;
|
||||
|
||||
if (static_cast<unsigned>(memoryType) > 10) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
{
|
||||
auto unpacked = unpackMapFlags(flags, dmem::kPageSize);
|
||||
alignment = unpacked.first;
|
||||
|
|
@ -1522,13 +1530,6 @@ orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr,
|
|||
|
||||
rx::EnumBitSet<AllocationFlags> allocFlags{};
|
||||
|
||||
if (!prot) {
|
||||
// HACK
|
||||
// FIXME: implement protect for pid
|
||||
prot = vmem::Protection::CpuRead | vmem::Protection::CpuWrite |
|
||||
vmem::Protection::GpuRead | vmem::Protection::GpuWrite;
|
||||
}
|
||||
|
||||
if (prot & vmem::Protection::CpuExec) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
|
@ -1548,7 +1549,7 @@ orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr,
|
|||
auto [range, errc] =
|
||||
vmem::mapDirect(thread->tproc, addr,
|
||||
rx::AddressRange::fromBeginSize(directMemoryStart, len),
|
||||
prot, allocFlags, name, alignment, memoryType);
|
||||
prot, allocFlags, name, alignment, callerAddress, memoryType);
|
||||
|
||||
if (errc != ErrorCode{}) {
|
||||
return errc;
|
||||
|
|
@ -1696,7 +1697,7 @@ orbis::SysResult orbis::sys_get_cpu_usage_proc(Thread *thread /* TODO */) {
|
|||
orbis::SysResult orbis::sys_get_map_statistics(Thread *thread /* TODO */) {
|
||||
return ErrorCode::NOSYS;
|
||||
}
|
||||
orbis::SysResult orbis::sys_set_chicken_switches(Thread *thread /* TODO */) {
|
||||
orbis::SysResult orbis::sys_set_chicken_switches(Thread *thread, sint flags) {
|
||||
return ErrorCode::NOSYS;
|
||||
}
|
||||
orbis::SysResult orbis::sys_extend_page_table_pool(Thread *thread) {
|
||||
|
|
|
|||
|
|
@ -4,10 +4,16 @@
|
|||
#include "KernelObject.hpp"
|
||||
#include "kernel/KernelObject.hpp"
|
||||
#include "rx/LinkedNode.hpp"
|
||||
#include "rx/Process.hpp"
|
||||
#include "rx/Serializer.hpp"
|
||||
#include "rx/SharedMutex.hpp"
|
||||
#include "rx/align.hpp"
|
||||
#include "rx/die.hpp"
|
||||
#include "thread/Thread.hpp"
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <csignal>
|
||||
#include <mutex>
|
||||
|
||||
struct ProcessIdList {
|
||||
rx::OwningIdMap<std::uint8_t, orbis::pid_t, 256, 0> pidMap;
|
||||
|
|
@ -132,6 +138,148 @@ orbis::Process *orbis::findProcessByHostId(std::uint64_t pid) {
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
struct InvokeDeliveryState {
|
||||
void (*invokeCb)(void *returnValue, void *fnPtr) = nullptr;
|
||||
alignas(8) std::byte returnData[64];
|
||||
alignas(8) std::byte callerObject[64];
|
||||
|
||||
static void invoke();
|
||||
|
||||
void serialize(rx::Serializer &) const {}
|
||||
void deserialize(rx::Deserializer &) {}
|
||||
};
|
||||
|
||||
struct AsyncInvokeDeliveryState {
|
||||
void (*invokeCb)() = nullptr;
|
||||
|
||||
static void invoke();
|
||||
|
||||
void serialize(rx::Serializer &) const {}
|
||||
void deserialize(rx::Deserializer &) {}
|
||||
};
|
||||
|
||||
auto g_invokeDelivery = orbis::createProcessLocalObject<
|
||||
kernel::LockableKernelObject<InvokeDeliveryState>>();
|
||||
auto g_asyncInvokeDelivery = orbis::createProcessLocalObject<
|
||||
kernel::LockableKernelObject<AsyncInvokeDeliveryState>>();
|
||||
|
||||
void InvokeDeliveryState::invoke() {
|
||||
auto state = orbis::g_currentThread->tproc->get(g_invokeDelivery);
|
||||
state->invokeCb(state->returnData, state->callerObject);
|
||||
state->invokeCb = {};
|
||||
state->unlock();
|
||||
}
|
||||
|
||||
void AsyncInvokeDeliveryState::invoke() {
|
||||
auto state = orbis::g_currentThread->tproc->get(g_asyncInvokeDelivery);
|
||||
auto cb = state->invokeCb;
|
||||
state->invokeCb = {};
|
||||
state->unlock();
|
||||
|
||||
cb();
|
||||
}
|
||||
|
||||
struct SignalHandlerObject {
|
||||
SignalHandlerObject() {
|
||||
struct sigaction act{};
|
||||
act.sa_sigaction = handleSignal;
|
||||
act.sa_flags = SA_SIGINFO | SA_ONSTACK;
|
||||
|
||||
if (sigaction(SIGUSR2, &act, nullptr)) {
|
||||
rx::die("SignalHandlerObject: failed to setup signal handler");
|
||||
}
|
||||
}
|
||||
|
||||
~SignalHandlerObject() {
|
||||
struct sigaction act{};
|
||||
act.sa_handler = SIG_DFL;
|
||||
|
||||
sigaction(SIGUSR2, &act, nullptr);
|
||||
}
|
||||
|
||||
[[gnu::no_stack_protector]] static void handleSignal(int sig, siginfo_t *info,
|
||||
void *context) {
|
||||
if (auto hostFs = _readgsbase_u64()) {
|
||||
_writefsbase_u64(hostFs);
|
||||
}
|
||||
|
||||
if (sig == SIGUSR2) {
|
||||
std::bit_cast<void (*)()>(info->si_value.sival_ptr)();
|
||||
}
|
||||
|
||||
auto ctx = reinterpret_cast<ucontext_t *>(context);
|
||||
|
||||
if (ctx->uc_mcontext.gregs[REG_RIP] < orbis::kMaxAddress) {
|
||||
if (auto thread = orbis::g_currentThread) {
|
||||
_writefsbase_u64(thread->fsBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void serialize(rx::Serializer &) const {}
|
||||
void deserialize(rx::Deserializer &) {}
|
||||
};
|
||||
|
||||
[[maybe_unused, gnu::used]] static auto g_signalHandler =
|
||||
orbis::createGlobalObject<SignalHandlerObject>();
|
||||
|
||||
void orbis::Process::invokeImpl(
|
||||
void *returnValue, void (*copyResult)(void *to, void *from), void *fnPtr,
|
||||
void (*constructObject)(void *to, void *from),
|
||||
void (*destroyObject)(void *to),
|
||||
void (*invokeCb)(void *returnValue, void *fnPtr)) {
|
||||
if (rx::getCurrentProcessId() == hostPid) {
|
||||
invokeCb(returnValue, fnPtr);
|
||||
return;
|
||||
}
|
||||
|
||||
auto invoker = get(g_invokeDelivery);
|
||||
|
||||
while (true) {
|
||||
std::lock_guard lock(*invoker);
|
||||
if (invoker->invokeCb != nullptr) {
|
||||
continue;
|
||||
}
|
||||
|
||||
invoker->invokeCb = invokeCb;
|
||||
constructObject(invoker->callerObject, fnPtr);
|
||||
sigqueue(hostPid, SIGUSR2,
|
||||
sigval{.sival_ptr =
|
||||
std::bit_cast<void *>(&InvokeDeliveryState::invoke)});
|
||||
invoker->lock();
|
||||
|
||||
destroyObject(invoker->callerObject);
|
||||
|
||||
if (returnValue != nullptr) {
|
||||
copyResult(returnValue, invoker->returnData);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void orbis::Process::invokeAsync(void (*fn)()) {
|
||||
if (rx::getCurrentProcessId() == hostPid) {
|
||||
fn();
|
||||
return;
|
||||
}
|
||||
|
||||
auto invoker = get(g_asyncInvokeDelivery);
|
||||
|
||||
while (true) {
|
||||
std::lock_guard lock(*invoker);
|
||||
if (invoker->invokeCb != nullptr) {
|
||||
continue;
|
||||
}
|
||||
|
||||
invoker->invokeCb = fn;
|
||||
sigqueue(hostPid, SIGUSR2,
|
||||
sigval{.sival_ptr = std::bit_cast<void *>(
|
||||
&AsyncInvokeDeliveryState::invoke)});
|
||||
invoker->lock();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void orbis::Process::serialize(rx::Serializer &s) const {
|
||||
Process::Storage::SerializeAll(storage, s);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ struct VirtualMemoryAllocation {
|
|||
orbis::MemoryType type = orbis::MemoryType::Invalid;
|
||||
rx::Ref<orbis::IoDevice> device;
|
||||
std::uint64_t deviceOffset = 0;
|
||||
std::uint64_t callerAddress = 0;
|
||||
rx::StaticString<31> name;
|
||||
|
||||
[[nodiscard]] bool isAllocated() const {
|
||||
|
|
@ -56,7 +57,8 @@ struct VirtualMemoryAllocation {
|
|||
isRelated(const VirtualMemoryAllocation &other, rx::AddressRange selfRange,
|
||||
[[maybe_unused]] rx::AddressRange rightRange) const {
|
||||
if (flags != other.flags || flagsEx != other.flagsEx ||
|
||||
prot != other.prot || type != other.type || device != other.device) {
|
||||
prot != other.prot || type != other.type || device != other.device ||
|
||||
callerAddress != other.callerAddress) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -64,13 +66,7 @@ struct VirtualMemoryAllocation {
|
|||
return true;
|
||||
}
|
||||
|
||||
bool isAnon = std::string_view(name).starts_with("anon:");
|
||||
|
||||
if (isAnon) {
|
||||
if (!std::string_view(other.name).starts_with("anon:")) {
|
||||
return false;
|
||||
}
|
||||
} else if (name != other.name) {
|
||||
if (name != other.name) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -117,6 +113,40 @@ struct VirtualMemoryAllocation {
|
|||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isFlex() const {
|
||||
return (flags & orbis::vmem::BlockFlags::FlexibleMemory) ==
|
||||
orbis::vmem::BlockFlags::FlexibleMemory;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isDirect() const {
|
||||
return (flags & orbis::vmem::BlockFlags::DirectMemory) ==
|
||||
orbis::vmem::BlockFlags::DirectMemory;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isPooled() const {
|
||||
return (flags & orbis::vmem::BlockFlags::PooledMemory) ==
|
||||
orbis::vmem::BlockFlags::PooledMemory;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isPoolCommited() const {
|
||||
return flags == (orbis::vmem::BlockFlags::PooledMemory |
|
||||
orbis::vmem::BlockFlags::Commited);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isPoolReserved() const {
|
||||
return flags == orbis::vmem::BlockFlags::PooledMemory;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isPoolControl() const {
|
||||
return flags == orbis::vmem::BlockFlags::PooledMemory &&
|
||||
(flagsEx & orbis::vmem::BlockFlagsEx::PoolControl);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool isVoid() const {
|
||||
return (flagsEx & orbis::vmem::BlockFlagsEx::Void) ==
|
||||
orbis::vmem::BlockFlagsEx::Void;
|
||||
}
|
||||
|
||||
void serialize(rx::Serializer &s) const {}
|
||||
void deserialize(rx::Deserializer &d) {}
|
||||
};
|
||||
|
|
@ -239,6 +269,7 @@ static void release(orbis::Process *process, decltype(g_vmInstance)::type *vmem,
|
|||
}
|
||||
|
||||
if (it->flags & orbis::vmem::BlockFlags::DirectMemory) {
|
||||
orbis::dmem::notifyUnmap(process, 0, it->deviceOffset, range);
|
||||
budget->release(orbis::BudgetResource::Dmem, blockRange.size());
|
||||
}
|
||||
|
||||
|
|
@ -258,12 +289,15 @@ static void release(orbis::Process *process, decltype(g_vmInstance)::type *vmem,
|
|||
}
|
||||
}
|
||||
|
||||
static orbis::ErrorCode validateRange(
|
||||
decltype(g_vmInstance)::type *vmem,
|
||||
decltype(g_vmInstance)::type::iterator it, rx::AddressRange range,
|
||||
rx::FunctionRef<orbis::ErrorCode(const VirtualMemoryAllocation &)> cb) {
|
||||
static orbis::ErrorCode
|
||||
validateRange(decltype(g_vmInstance)::type *vmem,
|
||||
decltype(g_vmInstance)::type::iterator it, rx::AddressRange range,
|
||||
rx::FunctionRef<orbis::ErrorCode(const VirtualMemoryAllocation &,
|
||||
rx::AddressRange)>
|
||||
cb) {
|
||||
while (it != vmem->end() && it.beginAddress() < range.endAddress()) {
|
||||
if (auto errc = cb(it.get()); errc != orbis::ErrorCode{}) {
|
||||
if (auto errc = cb(it.get(), it.range().intersection(range));
|
||||
errc != orbis::ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
|
|
@ -273,25 +307,45 @@ static orbis::ErrorCode validateRange(
|
|||
return {};
|
||||
}
|
||||
|
||||
static void modifyRange(
|
||||
static decltype(g_vmInstance)::type::iterator modifyRange(
|
||||
decltype(g_vmInstance)::type *vmem,
|
||||
decltype(g_vmInstance)::type::iterator it, rx::AddressRange range,
|
||||
rx::EnumBitSet<orbis::AllocationFlags> allocFlags,
|
||||
rx::FunctionRef<void(VirtualMemoryAllocation &, rx::AddressRange)> cb) {
|
||||
auto returnIt = it;
|
||||
while (it != vmem->end() && it.beginAddress() < range.endAddress()) {
|
||||
auto mapRange = range.intersection(it.range());
|
||||
auto allocInfo = it.get();
|
||||
auto itRange = it.range();
|
||||
auto mapRange = range.intersection(itRange);
|
||||
if (mapRange == itRange) {
|
||||
cb(it.get(), mapRange);
|
||||
|
||||
if (allocInfo.device != nullptr &&
|
||||
!(allocInfo.flags & orbis::vmem::BlockFlags::PooledMemory)) {
|
||||
allocInfo.deviceOffset += mapRange.beginAddress() - it.beginAddress();
|
||||
if (!(allocFlags & orbis::AllocationFlags::NoMerge)) {
|
||||
it = vmem->merge(it);
|
||||
}
|
||||
} else {
|
||||
auto allocInfo = it.get();
|
||||
|
||||
if (allocInfo.device != nullptr &&
|
||||
!(allocInfo.flags & orbis::vmem::BlockFlags::PooledMemory)) {
|
||||
allocInfo.deviceOffset += mapRange.beginAddress() - it.beginAddress();
|
||||
}
|
||||
|
||||
cb(allocInfo, mapRange);
|
||||
auto result = vmem->map(
|
||||
mapRange.beginAddress(), mapRange.size(), allocInfo,
|
||||
orbis::AllocationFlags::Fixed | allocFlags, orbis::vmem::kPageSize);
|
||||
|
||||
if (returnIt == it) {
|
||||
returnIt = result.it;
|
||||
}
|
||||
|
||||
it = result.it;
|
||||
}
|
||||
|
||||
cb(allocInfo, mapRange);
|
||||
vmem->map(mapRange.beginAddress(), mapRange.size(), allocInfo,
|
||||
orbis::AllocationFlags::Fixed, orbis::vmem::kPageSize);
|
||||
|
||||
++it;
|
||||
}
|
||||
|
||||
return returnIt;
|
||||
}
|
||||
|
||||
void orbis::vmem::initialize(Process *process, bool force) {
|
||||
|
|
@ -336,6 +390,7 @@ void orbis::vmem::initialize(Process *process, bool force) {
|
|||
for (auto usedRange : rx::mem::query(range)) {
|
||||
reserveRangeImpl(
|
||||
rx::AddressRange::fromBeginEnd(address, usedRange.beginAddress()));
|
||||
vmem->allocations.map(usedRange, {.flagsEx = BlockFlagsEx::Reserved});
|
||||
|
||||
address = usedRange.endAddress();
|
||||
}
|
||||
|
|
@ -373,17 +428,21 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
rx::EnumBitSet<BlockFlags> blockFlags,
|
||||
rx::EnumBitSet<BlockFlagsEx> blockFlagsEx, File *file,
|
||||
std::uint64_t fileOffset, std::string_view name, std::uint64_t alignment,
|
||||
MemoryType type) {
|
||||
std::uint64_t callerAddress, MemoryType type) {
|
||||
blockFlags |= file->device->blockFlags;
|
||||
|
||||
if (!validateProtection(prot)) {
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
if (blockFlags & BlockFlags::PooledMemory) {
|
||||
if (size < dmem::kPageSize * 2 || size % dmem::kPageSize) {
|
||||
return {{}, orbis::ErrorCode::INVAL};
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
}
|
||||
|
||||
if (blockFlags & (BlockFlags::DirectMemory | BlockFlags::PooledMemory)) {
|
||||
if (prot & vmem::Protection::CpuExec) {
|
||||
if (prot & Protection::CpuExec) {
|
||||
return {{}, ErrorCode::ACCES};
|
||||
}
|
||||
|
||||
|
|
@ -394,7 +453,7 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
|
||||
if (allocFlags & AllocationFlags::Fixed) {
|
||||
if (addressHint % alignment) {
|
||||
return {{}, orbis::ErrorCode::INVAL};
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -402,6 +461,14 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
!(allocFlags & AllocationFlags::NoOverwrite);
|
||||
|
||||
VirtualMemoryAllocation allocationInfo;
|
||||
if (process->sdkVersion < 0x5500000) {
|
||||
allocationInfo.callerAddress = callerAddress;
|
||||
}
|
||||
if (process->sdkVersion < 0x2000000 &&
|
||||
(blockFlags & (BlockFlags::DirectMemory | BlockFlags::PooledMemory))) {
|
||||
allocFlags |= AllocationFlags::NoMerge;
|
||||
}
|
||||
|
||||
allocationInfo.flagsEx = blockFlagsEx | BlockFlagsEx::Allocated;
|
||||
allocationInfo.device = file->device;
|
||||
allocationInfo.prot = prot;
|
||||
|
|
@ -444,13 +511,13 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
}
|
||||
|
||||
if (blockFlags & BlockFlags::FlexibleMemory) {
|
||||
if (prot) {
|
||||
if (!budget->acquire(BudgetResource::Fmem, size)) {
|
||||
rx::println(stderr, "map: fmem budget: failed to allocate {:#x} bytes",
|
||||
size);
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
if (!budget->acquire(BudgetResource::Fmem, size)) {
|
||||
rx::println(stderr, "map: fmem budget: failed to allocate {:#x} bytes",
|
||||
size);
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
if (prot) {
|
||||
blockFlags |= BlockFlags::Commited;
|
||||
}
|
||||
}
|
||||
|
|
@ -458,35 +525,38 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
allocFlags = AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge);
|
||||
|
||||
if (blockFlags & BlockFlags::DirectMemory) {
|
||||
if (prot) {
|
||||
if (!budget->acquire(BudgetResource::Dmem, size)) {
|
||||
rx::println(stderr, "map: dmem budget: failed to allocate {:#x} bytes",
|
||||
size);
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
if (!budget->acquire(BudgetResource::Dmem, size)) {
|
||||
rx::println(stderr, "map: dmem budget: failed to allocate {:#x} bytes",
|
||||
size);
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
if (prot) {
|
||||
blockFlags |= BlockFlags::Commited;
|
||||
}
|
||||
}
|
||||
|
||||
if (blockFlags & BlockFlags::PooledMemory) {
|
||||
if (auto errc = blockpool::allocateControlBlock();
|
||||
errc != orbis::ErrorCode{}) {
|
||||
if (auto errc = blockpool::allocateControlBlock(); errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
allocationInfo.flagsEx |= BlockFlagsEx::PoolControl;
|
||||
}
|
||||
|
||||
if (auto error = file->device->map(range, fileOffset, prot, file, process);
|
||||
error != ErrorCode{}) {
|
||||
if (prot) {
|
||||
if (blockFlags & BlockFlags::FlexibleMemory) {
|
||||
budget->release(BudgetResource::Fmem, size);
|
||||
}
|
||||
if (type != MemoryType::Invalid && !validateMemoryType(type, prot)) {
|
||||
return {{}, ErrorCode::ACCES};
|
||||
}
|
||||
|
||||
if (blockFlags & BlockFlags::DirectMemory) {
|
||||
budget->release(BudgetResource::Dmem, size);
|
||||
}
|
||||
if (auto error = process->invoke([=] {
|
||||
return file->device->map(range, fileOffset, prot, file, process);
|
||||
});
|
||||
error != ErrorCode{}) {
|
||||
if (blockFlags & BlockFlags::FlexibleMemory) {
|
||||
budget->release(BudgetResource::Fmem, size);
|
||||
}
|
||||
|
||||
if (blockFlags & BlockFlags::DirectMemory) {
|
||||
budget->release(BudgetResource::Dmem, size);
|
||||
}
|
||||
|
||||
if (blockFlags & BlockFlags::PooledMemory) {
|
||||
|
|
@ -531,8 +601,8 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
rx::dieIf(errc != std::errc{}, "failed to commit virtual memory {}", errc);
|
||||
}
|
||||
|
||||
// vmemDump(process, rx::format("mapped {:x}-{:x} {}", range.beginAddress(),
|
||||
// range.endAddress(), prot));
|
||||
vmemDump(process, rx::format("mapped {:x}-{:x} {}", range.beginAddress(),
|
||||
range.endAddress(), prot));
|
||||
|
||||
return {range, {}};
|
||||
}
|
||||
|
|
@ -540,9 +610,14 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
|
|||
std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
|
||||
Process *process, std::uint64_t addressHint, rx::AddressRange directRange,
|
||||
rx::EnumBitSet<Protection> prot, rx::EnumBitSet<AllocationFlags> allocFlags,
|
||||
std::string_view name, std::uint64_t alignment, MemoryType type) {
|
||||
std::string_view name, std::uint64_t alignment, std::uint64_t callerAddress,
|
||||
MemoryType type) {
|
||||
ScopedBudgetAcquire dmemResource;
|
||||
|
||||
if (!validateProtection(prot)) {
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
if (prot) {
|
||||
dmemResource = ScopedBudgetAcquire(
|
||||
process->getBudget(), BudgetResource::Dmem, directRange.size());
|
||||
|
|
@ -557,7 +632,13 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
|
|||
}
|
||||
|
||||
VirtualMemoryAllocation allocationInfo;
|
||||
allocationInfo.flags = orbis::vmem::BlockFlags::DirectMemory;
|
||||
if (process->sdkVersion < 0x5500000) {
|
||||
allocationInfo.callerAddress = callerAddress;
|
||||
}
|
||||
if (process->sdkVersion < 0x2000000) {
|
||||
allocFlags |= AllocationFlags::NoMerge;
|
||||
}
|
||||
allocationInfo.flags = BlockFlags::DirectMemory;
|
||||
allocationInfo.flagsEx = BlockFlagsEx::Allocated;
|
||||
allocationInfo.prot = prot;
|
||||
allocationInfo.device = g_context->dmem->device;
|
||||
|
|
@ -566,7 +647,7 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
|
|||
allocationInfo.setName(process, name);
|
||||
|
||||
if (prot) {
|
||||
allocationInfo.flags |= orbis::vmem::BlockFlags::Commited;
|
||||
allocationInfo.flags |= BlockFlags::Commited;
|
||||
}
|
||||
|
||||
bool canOverwrite = (allocFlags & AllocationFlags::Fixed) &&
|
||||
|
|
@ -594,13 +675,14 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
|
|||
|
||||
if (canOverwrite) {
|
||||
if (auto errc = validateOverwrite(vmem, range, false);
|
||||
errc != orbis::ErrorCode{}) {
|
||||
errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
}
|
||||
|
||||
if (auto errc = dmem::map(0, range, directRange.beginAddress(), prot);
|
||||
errc != orbis::ErrorCode{}) {
|
||||
if (auto errc =
|
||||
dmem::map(process, 0, range, directRange.beginAddress(), prot);
|
||||
errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
|
||||
|
|
@ -630,39 +712,40 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
|
|||
|
||||
amdgpu::mapMemory(process->pid, range, type, prot, pmemOffset);
|
||||
|
||||
// vmemDump(process, rx::format("mapped dmem {:x}-{:x}", range.beginAddress(),
|
||||
// range.endAddress()));
|
||||
vmemDump(process, rx::format("mapped dmem {:x}-{:x}", range.beginAddress(),
|
||||
range.endAddress()));
|
||||
|
||||
return {range, {}};
|
||||
}
|
||||
|
||||
std::pair<rx::AddressRange, orbis::ErrorCode>
|
||||
orbis::vmem::mapFlex(Process *process, std::uint64_t size,
|
||||
rx::EnumBitSet<Protection> prot, std::uint64_t addressHint,
|
||||
rx::EnumBitSet<AllocationFlags> allocFlags,
|
||||
rx::EnumBitSet<BlockFlags> blockFlags,
|
||||
std::string_view name, std::uint64_t alignment) {
|
||||
ScopedBudgetAcquire fmemResource;
|
||||
std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFlex(
|
||||
Process *process, std::uint64_t size, rx::EnumBitSet<Protection> prot,
|
||||
std::uint64_t addressHint, rx::EnumBitSet<AllocationFlags> allocFlags,
|
||||
rx::EnumBitSet<BlockFlags> blockFlags, std::string_view name,
|
||||
std::uint64_t alignment, std::uint64_t callerAddress) {
|
||||
|
||||
if (prot) {
|
||||
fmemResource =
|
||||
ScopedBudgetAcquire(process->getBudget(), BudgetResource::Fmem, size);
|
||||
if (!validateProtection(prot)) {
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
if (!fmemResource) {
|
||||
rx::println(stderr,
|
||||
"mapFlex: fmem budget: failed to allocate {:#x} bytes", size);
|
||||
ScopedBudgetAcquire fmemResource(process->getBudget(), BudgetResource::Fmem,
|
||||
size);
|
||||
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
if (!fmemResource) {
|
||||
rx::println(stderr, "mapFlex: fmem budget: failed to allocate {:#x} bytes",
|
||||
size);
|
||||
|
||||
blockFlags |= orbis::vmem::BlockFlags::Commited;
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
bool canOverwrite = (allocFlags & AllocationFlags::Fixed) &&
|
||||
!(allocFlags & AllocationFlags::NoOverwrite);
|
||||
|
||||
VirtualMemoryAllocation allocationInfo;
|
||||
allocationInfo.flags = orbis::vmem::BlockFlags::FlexibleMemory | blockFlags;
|
||||
if (process->sdkVersion < 0x5500000) {
|
||||
allocationInfo.callerAddress = callerAddress;
|
||||
}
|
||||
allocationInfo.flags = blockFlags | BlockFlags::FlexibleMemory;
|
||||
allocationInfo.flagsEx = BlockFlagsEx::Allocated;
|
||||
allocationInfo.prot = prot;
|
||||
allocationInfo.type = MemoryType::WbOnion;
|
||||
|
|
@ -690,40 +773,49 @@ orbis::vmem::mapFlex(Process *process, std::uint64_t size,
|
|||
|
||||
if (canOverwrite) {
|
||||
if (auto errc = validateOverwrite(vmem, vmemRange, false);
|
||||
errc != orbis::ErrorCode{}) {
|
||||
errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
}
|
||||
|
||||
rx::AddressRange flexRange;
|
||||
if (prot) {
|
||||
rx::AddressRange flexRange;
|
||||
{
|
||||
auto [range, errc] = fmem::allocate(size);
|
||||
|
||||
{
|
||||
auto [range, errc] = fmem::allocate(size);
|
||||
if (errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
|
||||
if (errc != orbis::ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
flexRange = range;
|
||||
}
|
||||
|
||||
flexRange = range;
|
||||
}
|
||||
allocationInfo.flags |= BlockFlags::Commited;
|
||||
allocationInfo.deviceOffset = flexRange.beginAddress();
|
||||
|
||||
allocationInfo.deviceOffset = flexRange.beginAddress();
|
||||
|
||||
if (auto errc =
|
||||
pmem::map(vmemRange.beginAddress(), flexRange, toCpuProtection(prot));
|
||||
errc != orbis::ErrorCode{}) {
|
||||
fmem::deallocate(flexRange);
|
||||
return {{}, errc};
|
||||
if (auto errc = process->invoke([=] {
|
||||
return pmem::map(vmemRange.beginAddress(), flexRange,
|
||||
toCpuProtection(prot));
|
||||
});
|
||||
errc != ErrorCode{}) {
|
||||
fmem::deallocate(flexRange);
|
||||
return {{}, errc};
|
||||
}
|
||||
} else {
|
||||
if (auto errc =
|
||||
process->invoke([=] { return rx::mem::protect(vmemRange, {}); });
|
||||
errc != std::errc{}) {
|
||||
return {{}, toErrorCode(errc)};
|
||||
}
|
||||
}
|
||||
|
||||
if (canOverwrite) {
|
||||
release(process, vmem, process->getBudget(), vmemRange);
|
||||
}
|
||||
|
||||
auto [it, _errc, _range] = vmem->map(
|
||||
vmemRange.beginAddress(), vmemRange.size(), allocationInfo,
|
||||
AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge),
|
||||
alignment);
|
||||
vmem->map(vmemRange.beginAddress(), vmemRange.size(), allocationInfo,
|
||||
AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge),
|
||||
alignment);
|
||||
|
||||
// vmemDump(process, rx::format("mapFlex {:x}-{:x}", vmemRange.beginAddress(),
|
||||
// vmemRange.endAddress()));
|
||||
|
|
@ -731,9 +823,78 @@ orbis::vmem::mapFlex(Process *process, std::uint64_t size,
|
|||
return {vmemRange, {}};
|
||||
}
|
||||
|
||||
std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapVoid(
|
||||
Process *process, std::uint64_t size, std::uint64_t addressHint,
|
||||
rx::EnumBitSet<AllocationFlags> allocFlags, std::string_view name,
|
||||
std::uint64_t alignment, std::uint64_t callerAddress) {
|
||||
bool canOverwrite = (allocFlags & AllocationFlags::Fixed) &&
|
||||
!(allocFlags & AllocationFlags::NoOverwrite);
|
||||
|
||||
VirtualMemoryAllocation allocationInfo;
|
||||
if (process->sdkVersion < 0x5500000) {
|
||||
allocationInfo.callerAddress = callerAddress;
|
||||
}
|
||||
allocationInfo.flagsEx = BlockFlagsEx::Void | BlockFlagsEx::Allocated;
|
||||
allocationInfo.type = MemoryType::WbOnion;
|
||||
allocationInfo.setName(process, name);
|
||||
|
||||
auto vmem = process->get(g_vmInstance);
|
||||
std::lock_guard lock(*vmem);
|
||||
|
||||
rx::AddressRange vmemRange;
|
||||
|
||||
{
|
||||
auto [_, errc, range] =
|
||||
vmem->map(addressHint, size, allocationInfo,
|
||||
allocFlags | AllocationFlags::Dry, alignment);
|
||||
if (errc != std::errc{}) {
|
||||
if (errc == std::errc::file_exists) {
|
||||
return {{}, ErrorCode::NOMEM};
|
||||
}
|
||||
|
||||
return {{}, toErrorCode(errc)};
|
||||
}
|
||||
|
||||
vmemRange = range;
|
||||
}
|
||||
|
||||
if (canOverwrite) {
|
||||
if (auto errc = validateOverwrite(vmem, vmemRange, false);
|
||||
errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
}
|
||||
|
||||
if (auto errc =
|
||||
process->invoke([=] { return rx::mem::protect(vmemRange, {}); });
|
||||
errc != std::errc{}) {
|
||||
return {{}, toErrorCode(errc)};
|
||||
}
|
||||
|
||||
if (canOverwrite) {
|
||||
release(process, vmem, process->getBudget(), vmemRange);
|
||||
}
|
||||
|
||||
vmem->map(vmemRange.beginAddress(), vmemRange.size(), allocationInfo,
|
||||
AllocationFlags::Fixed | (allocFlags & AllocationFlags::NoMerge),
|
||||
alignment);
|
||||
|
||||
// vmemDump(process, rx::format("mapVoid {:x}-{:x}", vmemRange.beginAddress(),
|
||||
// vmemRange.endAddress()));
|
||||
return {vmemRange, {}};
|
||||
}
|
||||
|
||||
std::pair<rx::AddressRange, orbis::ErrorCode>
|
||||
orbis::vmem::commitPooled(Process *process, rx::AddressRange range,
|
||||
MemoryType type, rx::EnumBitSet<Protection> prot) {
|
||||
if (!validateProtection(prot)) {
|
||||
return {{}, ErrorCode::INVAL};
|
||||
}
|
||||
|
||||
if (!validateMemoryType(type, prot)) {
|
||||
return {{}, ErrorCode::ACCES};
|
||||
}
|
||||
|
||||
VirtualMemoryAllocation allocationInfo;
|
||||
allocationInfo.flags = BlockFlags::PooledMemory | BlockFlags::Commited;
|
||||
allocationInfo.flagsEx = BlockFlagsEx::Allocated;
|
||||
|
|
@ -755,10 +916,8 @@ orbis::vmem::commitPooled(Process *process, rx::AddressRange range,
|
|||
auto controlBlockIt = it;
|
||||
|
||||
while (controlBlockIt != vmem->end() && controlBlockIt->isAllocated() &&
|
||||
(controlBlockIt->flags & BlockFlags::PooledMemory)) {
|
||||
if (!controlBlockIt->prot &&
|
||||
controlBlockIt->flags == BlockFlags::PooledMemory &&
|
||||
(controlBlockIt->flagsEx & orbis::vmem::BlockFlagsEx::PoolControl)) {
|
||||
controlBlockIt->isPooled()) {
|
||||
if (controlBlockIt->isPoolControl()) {
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -781,7 +940,7 @@ orbis::vmem::commitPooled(Process *process, rx::AddressRange range,
|
|||
}
|
||||
|
||||
if (auto errc = blockpool::commit(process, range, type, prot);
|
||||
errc != orbis::ErrorCode{}) {
|
||||
errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
|
||||
|
|
@ -796,7 +955,7 @@ orbis::vmem::commitPooled(Process *process, rx::AddressRange range,
|
|||
controlAllocationInfo, AllocationFlags::Fixed, kPageSize);
|
||||
} else {
|
||||
if (auto errc = blockpool::commit(process, range, type, prot);
|
||||
errc != orbis::ErrorCode{}) {
|
||||
errc != ErrorCode{}) {
|
||||
return {{}, errc};
|
||||
}
|
||||
}
|
||||
|
|
@ -887,68 +1046,148 @@ orbis::ErrorCode orbis::vmem::decommitPooled(Process *process,
|
|||
|
||||
orbis::ErrorCode orbis::vmem::protect(Process *process, rx::AddressRange range,
|
||||
rx::EnumBitSet<Protection> prot) {
|
||||
prot &= kProtCpuAll | kProtGpuAll;
|
||||
|
||||
if (!validateProtection(prot)) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
auto vmem = process->get(g_vmInstance);
|
||||
|
||||
range = rx::AddressRange::fromBeginEnd(
|
||||
rx::alignDown(range.beginAddress(), kPageSize),
|
||||
rx::alignUp(range.endAddress(), kPageSize));
|
||||
|
||||
{
|
||||
std::lock_guard lock(*vmem);
|
||||
auto it = vmem->query(range.beginAddress());
|
||||
std::size_t fmemSize = 0;
|
||||
auto it = vmem->lowerBound(range.beginAddress());
|
||||
|
||||
if (it == vmem->end()) {
|
||||
rx::println(stderr,
|
||||
"vmem: attempt to set protection of invalid address range: "
|
||||
"{:x}-{:x}",
|
||||
range.beginAddress(), range.endAddress());
|
||||
return orbis::ErrorCode::INVAL;
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
auto errc = validateRange(vmem, it, range,
|
||||
[](const VirtualMemoryAllocation &alloc) {
|
||||
if (alloc.flags == BlockFlags::PooledMemory) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
auto errc =
|
||||
validateRange(vmem, it, range,
|
||||
[&fmemSize, prot](const VirtualMemoryAllocation &alloc,
|
||||
rx::AddressRange range) {
|
||||
if (alloc.isPoolReserved()) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
if (alloc.flagsEx & BlockFlagsEx::Reserved) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
if (!validateMemoryType(alloc.type, prot)) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
return ErrorCode{};
|
||||
});
|
||||
if (alloc.isFlex()) {
|
||||
if ((prot && !alloc.prot) || (!prot && alloc.prot)) {
|
||||
fmemSize += range.size();
|
||||
}
|
||||
}
|
||||
|
||||
return ErrorCode{};
|
||||
});
|
||||
|
||||
if (errc != ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
if (auto errc =
|
||||
toErrorCode(rx::mem::protect(range, vmem::toCpuProtection(prot)));
|
||||
errc != ErrorCode{}) {
|
||||
return errc;
|
||||
if (fmemSize && prot) {
|
||||
if (!process->getBudget()->acquire(BudgetResource::Fmem, fmemSize)) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
}
|
||||
|
||||
modifyRange(vmem, it, range,
|
||||
[prot](VirtualMemoryAllocation &alloc, rx::AddressRange) {
|
||||
if (!alloc.isAllocated()) {
|
||||
return;
|
||||
}
|
||||
rx::EnumBitSet<AllocationFlags> modifyFlags{};
|
||||
|
||||
if (alloc.device != nullptr &&
|
||||
alloc.flags & BlockFlags::FlexibleMemory) {
|
||||
alloc.prot = prot & ~Protection::CpuExec;
|
||||
}
|
||||
modifyRange(
|
||||
vmem, it, range, modifyFlags,
|
||||
[process, prot, sdkVersion = process->sdkVersion](
|
||||
VirtualMemoryAllocation &alloc, rx::AddressRange range) {
|
||||
if (!alloc.isAllocated()) {
|
||||
return;
|
||||
}
|
||||
|
||||
alloc.flags = alloc.prot
|
||||
? alloc.flags | BlockFlags::Commited
|
||||
: alloc.flags & ~BlockFlags::Commited;
|
||||
});
|
||||
if (alloc.flagsEx & BlockFlagsEx::Reserved) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto blockProt = prot;
|
||||
|
||||
if (alloc.type == MemoryType::WbGarlic) {
|
||||
blockProt &= ~(Protection::CpuWrite | Protection::GpuWrite);
|
||||
}
|
||||
|
||||
if (!alloc.isVoid()) {
|
||||
if (alloc.isFlex() && alloc.device == nullptr) {
|
||||
if (blockProt && !alloc.prot) {
|
||||
auto [pmemRange, errc] = fmem::allocate(range.size());
|
||||
rx::dieIf(errc != ErrorCode{},
|
||||
"failed to allocate flexible memory");
|
||||
|
||||
errc = pmem::map(range.beginAddress(), pmemRange,
|
||||
toCpuProtection(blockProt));
|
||||
|
||||
rx::dieIf(errc != ErrorCode{}, "failed to map flexible memory");
|
||||
} else if (!blockProt && alloc.prot) {
|
||||
auto errc = fmem::deallocate(rx::AddressRange::fromBeginSize(
|
||||
alloc.deviceOffset, range.size()));
|
||||
|
||||
rx::dieIf(errc != ErrorCode{},
|
||||
"failed to deallocate flexible memory {:x}-{:x}",
|
||||
alloc.deviceOffset,
|
||||
alloc.deviceOffset + range.size());
|
||||
}
|
||||
}
|
||||
|
||||
if (sdkVersion > 0x1500000) {
|
||||
if (alloc.isDirect() || alloc.isPooled() ||
|
||||
(alloc.isFlex() && alloc.device != nullptr)) {
|
||||
blockProt &= ~Protection::CpuExec;
|
||||
}
|
||||
}
|
||||
|
||||
auto cpuBlockProt = toCpuProtection(blockProt);
|
||||
|
||||
if (cpuBlockProt != toCpuProtection(alloc.prot)) {
|
||||
auto errc = process->invoke(
|
||||
[=] { return rx::mem::protect(range, cpuBlockProt); });
|
||||
|
||||
rx::dieIf(errc != std::errc{},
|
||||
"failed to protect region {:x}-{:x}, prot {}, error {}",
|
||||
range.beginAddress(), range.endAddress(), cpuBlockProt,
|
||||
errc);
|
||||
}
|
||||
} else {
|
||||
blockProt = {};
|
||||
}
|
||||
|
||||
if (alloc.isDirect() || alloc.isPooled() ||
|
||||
(alloc.isFlex() && alloc.device != nullptr)) {
|
||||
blockProt &= ~Protection::CpuExec;
|
||||
}
|
||||
|
||||
if (alloc.isVoid() && sdkVersion <= 0x1500000) {
|
||||
alloc.prot = prot;
|
||||
} else {
|
||||
alloc.prot = blockProt;
|
||||
}
|
||||
alloc.flags = blockProt ? alloc.flags | BlockFlags::Commited
|
||||
: alloc.flags & ~BlockFlags::Commited;
|
||||
});
|
||||
if (fmemSize && !prot) {
|
||||
process->getBudget()->release(BudgetResource::Fmem, fmemSize);
|
||||
}
|
||||
}
|
||||
|
||||
amdgpu::protectMemory(process->pid, range, prot);
|
||||
|
||||
// vmemDump(process, rx::format("protected {:x}-{:x} {}",
|
||||
// range.beginAddress(),
|
||||
// range.endAddress(), prot));
|
||||
vmemDump(process, rx::format("protected {:x}-{:x} {}", range.beginAddress(),
|
||||
range.endAddress(), prot));
|
||||
|
||||
return {};
|
||||
}
|
||||
|
|
@ -965,10 +1204,16 @@ orbis::ErrorCode orbis::vmem::setName(Process *process, rx::AddressRange range,
|
|||
"vmem: attempt to set name of invalid address range: "
|
||||
"{:x}-{:x}, name: {}",
|
||||
range.beginAddress(), range.endAddress(), name);
|
||||
return orbis::ErrorCode::INVAL;
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
modifyRange(vmem, it, range,
|
||||
rx::EnumBitSet<AllocationFlags> modifyFlags{};
|
||||
|
||||
if (process->sdkVersion <= 0x1500000) {
|
||||
modifyFlags |= AllocationFlags::NoMerge;
|
||||
}
|
||||
|
||||
modifyRange(vmem, it, range, modifyFlags,
|
||||
[name](VirtualMemoryAllocation &alloc, rx::AddressRange) {
|
||||
if (alloc.isAllocated()) {
|
||||
alloc.name = name;
|
||||
|
|
@ -978,93 +1223,84 @@ orbis::ErrorCode orbis::vmem::setName(Process *process, rx::AddressRange range,
|
|||
return {};
|
||||
}
|
||||
|
||||
orbis::ErrorCode orbis::vmem::setType(Process *process, rx::AddressRange range,
|
||||
MemoryType type) {
|
||||
orbis::ErrorCode
|
||||
orbis::vmem::setTypeAndProtect(Process *process, rx::AddressRange range,
|
||||
MemoryType type,
|
||||
rx::EnumBitSet<Protection> prot) {
|
||||
if (!validateProtection(prot)) {
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
if (!validateMemoryType(type, prot)) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
prot &= ~Protection::CpuExec;
|
||||
|
||||
auto vmem = process->get(g_vmInstance);
|
||||
|
||||
std::lock_guard lock(*vmem);
|
||||
auto it = vmem->query(range.beginAddress());
|
||||
|
||||
if (it == vmem->end()) {
|
||||
return orbis::ErrorCode::INVAL;
|
||||
return ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
auto errc =
|
||||
validateRange(vmem, it, range,
|
||||
[](const VirtualMemoryAllocation &alloc, rx::AddressRange) {
|
||||
if (alloc.flags == BlockFlags::PooledMemory) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
if (alloc.flagsEx & BlockFlagsEx::Reserved) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
if (!(alloc.flags & (BlockFlags::PooledMemory |
|
||||
BlockFlags::DirectMemory))) {
|
||||
return ErrorCode::OPNOTSUPP;
|
||||
}
|
||||
|
||||
return ErrorCode{};
|
||||
});
|
||||
|
||||
if (errc != ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
if (auto errc = process->invoke([=] {
|
||||
return toErrorCode(rx::mem::protect(range, toCpuProtection(prot)));
|
||||
});
|
||||
errc != ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
rx::EnumBitSet<AllocationFlags> modifyFlags{};
|
||||
|
||||
if (process->sdkVersion < 0x1700000) {
|
||||
modifyFlags |= AllocationFlags::NoMerge;
|
||||
}
|
||||
|
||||
modifyRange(
|
||||
vmem, it, range,
|
||||
[type](VirtualMemoryAllocation &alloc, rx::AddressRange range) {
|
||||
vmem, it, range, modifyFlags,
|
||||
[type, prot](VirtualMemoryAllocation &alloc, rx::AddressRange range) {
|
||||
if (!alloc.isAllocated()) {
|
||||
return;
|
||||
}
|
||||
|
||||
alloc.type = type;
|
||||
alloc.prot = prot;
|
||||
alloc.flags = prot ? alloc.flags | BlockFlags::Commited
|
||||
: alloc.flags & ~BlockFlags::Commited;
|
||||
|
||||
if (alloc.flags & BlockFlags::DirectMemory) {
|
||||
dmem::setType(
|
||||
0,
|
||||
rx::AddressRange::fromBeginSize(alloc.deviceOffset, range.size()),
|
||||
type);
|
||||
alloc.type = type;
|
||||
return;
|
||||
}
|
||||
|
||||
if (alloc.flags != (BlockFlags::PooledMemory | BlockFlags::Commited)) {
|
||||
alloc.type = type;
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
orbis::ErrorCode
|
||||
orbis::vmem::setTypeAndProtect(Process *process, rx::AddressRange range,
|
||||
MemoryType type,
|
||||
rx::EnumBitSet<Protection> prot) {
|
||||
auto vmem = process->get(g_vmInstance);
|
||||
|
||||
std::lock_guard lock(*vmem);
|
||||
auto it = vmem->query(range.beginAddress());
|
||||
|
||||
if (it == vmem->end()) {
|
||||
return orbis::ErrorCode::INVAL;
|
||||
}
|
||||
|
||||
auto errc =
|
||||
validateRange(vmem, it, range, [](const VirtualMemoryAllocation &alloc) {
|
||||
if (alloc.flags == BlockFlags::PooledMemory) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
if (alloc.flagsEx & BlockFlagsEx::Reserved) {
|
||||
return ErrorCode::ACCES;
|
||||
}
|
||||
|
||||
return ErrorCode{};
|
||||
});
|
||||
|
||||
if (errc != ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
if (auto errc =
|
||||
toErrorCode(rx::mem::protect(range, vmem::toCpuProtection(prot)));
|
||||
errc != ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
modifyRange(
|
||||
vmem, it, range,
|
||||
[type, prot](VirtualMemoryAllocation &alloc, rx::AddressRange range) {
|
||||
if (alloc.isAllocated()) {
|
||||
alloc.type = type;
|
||||
alloc.prot = prot;
|
||||
|
||||
if (alloc.flags & BlockFlags::DirectMemory) {
|
||||
dmem::setType(0,
|
||||
rx::AddressRange::fromBeginSize(alloc.deviceOffset,
|
||||
range.size()),
|
||||
type);
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
amdgpu::protectMemory(process->pid, range, prot);
|
||||
|
|
@ -1080,12 +1316,11 @@ orbis::ErrorCode orbis::vmem::unmap(Process *process, rx::AddressRange range) {
|
|||
rx::alignDown(range.beginAddress(), kPageSize),
|
||||
rx::alignUp(range.endAddress(), kPageSize));
|
||||
|
||||
orbis::ErrorCode result;
|
||||
ErrorCode result;
|
||||
{
|
||||
std::lock_guard lock(*vmem);
|
||||
|
||||
if (auto errc = validateOverwrite(vmem, range, true);
|
||||
errc != orbis::ErrorCode{}) {
|
||||
if (auto errc = validateOverwrite(vmem, range, true); errc != ErrorCode{}) {
|
||||
return errc;
|
||||
}
|
||||
|
||||
|
|
@ -1098,7 +1333,7 @@ orbis::ErrorCode orbis::vmem::unmap(Process *process, rx::AddressRange range) {
|
|||
result = toErrorCode(errc);
|
||||
}
|
||||
|
||||
rx::mem::release(range, kPageSize);
|
||||
process->invoke([=] { rx::mem::release(range, kPageSize); });
|
||||
amdgpu::unmapMemory(process->pid, range);
|
||||
|
||||
// vmemDump(process, rx::format("unmap {:x}-{:x}", range.beginAddress(),
|
||||
|
|
@ -1134,7 +1369,7 @@ orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) {
|
|||
return {};
|
||||
}
|
||||
|
||||
orbis::vmem::QueryResult result{};
|
||||
QueryResult result{};
|
||||
result.start = it.beginAddress();
|
||||
result.end = it.endAddress();
|
||||
|
||||
|
|
@ -1148,7 +1383,9 @@ orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) {
|
|||
result.memoryType = it->type;
|
||||
}
|
||||
|
||||
result.protection = it->prot;
|
||||
if (!it->isVoid()) {
|
||||
result.protection = it->prot;
|
||||
}
|
||||
result.flags = it->flags;
|
||||
result.name = it->name;
|
||||
|
||||
|
|
@ -1184,7 +1421,7 @@ orbis::vmem::queryProtection(Process *process, std::uint64_t address,
|
|||
return {};
|
||||
}
|
||||
|
||||
orbis::vmem::MemoryProtection result{};
|
||||
MemoryProtection result{};
|
||||
result.startAddress = it.beginAddress();
|
||||
result.endAddress = it.endAddress();
|
||||
result.prot = it->prot;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue