orbis: fix sys_batch_map & extend virtual memory range

tweaks for flags validation

kernel/orbis/src/KernelAllocator.cpp

@@ -8,7 +8,7 @@
 #include "rx/print.hpp"
 
 static const std::uint64_t g_allocProtWord = 0xDEADBEAFBADCAFE1;
-static constexpr std::uintptr_t kHeapBaseAddress = 0x00000600'0000'0000;
+static constexpr std::uintptr_t kHeapBaseAddress = 0x7100'0000'0000;
 static constexpr auto kHeapSize = 0x1'0000'0000;
 static constexpr int kDebugHeap = 0;
 

kernel/orbis/src/dmem.cpp

@@ -204,7 +204,7 @@ orbis::dmem::allocate(unsigned dmemIndex, rx::AddressRange searchRange,
   auto allocResult = dmem->map(searchRange.beginAddress(), len, allocation,
                                AllocationFlags::Dry, alignment);
   if (allocResult.errc != std::errc{}) {
-    return {{}, toErrorCode(allocResult.errc)};
+    return {{}, ErrorCode::AGAIN};
   }
 
   auto result = allocResult.range.beginAddress();

kernel/orbis/src/sys/sys_sce.cpp

@@ -452,6 +452,7 @@ orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk,
           static_cast<MemoryType>(_entry.type),
           rx::EnumBitSet<vmem::Protection>::fromUnderlying(_entry.protection),
           flags, _entry.offset);
+      break;
     case 1:
       result = sys_munmap(thread, _entry.start, _entry.length);
       break;
@@ -913,7 +914,7 @@ orbis::SysResult orbis::sys_virtual_query(Thread *thread, uintptr_t addr,
 
   auto result = vmem::query(thread->tproc, addr, flags & 1);
 
-  if (!result || result->start >= 0x800000000) {
+  if (!result) {
     return ErrorCode::ACCES;
   }
 
@@ -1528,6 +1529,10 @@ orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr,
            vmem::Protection::GpuRead | vmem::Protection::GpuWrite;
   }
 
+  if (prot & vmem::Protection::CpuExec) {
+    return ErrorCode::INVAL;
+  }
+
   if (flags & vmem::MapFlags::Fixed) {
     allocFlags = AllocationFlags::Fixed;
   } else if (addr == 0) {
@@ -1538,8 +1543,7 @@ orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr,
     allocFlags |= AllocationFlags::NoOverwrite;
   }
 
-  auto name =
-      callerAddress ? rx::format("anon:{:012x}", callerAddress) : "<dmem unk>";
+  auto name = callerAddress ? rx::format("anon:{:012x}", callerAddress) : "";
 
   auto [range, errc] =
       vmem::mapDirect(thread->tproc, addr,

kernel/orbis/src/sys/sys_vm_mmap.cpp

@@ -2,7 +2,6 @@
 #include "error.hpp"
 #include "rx/AddressRange.hpp"
 #include "rx/align.hpp"
-#include "rx/debug.hpp"
 #include "rx/format.hpp"
 #include "rx/print.hpp"
 #include "sys/sysproto.hpp"
@@ -27,7 +26,16 @@ orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len,
   std::uint64_t callerAddress = getCallerAddress(thread);
 
   auto shift = addr & (vmem::kPageSize - 1);
-  addr = rx::alignDown(addr, vmem::kPageSize);
+
+  if ((flags & vmem::MapFlags::Fixed) && shift != 0) {
+    return ErrorCode::INVAL;
+  }
+
+  if (len == 0) {
+    return ErrorCode::INVAL;
+  }
+
+  addr = rx::alignUp(addr, vmem::kPageSize);
   rx::EnumBitSet<AllocationFlags> allocFlags{};
   rx::EnumBitSet<vmem::BlockFlags> blockFlags{};
   rx::EnumBitSet<vmem::BlockFlagsEx> blockFlagsEx{};
@@ -93,6 +101,8 @@ orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len,
         (vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) {
       return ErrorCode::INVAL;
     }
+  } else {
+    shift = 0;
   }
 
   auto name = callerAddress ? rx::format("anon:{:012x}", callerAddress) : "";
@@ -102,10 +112,6 @@ orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len,
       return ErrorCode::INVAL;
     }
 
-    if (prot & (vmem::Protection::GpuRead | vmem::Protection::GpuWrite)) {
-      return ErrorCode::INVAL;
-    }
-
     auto [range, errc] = vmem::mapFlex(thread->tproc, len, prot, addr,
                                        allocFlags, blockFlags, name, alignment);
 
@@ -126,8 +132,6 @@ orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len,
     blockFlags |= vmem::BlockFlags::FlexibleMemory;
   }
 
-  prot &= ~vmem::Protection::CpuExec;
-
   auto [range, errc] =
       vmem::mapFile(thread->tproc, addr, len, allocFlags, prot, blockFlags,
                     blockFlagsEx, file.get(), pos, name, alignment);
@@ -153,6 +157,10 @@ orbis::SysResult orbis::sys_msync(Thread *thread, uintptr_t addr, size_t len,
   return {};
 }
 orbis::SysResult orbis::sys_munmap(Thread *thread, uintptr_t addr, size_t len) {
+  if (len == 0) {
+    return ErrorCode::INVAL;
+  }
+
   auto range = rx::AddressRange::fromBeginSize(addr, len);
 
   return vmem::unmap(thread->tproc, range);

kernel/orbis/src/vmem.cpp

@@ -290,7 +290,7 @@ void orbis::vmem::initialize(Process *process, bool force) {
   std::lock_guard lock(*vmem);
 
   // FIXME: for PS5 should be extended range
-  auto range = rx::AddressRange::fromBeginEnd(0x400000, 0x10000000000);
+  auto range = rx::AddressRange::fromBeginEnd(0x400000, 0x7000'0000'0000);
   vmem->create(range);
 
   std::size_t address = range.beginAddress();
@@ -373,11 +373,21 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
   }
 
   if (blockFlags & (BlockFlags::DirectMemory | BlockFlags::PooledMemory)) {
+    if (prot & vmem::Protection::CpuExec) {
+      return {{}, ErrorCode::ACCES};
+    }
+
     if (alignment < dmem::kPageSize) {
       alignment = dmem::kPageSize;
     }
   }
 
+  if (allocFlags & AllocationFlags::Fixed) {
+    if (addressHint % alignment) {
+      return {{}, orbis::ErrorCode::INVAL};
+    }
+  }
+
   bool canOverwrite = (allocFlags & AllocationFlags::Fixed) &&
                       !(allocFlags & AllocationFlags::NoOverwrite);
 
@@ -401,6 +411,10 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
       return {{}, ErrorCode::INVAL};
     }
 
+    if (errc == std::errc::file_exists) {
+      return {{}, ErrorCode::NOMEM};
+    }
+
     return {{}, toErrorCode(errc)};
   }
 
@@ -503,8 +517,8 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapFile(
     rx::dieIf(errc != std::errc{}, "failed to commit virtual memory {}", errc);
   }
 
-  // vmemDump(process, rx::format("mapped {:x}-{:x}", range.beginAddress(),
-  //                              range.endAddress()));
+  // vmemDump(process, rx::format("mapped {:x}-{:x} {}", range.beginAddress(),
+  //                              range.endAddress(), prot));
 
   return {range, {}};
 }
@@ -545,6 +559,10 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
         vmem->map(addressHint, directRange.size(), allocationInfo,
                   allocFlags | AllocationFlags::Dry, alignment);
     if (errc != std::errc{}) {
+      if (errc == std::errc::file_exists) {
+        return {{}, ErrorCode::NOMEM};
+      }
+
       return {{}, toErrorCode(errc)};
     }
 
@@ -589,9 +607,8 @@ std::pair<rx::AddressRange, orbis::ErrorCode> orbis::vmem::mapDirect(
 
   amdgpu::mapMemory(process->pid, range, type, prot, pmemOffset);
 
-  // vmemDump(process,
-  //          rx::format("mapped dmem {:x}-{:x}", range.beginAddress(),
-  //                     range.endAddress()));
+  vmemDump(process, rx::format("mapped dmem {:x}-{:x}", range.beginAddress(),
+                               range.endAddress()));
 
   return {range, {}};
 }
@@ -615,6 +632,7 @@ orbis::vmem::mapFlex(Process *process, std::uint64_t size,
   allocationInfo.flags = orbis::vmem::BlockFlags::FlexibleMemory | blockFlags;
   allocationInfo.flagsEx = BlockFlagsEx::Allocated;
   allocationInfo.prot = prot;
+  allocationInfo.type = MemoryType::WbOnion;
   allocationInfo.setName(process, name);
 
   if (prot) {
@@ -631,6 +649,10 @@ orbis::vmem::mapFlex(Process *process, std::uint64_t size,
         vmem->map(addressHint, size, allocationInfo,
                   allocFlags | AllocationFlags::Dry, alignment);
     if (errc != std::errc{}) {
+      if (errc == std::errc::file_exists) {
+        return {{}, ErrorCode::NOMEM};
+      }
+
       return {{}, toErrorCode(errc)};
     }
 
@@ -1063,8 +1085,15 @@ orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) {
 
   auto it = vmem->lowerBound(address);
 
+  constexpr auto restrictedArea =
+      rx::AddressRange::fromBeginSize(0x800000000, 0x100000000);
+
   if (lowerBound) {
-    while (it != vmem->end() && !it->isAllocated()) {
+    while (it != vmem->end()) {
+      if (it->isAllocated() && !restrictedArea.intersects(it.range())) {
+        break;
+      }
+
       ++it;
     }
 
@@ -1072,7 +1101,7 @@ orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) {
       return {};
     }
   } else if (it == vmem->end() || !it.range().contains(address) ||
-             !it->isAllocated()) {
+             !it->isAllocated() || restrictedArea.intersects(it.range())) {
     return {};
   }
 
@@ -1085,9 +1114,7 @@ orbis::vmem::query(Process *process, std::uint64_t address, bool lowerBound) {
   }
 
   if (it->flags & BlockFlags::DirectMemory) {
-    // if (auto queryResult = dmem::query(0, it->deviceOffset)) {
     result.memoryType = it->type;
-    // }
   } else if (it->flags == (BlockFlags::PooledMemory | BlockFlags::Commited)) {
     result.memoryType = it->type;
   }

rpcsx/CMakeLists.txt

@@ -80,7 +80,7 @@ if(LINUX AND WITH_PS4)
     ipmi.cpp
   )
 
-  target_base_address(rpcsx 0x0000070000000000)
+  target_base_address(rpcsx 0x00000700000000000)
   target_compile_options(rpcsx PRIVATE "-mfsgsbase")
   set_target_properties(rpcsx PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
   install(TARGETS rpcsx RUNTIME DESTINATION bin)

rpcsx/gpu/Device.cpp

@@ -68,11 +68,11 @@ static vk::Context createVkContext(Device *device) {
   bool enableValidation = rx::g_config.validateGpu;
 
   for (std::size_t process = 0; process < 6; ++process) {
-    if (auto errc = rx::mem::reserve(rx::AddressRange::fromBeginSize(
-            orbis::kMinAddress + orbis::kMaxAddress * process,
-            orbis::kMaxAddress - orbis::kMinAddress));
-        errc != std::errc{}) {
-      rx::die("failed to reserve userspace memory: {}", (int)errc);
+    auto range = rx::AddressRange::fromBeginSize(
+        0x40'0000 + 0x100'0000'0000 * process, 0x100'0000'0000 - 0x40'0000);
+    if (auto errc = rx::mem::reserve(range); errc != std::errc{}) {
+      rx::die("failed to reserve userspace memory: {} {:x}-{:x}", (int)errc,
+              range.beginAddress(), range.endAddress());
     }
   }
 
@@ -585,6 +585,10 @@ void Device::unmapProcess(std::uint32_t pid) {
 void Device::protectMemory(std::uint32_t pid, std::uint64_t address,
                            std::uint64_t size,
                            rx::EnumBitSet<orbis::vmem::Protection> prot) {
+  if (address + size > 0x100'0000'0000) {
+    return;
+  }
+
   auto &process = processInfo[pid];
 
   auto vmSlotIt = process.vmTable.queryArea(address);
@@ -957,6 +961,10 @@ void Device::mapMemory(std::uint32_t pid, rx::AddressRange virtualRange,
                        orbis::MemoryType memoryType,
                        rx::EnumBitSet<orbis::vmem::Protection> prot,
                        std::uint64_t physicalOffset) {
+  if (virtualRange.endAddress() > 0x100'0000'0000) {
+    return;
+  }
+
   auto &process = processInfo[pid];
 
   process.vmTable.map(virtualRange,