diff --git a/rpcsx-os/linker.cpp b/rpcsx-os/linker.cpp
index 10153d34c..15dbff46c 100644
--- a/rpcsx-os/linker.cpp
+++ b/rpcsx-os/linker.cpp
@@ -865,7 +865,7 @@ Ref<orbis::Module> rx::linker::loadModule(std::span<std::byte> image,
         phdr.p_flags |= vm::kMapProtCpuWrite; // TODO: reprotect on relocations
       }
 
-      vm::protect(imageBase + segmentBegin, segmentSize, phdr.p_flags);
+      vm::protect(imageBase + segmentBegin, segmentSize, phdr.p_flags & (vm::kMapProtCpuAll | vm::kMapProtGpuAll));
 
       if (phdr.p_type == kElfProgramTypeLoad) {
         if (result->segmentCount >= std::size(result->segments)) {