Added System-V ABI support in x64 backend & fixed mmap usage

src/xenia/base/mapped_memory_posix.cc

@@ -56,6 +56,9 @@ class PosixMappedMemory : public MappedMemory {
       map_length = size_t(file_stat.st_size);
     }
 
+    // Ensure that the file is large enough.
+    ftruncate(file_descriptor, map_length);
+
     void* data =
         mmap(0, map_length, protection, MAP_SHARED, file_descriptor, offset);
     if (!data) {

src/xenia/base/memory_posix.cc

@@ -85,8 +85,11 @@ void* AllocFixed(void* base_address, size_t length,
                  AllocationType allocation_type, PageAccess access) {
   // mmap does not support reserve / commit, so ignore allocation_type.
   uint32_t prot = ToPosixProtectFlags(access);
-  void* result = mmap(base_address, length, prot,
+  int flags = MAP_PRIVATE | MAP_ANONYMOUS;
-                      MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0);
+  if (base_address != nullptr) {
+    flags |= MAP_FIXED;
+  }
+  void* result = mmap(base_address, length, prot, flags, -1, 0);
   if (result == MAP_FAILED) {
     return nullptr;
   } else {

src/xenia/cpu/backend/x64/x64_backend.cc

@@ -418,6 +418,7 @@ X64ThunkEmitter::X64ThunkEmitter(X64Backend* backend, XbyakAllocator* allocator)
 X64ThunkEmitter::~X64ThunkEmitter() {}
 
 HostToGuestThunk X64ThunkEmitter::EmitHostToGuestThunk() {
+#if XE_PLATFORM_WIN32
   // rcx = target
   // rdx = arg0 (context)
   // r8 = arg1 (guest return address)
@@ -460,6 +461,53 @@ HostToGuestThunk X64ThunkEmitter::EmitHostToGuestThunk() {
   mov(rdx, qword[rsp + 8 * 2]);
   mov(r8, qword[rsp + 8 * 3]);
   ret();
+#elif XE_PLATFORM_LINUX || XE_PLATFORM_MAC
+  // System-V ABI args:
+  // rdi = target
+  // rsi = arg0 (context)
+  // rdx = arg1 (guest return address)
+
+  struct _code_offsets {
+    size_t prolog;
+    size_t prolog_stack_alloc;
+    size_t body;
+    size_t epilog;
+    size_t tail;
+  } code_offsets = {};
+
+  const size_t stack_size = StackLayout::THUNK_STACK_SIZE;
+
+  code_offsets.prolog = getSize();
+
+  // rsp + 0 = return address
+  mov(qword[rsp + 8 * 3], rdx);
+  mov(qword[rsp + 8 * 2], rsi);
+  mov(qword[rsp + 8 * 1], rdi);
+  sub(rsp, stack_size);
+
+  code_offsets.prolog_stack_alloc = getSize();
+  code_offsets.body = getSize();
+
+  // Save nonvolatile registers.
+  EmitSaveNonvolatileRegs();
+
+  mov(rax, rdi);
+  // mov(rsi, rsi);   // context
+  mov(rcx, rdx);  // return address
+  call(rax);
+
+  EmitLoadNonvolatileRegs();
+
+  code_offsets.epilog = getSize();
+
+  add(rsp, stack_size);
+  mov(rdi, qword[rsp + 8 * 1]);
+  mov(rsi, qword[rsp + 8 * 2]);
+  mov(rdx, qword[rsp + 8 * 3]);
+  ret();
+#else
+  assert_always("Unknown platform ABI in host to guest thunk!");
+#endif
 
   code_offsets.tail = getSize();
 
@@ -479,6 +527,7 @@ HostToGuestThunk X64ThunkEmitter::EmitHostToGuestThunk() {
 }
 
 GuestToHostThunk X64ThunkEmitter::EmitGuestToHostThunk() {
+#if XE_PLATFORM_WIN32
   // rcx = target function
   // rdx = arg0
   // r8  = arg1
@@ -515,6 +564,57 @@ GuestToHostThunk X64ThunkEmitter::EmitGuestToHostThunk() {
 
   add(rsp, stack_size);
   ret();
+#elif XE_PLATFORM_LINUX || XE_PLATFORM_MAC
+  // This function is being called using the Microsoft ABI from CallNative
+  // rcx = target function
+  // rdx = arg0
+  // r8  = arg1
+  // r9  = arg2
+
+  // Must be translated to System-V ABI:
+  // rdi = target function
+  // rsi = arg0
+  // rdx = arg1
+  // rcx = arg2
+  // r8, r9 - unused argument registers
+
+  struct _code_offsets {
+    size_t prolog;
+    size_t prolog_stack_alloc;
+    size_t body;
+    size_t epilog;
+    size_t tail;
+  } code_offsets = {};
+
+  const size_t stack_size = StackLayout::THUNK_STACK_SIZE;
+
+  code_offsets.prolog = getSize();
+
+  // rsp + 0 = return address
+  sub(rsp, stack_size);
+
+  code_offsets.prolog_stack_alloc = getSize();
+  code_offsets.body = getSize();
+
+  // Save off volatile registers.
+  EmitSaveVolatileRegs();
+
+  mov(rax, rcx);              // function
+  mov(rdi, GetContextReg());  // context
+  mov(rsi, rdx);              // arg0
+  mov(rdx, r8);               // arg1
+  mov(rcx, r9);               // arg2
+  call(rax);
+
+  EmitLoadVolatileRegs();
+
+  code_offsets.epilog = getSize();
+
+  add(rsp, stack_size);
+  ret();
+#else
+  assert_always("Unknown platform ABI in guest to host thunk!")
+#endif
 
   code_offsets.tail = getSize();
 
@@ -537,6 +637,7 @@ GuestToHostThunk X64ThunkEmitter::EmitGuestToHostThunk() {
 uint64_t ResolveFunction(void* raw_context, uint64_t target_address);
 
 ResolveFunctionThunk X64ThunkEmitter::EmitResolveFunctionThunk() {
+#if XE_PLATFORM_WIN32
   // ebx = target PPC address
   // rcx = context
 
@@ -572,6 +673,49 @@ ResolveFunctionThunk X64ThunkEmitter::EmitResolveFunctionThunk() {
 
   add(rsp, stack_size);
   jmp(rax);
+#elif XE_PLATFORM_LINUX || XE_PLATFORM_MAC
+  // Function is called with the following params:
+  // ebx = target PPC address
+  // rsi = context
+
+  // System-V ABI args:
+  // rdi = context
+  // rsi = target PPC address
+
+  struct _code_offsets {
+    size_t prolog;
+    size_t prolog_stack_alloc;
+    size_t body;
+    size_t epilog;
+    size_t tail;
+  } code_offsets = {};
+
+  const size_t stack_size = StackLayout::THUNK_STACK_SIZE;
+
+  code_offsets.prolog = getSize();
+
+  // rsp + 0 = return address
+  sub(rsp, stack_size);
+
+  code_offsets.prolog_stack_alloc = getSize();
+  code_offsets.body = getSize();
+
+  // Save volatile registers
+  EmitSaveVolatileRegs();
+  mov(rdi, rsi);  // context
+  mov(rsi, rbx);  // target PPC address
+  mov(rax, reinterpret_cast<uint64_t>(&ResolveFunction));
+  call(rax);
+
+  EmitLoadVolatileRegs();
+
+  code_offsets.epilog = getSize();
+
+  add(rsp, stack_size);
+  jmp(rax);
+#else
+  assert_always("Unknown platform ABI in resolve function!");
+#endif
 
   code_offsets.tail = getSize();