orbis: fix dmem::release

add validation in debug build

kernel/orbis/src/blockpool.cpp

@@ -46,6 +46,26 @@ struct PooledMemoryResource {
     reservedPages.clear();
   }
 
+  std::size_t calcUsed() const {
+    std::size_t result = 0;
+
+    for (auto block : usedBlocks) {
+      result += block.size();
+    }
+
+    return result + reservedPages.size() * orbis::dmem::kPageSize;
+  }
+
+  std::size_t calcFree() const {
+    std::size_t result = 0;
+
+    for (auto block : freeBlocks) {
+      result += block.size();
+    }
+
+    return result;
+  }
+
   void addFreeBlock(rx::AddressRange dmemRange) {
     if (freeBlocks.empty()) {
       freeBlocks.push_back(dmemRange);
@@ -64,8 +84,8 @@ struct PooledMemoryResource {
 
     if (it != freeBlocks.end() &&
         dmemRange.endAddress() == it->beginAddress()) {
-      *it = rx::AddressRange::fromBeginEnd(it->beginAddress(),
-                                           dmemRange.endAddress());
+      *it = rx::AddressRange::fromBeginEnd(dmemRange.beginAddress(),
+                                           it->endAddress());
       return;
     }
 
@@ -86,15 +106,8 @@ struct PooledMemoryResource {
   void expand(rx::AddressRange dmemRange) {
     addFreeBlock(dmemRange);
     total += dmemRange.size();
-  }
-
-  orbis::ErrorCode reserve(std::size_t size) {
-    if (size > total) {
-      return orbis::ErrorCode::INVAL;
-    }
-
-    used += size;
-    return {};
+    assert(calcUsed() == used);
+    assert(calcFree() + used == total);
   }
 
   std::pair<std::uint64_t, orbis::ErrorCode> reservePage() {
@@ -114,6 +127,8 @@ struct PooledMemoryResource {
     }
 
     reservedPages.push_back(allocatedPage);
+    assert(calcUsed() == used);
+    assert(calcFree() + used == total);
     return {allocatedPage, {}};
   }
 
@@ -127,6 +142,8 @@ struct PooledMemoryResource {
 
     auto address = reservedPages.back();
     reservedPages.pop_back();
+    assert(calcUsed() == used);
+    assert(calcFree() + used == total);
     return {address, {}};
   }
 
@@ -152,6 +169,9 @@ struct PooledMemoryResource {
       other.expand(block);
       freeBlocks.pop_back();
     }
+
+    assert(calcUsed() == used);
+    assert(calcFree() + used == total);
   }
 
   void commit(orbis::Process *process, rx::AddressRange virtualRange,
@@ -210,6 +230,9 @@ struct PooledMemoryResource {
       virtualRange = rx::AddressRange::fromBeginEnd(
           mapVirtualRange.endAddress(), virtualRange.endAddress());
     }
+
+    assert(calcUsed() == used);
+    assert(calcFree() + used == total);
   }
 
   void decommit(orbis::Process *process, rx::AddressRange virtualRange) {
@@ -248,6 +271,9 @@ struct PooledMemoryResource {
         break;
       }
     }
+
+    assert(calcUsed() == used);
+    assert(calcFree() + used == total);
   }
 
   std::optional<orbis::MemoryType> getMemoryType(std::uint64_t address) {
@@ -350,6 +376,9 @@ orbis::blockpool::commit(Process *process, rx::AddressRange vmemRange,
 }
 
 void orbis::blockpool::decommit(Process *process, rx::AddressRange vmemRange) {
+  rx::println(stderr, "blockpool::decommit({:x}-{:x})",
+              vmemRange.beginAddress(), vmemRange.endAddress());
+
   std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool);
   g_cachedBlockpool->decommit(process, vmemRange);
   g_blockpool->decommit(process, vmemRange);

kernel/orbis/src/dmem.cpp

@@ -26,6 +26,7 @@ struct DirectMemoryAllocation {
   struct Mapping {
     orbis::Process *process;
     rx::AddressRange vmRange;
+    std::uint64_t dmemOffset;
 
     void serialize(rx::Serializer &s) const {
       s.serialize(process->pid);
@@ -79,6 +80,58 @@ struct DirectMemoryAllocation {
     return result;
   }
 
+  void unmap(orbis::Process *process, rx::AddressRange range) {
+    for (std::size_t i = 0; i < mappings.size(); ++i) {
+      auto &map = mappings[i];
+
+      if (process != nullptr && map.process != process) {
+        continue;
+      }
+
+      auto blockRange = range.intersection(map.vmRange);
+
+      if (!blockRange.isValid()) {
+        continue;
+      }
+
+      if (map.vmRange == blockRange) {
+        if (i != mappings.size() - 1) {
+          std::swap(mappings[i], mappings.back());
+        }
+        mappings.pop_back();
+        --i;
+        continue;
+      }
+
+      if (map.vmRange.beginAddress() == blockRange.beginAddress()) {
+        map.vmRange = rx::AddressRange::fromBeginEnd(blockRange.endAddress(),
+                                                     map.vmRange.endAddress());
+        map.dmemOffset += blockRange.size();
+        continue;
+      }
+
+      if (map.vmRange.endAddress() == blockRange.endAddress()) {
+        map.vmRange = rx::AddressRange::fromBeginEnd(map.vmRange.beginAddress(),
+                                                     blockRange.beginAddress());
+        continue;
+      }
+
+      auto leftAllocation = rx::AddressRange::fromBeginEnd(
+          map.vmRange.beginAddress(), blockRange.beginAddress());
+
+      auto rightAllocation = rx::AddressRange::fromBeginEnd(
+          blockRange.endAddress(), map.vmRange.endAddress());
+
+      map.vmRange = leftAllocation;
+      mappings.push_back({
+          .process = process,
+          .vmRange = rightAllocation,
+          .dmemOffset = map.dmemOffset + (rightAllocation.beginAddress() -
+                                          leftAllocation.beginAddress()),
+      });
+    }
+  }
+
   bool operator==(const DirectMemoryAllocation &) const = default;
 };
 
@@ -326,43 +379,78 @@ orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex,
     return ErrorCode::INVAL;
   }
 
-  auto dmem = g_dmemPools[dmemIndex];
-  std::lock_guard lock(*dmem);
+  orbis::kvector<DirectMemoryAllocation::Mapping> clearMappings;
+  {
+    auto dmem = g_dmemPools[dmemIndex];
+    std::lock_guard lock(*dmem);
 
-  constexpr auto razorGpuMemory =
-      rx::AddressRange::fromBeginSize(0x3000000000, 0x20000000);
+    constexpr auto razorGpuMemory =
+        rx::AddressRange::fromBeginSize(0x3000000000, 0x20000000);
 
-  if (dmemIndex == 0 && razorGpuMemory.contains(range.beginAddress())) {
-    return ErrorCode::OPNOTSUPP;
+    if (dmemIndex == 0 && razorGpuMemory.contains(range.beginAddress())) {
+      return ErrorCode::OPNOTSUPP;
+    }
+
+    auto beginIt = dmem->query(range.beginAddress());
+
+    if (beginIt == dmem->end()) {
+      return ErrorCode::NOENT;
+    }
+
+    auto endIt = beginIt;
+    while (endIt != dmem->end() && endIt.beginAddress() < range.endAddress()) {
+      if (!beginIt->isAllocated()) {
+        return ErrorCode::NOENT;
+      }
+
+      if (endIt->isPooled() && !pooled) {
+        return ErrorCode::NOENT;
+      }
+
+      ++endIt;
+    }
+
+    for (auto it = beginIt; it != endIt; ++it) {
+      for (auto &mapping : it->mappings) {
+        auto mapRange = rx::AddressRange::fromBeginSize(mapping.dmemOffset,
+                                                        mapping.vmRange.size());
+        auto releaseRange = mapRange.intersection(range);
+
+        if (!releaseRange.isValid()) {
+          continue;
+        }
+
+        auto releaseVirtualRange = rx::AddressRange::fromBeginSize(
+            mapping.vmRange.beginAddress() +
+                (releaseRange.beginAddress() - mapRange.beginAddress()),
+            releaseRange.size());
+
+        clearMappings.push_back({
+            .process = mapping.process,
+            .vmRange = releaseVirtualRange,
+        });
+      }
+
+      it->unmap(nullptr, range);
+    }
+
+    DirectMemoryAllocation allocation{};
+    auto result = dmem->map(range.beginAddress(), range.size(), allocation,
+                            AllocationFlags::Fixed, vmem::kPageSize);
+
+    if (result.errc != std::errc{}) {
+      return toErrorCode(result.errc);
+    }
+
+    dmemDump(dmemIndex, rx::format("released {:x}-{:x}", range.beginAddress(),
+                                   range.endAddress()));
   }
 
-  auto it = dmem->query(range.beginAddress());
-
-  if (it == dmem->end() || !it->isAllocated()) {
-    return ErrorCode::NOENT;
-  }
-
-  if (it->isPooled() && !pooled) {
-    return ErrorCode::NOENT;
-  }
-
-  for (auto mapping : it->mappings) {
+  for (auto mapping : clearMappings) {
     mapping.process->invoke(
         [=] { vmem::unmap(mapping.process, mapping.vmRange); });
   }
 
-  it->mappings.clear();
-
-  DirectMemoryAllocation allocation{};
-  auto result = dmem->map(range.beginAddress(), range.size(), allocation,
-                          AllocationFlags::Fixed, vmem::kPageSize);
-
-  if (result.errc != std::errc{}) {
-    return toErrorCode(result.errc);
-  }
-
-  dmemDump(dmemIndex, rx::format("released {:x}-{:x}", range.beginAddress(),
-                                 range.endAddress()));
   return {};
 }
 
@@ -647,41 +735,44 @@ orbis::ErrorCode orbis::dmem::map(orbis::Process *process, unsigned dmemIndex,
     return orbis::ErrorCode::ACCES;
   }
 
-  auto allocationInfoIt = dmem->query(offset);
+  auto beginIt = dmem->query(offset);
 
-  if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) {
-    if (allocationInfoIt != dmem->end()) {
-      dmemDump(
-          dmemIndex,
-          rx::format("map unallocated {:x}-{:x}, requested range {:x}-{:x}",
-                     allocationInfoIt.beginAddress(),
-                     allocationInfoIt.endAddress(), range.beginAddress(),
-                     range.endAddress()));
-    } else {
-      dmemDump(dmemIndex, rx::format("map out of memory {:x}-{:x}",
-                                     range.beginAddress(), range.endAddress()));
+  if (beginIt == dmem->end()) {
+    dmemDump(dmemIndex, rx::format("map out of memory {:x}-{:x}",
+                                   range.beginAddress(), range.endAddress()));
+    return orbis::ErrorCode::ACCES;
+  }
+
+  auto endIt = beginIt;
+  while (endIt != dmem->end() && endIt.beginAddress() < offset + range.size()) {
+    if (!endIt->isAllocated() || endIt->isPooled()) {
+      return orbis::ErrorCode::ACCES;
     }
+
+    if (!vmem::validateMemoryType(endIt->getMemoryType(), protection)) {
+      return ErrorCode::ACCES;
+    }
+
+    // if (!endIt->mappings.empty() && !process->allowDmemAliasing) {
+    //   return ErrorCode::INVAL;
+    // }
+
+    ++endIt;
+  }
+
+  if (auto last = endIt; (--last).endAddress() < offset + range.size()) {
     return orbis::ErrorCode::ACCES;
   }
 
-  if (allocationInfoIt->isPooled()) {
-    return orbis::ErrorCode::ACCES;
-  }
-
-  if (!vmem::validateMemoryType(allocationInfoIt->getMemoryType(),
-                                protection)) {
-    return ErrorCode::ACCES;
-  }
-
-  // if (!allocationInfoIt->mappings.empty() && !process->allowDmemAliasing) {
-  //   return ErrorCode::INVAL;
-  // }
-
-  auto directRange = rx::AddressRange::fromBeginSize(offset, range.size())
-                         .intersection(allocationInfoIt.range());
-
-  if (range.size() > directRange.size()) {
-    return orbis::ErrorCode::INVAL;
+  for (auto it = beginIt; it != endIt; ++it) {
+    auto itRange = it.range();
+    auto mappingRange = range.intersection(itRange);
+    it->mappings.push_back({
+        .process = process,
+        .vmRange = mappingRange,
+        .dmemOffset =
+            offset + (mappingRange.beginAddress() - itRange.beginAddress()),
+    });
   }
 
   auto physicalRange =
@@ -692,14 +783,8 @@ orbis::ErrorCode orbis::dmem::map(orbis::Process *process, unsigned dmemIndex,
                             vmem::toCpuProtection(protection));
   });
 
-  if (result == ErrorCode{}) {
-    allocationInfoIt->mappings.push_back({
-        .process = process,
-        .vmRange = range,
-    });
-  }
-
-  return result;
+  rx::dieIf(result != ErrorCode{}, "failed to map physical memory");
+  return {};
 }
 
 orbis::ErrorCode orbis::dmem::notifyUnmap(orbis::Process *process,
@@ -713,43 +798,11 @@ orbis::ErrorCode orbis::dmem::notifyUnmap(orbis::Process *process,
   auto dmem = g_dmemPools[dmemIndex];
   std::lock_guard lock(*dmem);
 
-  auto it = dmem->query(offset);
-  if (it == dmem->end()) {
-    return ErrorCode::INVAL;
-  }
+  auto it = dmem->lowerBound(offset);
 
-  for (auto mapIt = it->mappings.begin(); mapIt != it->mappings.end();) {
-    if (mapIt->process == process && mapIt->vmRange.intersects(range)) {
-      auto blockRange = range.intersection(mapIt->vmRange);
-      if (mapIt->vmRange == blockRange) {
-        mapIt = it->mappings.erase(mapIt);
-        break;
-      }
-
-      if (mapIt->vmRange.beginAddress() == blockRange.beginAddress()) {
-        mapIt->vmRange = rx::AddressRange::fromBeginEnd(
-            blockRange.endAddress(), mapIt->vmRange.endAddress());
-        break;
-      }
-
-      if (mapIt->vmRange.endAddress() == blockRange.endAddress()) {
-        mapIt->vmRange = rx::AddressRange::fromBeginEnd(
-            mapIt->vmRange.beginAddress(), blockRange.beginAddress());
-        break;
-      }
-
-      auto leftAllocation = rx::AddressRange::fromBeginEnd(
-          mapIt->vmRange.beginAddress(), blockRange.beginAddress());
-
-      auto rightAllocation = rx::AddressRange::fromBeginEnd(
-          blockRange.endAddress(), mapIt->vmRange.endAddress());
-
-      mapIt->vmRange = leftAllocation;
-      it->mappings.push_back({.process = process, .vmRange = rightAllocation});
-      break;
-    }
-
-    ++mapIt;
+  while (it != dmem->end() && it.beginAddress() < offset + range.size()) {
+    it->unmap(process, range);
+    ++it;
   }
 
   return {};

rpcsx/gpu/Device.cpp

@@ -984,7 +984,7 @@ void Device::mapMemory(std::uint32_t pid, rx::AddressRange virtualRange,
   auto vmemAddress = memory.getVirtualAddress(virtualRange.beginAddress());
   auto errc = orbis::pmem::map(vmemAddress,
                                rx::AddressRange::fromBeginSize(
-                                   physicalOffset, virtualRange.beginAddress()),
+                                   physicalOffset, virtualRange.size()),
                                orbis::vmem::toGpuProtection(prot));
   if (errc != orbis::ErrorCode{}) {
     rx::die("failed to map process {} memory, address {:x}-{:x}, type {}, "