RPCSX/rpcsx
1
0
Fork 0
mirror of https://github.com/RPCSX/rpcsx.git synced 2026-04-20 22:05:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

orbis: implement physical memory emulation level & utils improvement

Browse source 
fix blockpool & dmem implementation
modernize blockpool & dmem io devices
use budgets per allocation
add serialization support for MemoryTableWithPayload and AddressRange utils
add format support for EnumBitSet util
implemented trace formatter per syscall
increased allowed reference count for Ref
This commit is contained in:
DH 2025-11-30 15:46:37 +03:00
parent 479b09b2df
commit d7ad77b406
70 changed files with 4268 additions and 3160 deletions
Download patch file Download diff file Expand all files Collapse all files

20
kernel/orbis/src/IoDevice.cpp
View file

@ -1,6 +1,9 @@
#include "IoDevice.hpp"
#include "file.hpp"
#include "rx/Mappable.hpp"
#include "thread/Thread.hpp"
#include "utils/Logs.hpp"
#include "vmem.hpp"
static std::string iocGroupToString(unsigned iocGroup) {
if (iocGroup >= 128) {
@ -64,6 +67,23 @@ static std::string iocGroupToString(unsigned iocGroup) {
return "'?'";
}
orbis::ErrorCode
orbis::IoDevice::map(rx::AddressRange range, std::int64_t offset,
rx::EnumBitSet<vmem::Protection> protection, File *file,
Process *) {
if (!file->dirEntries.empty()) {
return orbis::ErrorCode::ISDIR;
}
if (!file->hostFd) {
return ErrorCode::NOTSUP;
}
auto errc = file->hostFd.map(range, offset, vmem::toCpuProtection(protection),
orbis::vmem::kPageSize);
return orbis::toErrorCode(errc);
}
orbis::ErrorCode orbis::IoDevice::ioctl(std::uint64_t request,
orbis::ptr<void> argp, Thread *thread) {
auto group = iocGroupToString(ioctl::group(request));

370
kernel/orbis/src/blockpool.cpp Normal file
View file

@ -0,0 +1,370 @@
#include "blockpool.hpp"
#include "KernelAllocator.hpp"
#include "KernelObject.hpp"
#include "dmem.hpp"
#include "pmem.hpp"
#include "rx/AddressRange.hpp"
#include "rx/MemoryTable.hpp"
#include "rx/die.hpp"
#include "thread/Process.hpp"
#include "vmem.hpp"
#include <algorithm>
#include <iterator>
#include <mutex>
// FIXME: remove
namespace amdgpu {
void mapMemory(std::uint32_t pid, rx::AddressRange virtualRange,
orbis::MemoryType memoryType,
rx::EnumBitSet<orbis::vmem::Protection> prot,
std::uint64_t offset);
void unmapMemory(std::uint32_t pid, rx::AddressRange virtualRange);
void protectMemory(std::uint32_t pid, rx::AddressRange virtualRange,
rx::EnumBitSet<orbis::vmem::Protection> prot);
} // namespace amdgpu
struct PooledMemoryResource {
struct CommitedBlock {
std::uint64_t pmemAddress;
orbis::MemoryType type;
bool operator==(const CommitedBlock &) const = default;
};
std::uint64_t total = 0;
std::uint64_t used = 0;
orbis::kvector<rx::AddressRange> freeBlocks;
rx::MemoryTableWithPayload<CommitedBlock, orbis::kallocator> usedBlocks;
orbis::kvector<std::uint64_t> reservedPages;
void clear() {
total = 0;
used = 0;
freeBlocks.clear();
usedBlocks.clear();
reservedPages.clear();
}
void addFreeBlock(rx::AddressRange dmemRange) {
if (freeBlocks.empty()) {
freeBlocks.push_back(dmemRange);
return;
}
auto it = std::upper_bound(
freeBlocks.begin(), freeBlocks.end(), dmemRange.beginAddress(),
[](auto lhs, auto rhs) {
if constexpr (requires { lhs.beginAddress() < rhs; }) {
return lhs.beginAddress() < rhs;
} else {
return lhs < rhs.beginAddress();
}
});
if (it != freeBlocks.end() &&
dmemRange.endAddress() == it->beginAddress()) {
*it = rx::AddressRange::fromBeginEnd(it->beginAddress(),
dmemRange.endAddress());
return;
}
if (it != freeBlocks.begin()) {
auto prev = std::prev(it);
if (prev->endAddress() == dmemRange.beginAddress()) {
*prev = rx::AddressRange::fromBeginEnd(prev->beginAddress(),
dmemRange.endAddress());
return;
}
}
freeBlocks.insert(it, dmemRange);
return;
}
void expand(rx::AddressRange dmemRange) {
addFreeBlock(dmemRange);
total += dmemRange.size();
}
orbis::ErrorCode reserve(std::size_t size) {
if (size > total) {
return orbis::ErrorCode::INVAL;
}
used += size;
return {};
}
std::pair<std::uint64_t, orbis::ErrorCode> reservePage() {
if (total - used < orbis::dmem::kPageSize) {
return {{}, orbis::ErrorCode::INVAL};
}
used += orbis::dmem::kPageSize;
auto &block = freeBlocks.back();
auto allocatedPage = block.beginAddress();
if (block.endAddress() == allocatedPage + orbis::dmem::kPageSize) {
freeBlocks.pop_back();
} else {
block = rx::AddressRange::fromBeginEnd(
allocatedPage + orbis::dmem::kPageSize, block.endAddress());
}
reservedPages.push_back(allocatedPage);
return {allocatedPage, {}};
}
std::pair<std::uint64_t, orbis::ErrorCode> releasePage() {
if (reservedPages.empty()) {
return {{}, orbis::ErrorCode::INVAL};
}
used -= orbis::dmem::kPageSize;
total -= orbis::dmem::kPageSize;
auto address = reservedPages.back();
reservedPages.pop_back();
return {address, {}};
}
void moveTo(PooledMemoryResource &other, std::size_t size) {
while (size > 0 && total > 0) {
auto &block = freeBlocks.back();
if (block.size() > size) {
total -= size;
auto moveRange =
rx::AddressRange::fromBeginSize(block.beginAddress(), size);
other.expand(moveRange);
block = rx::AddressRange::fromBeginEnd(moveRange.endAddress(),
block.endAddress());
break;
}
total -= block.size();
size -= block.size();
other.expand(block);
freeBlocks.pop_back();
}
}
void commit(orbis::Process *process, rx::AddressRange virtualRange,
orbis::MemoryType type,
rx::EnumBitSet<orbis::vmem::Protection> protection) {
while (virtualRange.isValid()) {
auto &block = freeBlocks.back();
if (block.size() >= virtualRange.size()) [[likely]] {
auto mapPhysicalRange = rx::AddressRange::fromBeginSize(
block.beginAddress(), virtualRange.size());
auto errc =
orbis::pmem::map(virtualRange.beginAddress(), mapPhysicalRange,
orbis::vmem::toCpuProtection(protection));
rx::dieIf(errc != orbis::ErrorCode{},
"blockpool: failed to map physical memory");
amdgpu::mapMemory(process->pid, virtualRange, type, protection,
block.beginAddress());
if (mapPhysicalRange.endAddress() == block.endAddress()) {
freeBlocks.pop_back();
} else {
block = rx::AddressRange::fromBeginEnd(mapPhysicalRange.endAddress(),
block.endAddress());
}
usedBlocks.map(virtualRange,
{.pmemAddress = block.beginAddress(), .type = type},
false);
used += virtualRange.size();
break;
}
auto mapVirtualRange = rx::AddressRange::fromBeginSize(
virtualRange.beginAddress(), block.size());
virtualRange = rx::AddressRange::fromBeginEnd(
mapVirtualRange.endAddress(), virtualRange.endAddress());
auto errc = orbis::pmem::map(mapVirtualRange.beginAddress(), block,
orbis::vmem::toCpuProtection(protection));
rx::dieIf(errc != orbis::ErrorCode{},
"blockpool: failed to map physical memory");
amdgpu::mapMemory(process->pid, mapVirtualRange, type, protection,
block.beginAddress());
usedBlocks.map(mapVirtualRange,
{.pmemAddress = block.beginAddress(), .type = type},
false);
freeBlocks.pop_back();
}
}
void decommit(orbis::Process *process, rx::AddressRange virtualRange) {
auto it = usedBlocks.lowerBound(virtualRange.beginAddress());
if (it != usedBlocks.end() &&
it.beginAddress() < virtualRange.beginAddress()) {
auto itRange = it.range();
auto decommitRange = itRange.intersection(virtualRange);
used -= decommitRange.size();
auto decommitPmemRange = rx::AddressRange::fromBeginSize(
it->pmemAddress +
(virtualRange.beginAddress() - itRange.beginAddress()),
decommitRange.size());
addFreeBlock(decommitPmemRange);
usedBlocks.unmap(decommitRange);
amdgpu::unmapMemory(process->pid, decommitRange);
++it;
}
while (it != usedBlocks.end() &&
it.beginAddress() < virtualRange.endAddress()) {
auto itRange = it.range();
auto decommitRange = itRange.intersection(virtualRange);
used -= decommitRange.size();
addFreeBlock(rx::AddressRange::fromBeginSize(it->pmemAddress,
decommitRange.size()));
amdgpu::unmapMemory(process->pid, decommitRange);
if (itRange == decommitRange) {
it = usedBlocks.unmap(it);
} else {
usedBlocks.unmap(decommitRange);
break;
}
}
}
std::optional<orbis::MemoryType> getMemoryType(std::uint64_t address) {
auto it = usedBlocks.queryArea(address);
if (it == usedBlocks.end()) {
return {};
}
return it->type;
}
};
static auto g_blockpool = orbis::createGlobalObject<
kernel::LockableKernelObject<PooledMemoryResource>>();
static auto g_cachedBlockpool = orbis::createGlobalObject<
kernel::LockableKernelObject<PooledMemoryResource>>();
void orbis::blockpool::clear() {
std::scoped_lock lock(*g_blockpool, *g_cachedBlockpool);
g_blockpool->clear();
g_cachedBlockpool->clear();
}
orbis::ErrorCode orbis::blockpool::expand(rx::AddressRange dmemRange) {
std::scoped_lock lock(*g_blockpool);
g_blockpool->expand(dmemRange);
return {};
}
orbis::ErrorCode orbis::blockpool::allocateControlBlock() {
std::scoped_lock cachedLock(*g_cachedBlockpool);
if (g_cachedBlockpool->used < g_cachedBlockpool->total) {
return g_cachedBlockpool->reservePage().second;
}
std::scoped_lock lock(*g_blockpool);
if (g_blockpool->total - g_blockpool->used < dmem::kPageSize) {
return ErrorCode::INVAL;
}
g_blockpool->moveTo(*g_cachedBlockpool, dmem::kPageSize);
return g_cachedBlockpool->reservePage().second;
}
orbis::ErrorCode orbis::blockpool::releaseControlBlock() {
std::scoped_lock lock(*g_cachedBlockpool);
// control block is always cached
if (!g_cachedBlockpool->reservedPages.empty()) {
auto [page, errc] = g_cachedBlockpool->releasePage();
if (errc != ErrorCode{}) {
return errc;
}
g_cachedBlockpool->expand(
rx::AddressRange::fromBeginSize(page, dmem::kPageSize));
return {};
}
return ErrorCode::INVAL;
}
orbis::ErrorCode
orbis::blockpool::commit(Process *process, rx::AddressRange vmemRange,
MemoryType type,
rx::EnumBitSet<orbis::vmem::Protection> protection) {
auto pool =
type == MemoryType::WbOnion ? g_cachedBlockpool : g_blockpool;
auto otherPool =
type == MemoryType::WbOnion ? g_blockpool : g_cachedBlockpool;
std::scoped_lock lock(*pool);
if (auto avail = pool->total - pool->used; avail < vmemRange.size()) {
// try to steal free blocks from other pool
std::scoped_lock lock(*otherPool);
auto pullSize = vmemRange.size() - avail;
if (otherPool->total - otherPool->used < pullSize) {
return ErrorCode::NOMEM;
}
otherPool->moveTo(*pool, pullSize);
}
pool->commit(process, vmemRange, type, protection);
return {};
}
void orbis::blockpool::decommit(Process *process, rx::AddressRange vmemRange) {
std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool);
g_cachedBlockpool->decommit(process, vmemRange);
g_blockpool->decommit(process, vmemRange);
}
std::optional<orbis::MemoryType>
orbis::blockpool::getType(std::uint64_t address) {
{
std::scoped_lock lock(*g_cachedBlockpool);
if (auto result = g_cachedBlockpool->getMemoryType(address)) {
return result;
}
}
std::scoped_lock lock(*g_blockpool);
return g_blockpool->getMemoryType(address);
}
orbis::blockpool::BlockStats orbis::blockpool::stats() {
BlockStats result{};
{
std::scoped_lock lock(*g_cachedBlockpool, *g_blockpool);
result.availFlushedBlocks =
(g_blockpool->total - g_blockpool->used) / dmem::kPageSize;
result.availCachedBlocks =
(g_cachedBlockpool->total - g_cachedBlockpool->used) / dmem::kPageSize;
result.commitFlushedBlocks = g_blockpool->used / dmem::kPageSize;
result.commitCachedBlocks = g_cachedBlockpool->used / dmem::kPageSize;
}
return result;
}

610
kernel/orbis/src/dmem.cpp Normal file
View file

@ -0,0 +1,610 @@
#include "dmem.hpp"
#include "KernelAllocator.hpp"
#include "KernelObject.hpp"
#include "error.hpp"
#include "kernel/KernelObject.hpp"
#include "pmem.hpp"
#include "rx/AddressRange.hpp"
#include "rx/FileLock.hpp"
#include "rx/Serializer.hpp"
#include "rx/die.hpp"
#include "rx/print.hpp"
#include "utils/Logs.hpp"
#include "vmem.hpp"
#include <array>
#include <rx/format.hpp>
#include <string_view>
#include <utility>
struct DirectMemoryAllocation {
static constexpr std::uint32_t kAllocatedBit = 1 << 31;
static constexpr std::uint32_t kPooledBit = 1 << 30;
std::uint32_t type = 0;
[[nodiscard]] bool isPooled() const { return type & kPooledBit; }
void markAsPooled() { type |= kPooledBit | kAllocatedBit; }
void setMemoryType(orbis::MemoryType memoryType) {
type = (std::to_underlying(memoryType) & ~(kAllocatedBit | kPooledBit)) |
(type & kPooledBit) | kAllocatedBit;
}
[[nodiscard]] orbis::MemoryType getMemoryType() const {
return static_cast<orbis::MemoryType>(type & ~(kAllocatedBit | kPooledBit));
}
[[nodiscard]] bool isAllocated() const { return (type & kAllocatedBit) != 0; }
[[nodiscard]] bool isRelated(const DirectMemoryAllocation &other,
rx::AddressRange, rx::AddressRange) const {
return type == other.type;
}
[[nodiscard]] DirectMemoryAllocation
merge(const DirectMemoryAllocation &other, rx::AddressRange,
rx::AddressRange) const {
return other;
}
bool operator==(const DirectMemoryAllocation &) const = default;
};
struct DirectMemoryResourceState {
std::uint64_t pmemOffset{};
std::uint64_t dmemTotalSize{};
std::uint64_t dmemReservedSize{};
};
struct DirectMemoryResource
: kernel::AllocableResource<DirectMemoryAllocation, orbis::kallocator>,
DirectMemoryResourceState {
using BaseResource =
kernel::AllocableResource<DirectMemoryAllocation, orbis::kallocator>;
void create(std::size_t size) {
auto [pmemRange, errc] = orbis::pmem::allocate(0, size, {}, 64 * 1024);
rx::dieIf(errc != orbis::ErrorCode{},
"failed to allocate direct memory: size {:x}, error {}", size,
static_cast<int>(errc));
dmemTotalSize = pmemRange.size();
pmemOffset = pmemRange.beginAddress();
auto dmemResourceRange =
rx::AddressRange::fromBeginSize(0, pmemRange.size());
if (auto errc = BaseResource::create(dmemResourceRange);
errc != std::errc{}) {
rx::die("failed to create direct memory resource {:x}, error {}",
pmemRange.size(), errc);
}
}
orbis::ErrorCode clear() {
auto result = BaseResource::create(
rx::AddressRange::fromBeginSize(0, dmemTotalSize + dmemReservedSize));
if (result != std::errc{}) {
return orbis::toErrorCode(result);
}
dmemReservedSize = 0;
return {};
}
void serialize(rx::Serializer &s) const {
s.serialize(static_cast<const DirectMemoryResourceState &>(*this));
BaseResource::serialize(s);
}
void deserialize(rx::Deserializer &d) {
d.deserialize(static_cast<DirectMemoryResourceState &>(*this));
BaseResource::deserialize(d);
}
};
static std::array g_dmemPools = {
orbis::createGlobalObject<
kernel::LockableKernelObject<DirectMemoryResource>>(),
orbis::createGlobalObject<
kernel::LockableKernelObject<DirectMemoryResource>>(),
orbis::createGlobalObject<
kernel::LockableKernelObject<DirectMemoryResource>>(),
};
orbis::ErrorCode orbis::dmem::initialize() {
g_dmemPools[0]->create(0x120000000);
g_dmemPools[1]->create(0x1000000);
g_dmemPools[2]->create(0x1000000);
return {};
}
orbis::ErrorCode orbis::dmem::clear(unsigned dmemIndex) {
if (dmemIndex >= std::size(g_dmemPools)) {
return ErrorCode::INVAL;
}
{
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
auto result = dmem->clear();
if (result != orbis::ErrorCode{}) {
return result;
}
}
return {};
}
static void dmemDump(unsigned dmemIndex, std::string_view message = {}) {
auto dmem = g_dmemPools[dmemIndex];
rx::ScopedFileLock lock(stderr);
rx::println(stderr, "dmem0 {}", message);
for (auto alloc : *dmem) {
rx::println(stderr, " {:012x}-{:012x}: {}{} {}", alloc.beginAddress(),
alloc.endAddress(), "_A"[alloc->isAllocated()],
"_P"[alloc->isPooled()], alloc->getMemoryType());
}
}
std::pair<std::uint64_t, orbis::ErrorCode>
orbis::dmem::allocate(unsigned dmemIndex, rx::AddressRange searchRange,
std::uint64_t len, MemoryType memoryType,
std::uint64_t alignment, bool pooled) {
if (dmemIndex >= std::size(g_dmemPools)) {
return {{}, ErrorCode::INVAL};
}
if (searchRange.endAddress() != 0 &&
(!searchRange.isValid() || searchRange.size() < len)) {
ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(),
searchRange.endAddress(), len);
return {{}, orbis::ErrorCode::INVAL};
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
alignment = alignment == 0 ? kPageSize : alignment;
len = rx::alignUp(len, dmem::kPageSize);
if (searchRange.endAddress() == 0) {
searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(),
dmem->dmemTotalSize -
dmem->dmemReservedSize);
}
if (!searchRange.isValid() || searchRange.size() < len) {
ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(),
searchRange.endAddress(), len);
return {{}, orbis::ErrorCode::INVAL};
}
if (len == 0) {
ORBIS_LOG_ERROR(__FUNCTION__, "len is 0", searchRange.beginAddress(),
searchRange.endAddress(), len);
return {{}, ErrorCode::INVAL};
}
if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) {
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
searchRange.beginAddress(), searchRange.endAddress(),
dmem->dmemTotalSize);
return {{}, ErrorCode::INVAL};
}
DirectMemoryAllocation allocation;
allocation.setMemoryType(memoryType);
if (pooled) {
allocation.markAsPooled();
}
auto allocResult = dmem->map(searchRange.beginAddress(), len, allocation,
AllocationFlags::Dry, alignment);
if (allocResult.errc != std::errc{}) {
return {{}, toErrorCode(allocResult.errc)};
}
auto result = allocResult.range.beginAddress();
auto commitResult =
dmem->map(result, len, allocation, AllocationFlags::Fixed, alignment);
rx::dieIf(commitResult.errc != std::errc{},
"dmem: failed to commit memory, error {}", commitResult.errc);
dmemDump(dmemIndex,
rx::format("allocated {:x}-{:x}", allocResult.range.beginAddress(),
allocResult.range.endAddress()));
return {result, {}};
}
orbis::ErrorCode orbis::dmem::release(unsigned dmemIndex,
rx::AddressRange range, bool pooled) {
if (dmemIndex >= std::size(g_dmemPools)) {
return ErrorCode::INVAL;
}
if (range.beginAddress() % kPageSize || range.endAddress() % kPageSize ||
!range.isValid()) {
return ErrorCode::INVAL;
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
constexpr auto razorGpuMemory =
rx::AddressRange::fromBeginSize(0x3000000000, 0x20000000);
if (dmemIndex == 0 && razorGpuMemory.contains(range.beginAddress())) {
return ErrorCode::OPNOTSUPP;
}
auto it = dmem->query(range.beginAddress());
if (it == dmem->end() || !it->isAllocated()) {
return ErrorCode::NOENT;
}
if (it->isPooled() && !pooled) {
return ErrorCode::NOENT;
}
DirectMemoryAllocation allocation{};
auto result = dmem->map(range.beginAddress(), range.size(), allocation,
AllocationFlags::Fixed, vmem::kPageSize);
if (result.errc != std::errc{}) {
return toErrorCode(result.errc);
}
return {};
}
std::pair<std::uint64_t, orbis::ErrorCode>
orbis::dmem::reserveSystem(unsigned dmemIndex, std::uint64_t size) {
if (dmemIndex >= std::size(g_dmemPools)) {
return {{}, ErrorCode::INVAL};
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
if (size == 0 || size % vmem::kPageSize) {
return {{}, ErrorCode::INVAL};
}
if (dmem->dmemReservedSize + size > dmem->dmemTotalSize) {
return {{}, ErrorCode::NOMEM};
}
DirectMemoryAllocation alloc;
alloc.setMemoryType(MemoryType::WbOnion);
auto result = dmem->map(0, size, alloc, AllocationFlags::Stack, kPageSize);
if (result.errc != std::errc{}) {
return {{}, toErrorCode(result.errc)};
}
dmem->dmemReservedSize += size;
return {result.range.beginAddress() | 0x4000000000, {}};
}
std::pair<std::uint64_t, orbis::ErrorCode>
orbis::dmem::reservePooled(unsigned dmemIndex, rx::AddressRange searchRange,
std::uint64_t size, std::uint64_t alignment) {
if (dmemIndex >= std::size(g_dmemPools)) {
return {{}, ErrorCode::INVAL};
}
alignment = alignment == 0 ? kPageSize : alignment;
if (alignment % kPageSize) {
return {{}, ErrorCode::INVAL};
}
auto dmem = g_dmemPools[dmemIndex];
if (searchRange.endAddress() > dmem->dmemTotalSize) {
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
searchRange.endAddress(), dmem->dmemTotalSize);
return {{}, ErrorCode::INVAL};
}
if (!searchRange.isValid() &&
searchRange.beginAddress() != searchRange.endAddress()) {
ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(),
searchRange.endAddress());
return {{}, ErrorCode::INVAL};
}
std::lock_guard lock(*dmem);
if (searchRange.beginAddress() == searchRange.endAddress()) {
searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(),
dmem->dmemTotalSize -
dmem->dmemReservedSize);
if (!searchRange.isValid()) {
ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(),
searchRange.endAddress());
return {{}, ErrorCode::INVAL};
}
}
if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) {
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
searchRange.endAddress(), dmem->dmemTotalSize,
dmem->dmemReservedSize);
return {{}, ErrorCode::INVAL};
}
auto it = dmem->lowerBound(searchRange.beginAddress());
rx::AddressRange result;
while (it != dmem->end() && it.beginAddress() < searchRange.endAddress()) {
if (!it->isAllocated()) {
auto viewRange = searchRange.intersection(it.range());
viewRange = rx::AddressRange::fromBeginEnd(
rx::alignUp(viewRange.beginAddress(), alignment),
viewRange.endAddress());
if (viewRange.isValid() && viewRange.size() >= size) {
result =
rx::AddressRange::fromBeginSize(viewRange.beginAddress(), size);
break;
}
}
++it;
}
if (!result.isValid()) {
return {{}, ErrorCode::NOMEM};
}
DirectMemoryAllocation allocation;
allocation.markAsPooled();
auto commitResult = dmem->map(result.beginAddress(), result.size(),
allocation, AllocationFlags::Fixed, alignment);
if (commitResult.errc != std::errc{}) {
return {{}, toErrorCode(commitResult.errc)};
}
return {result.beginAddress(), {}};
}
orbis::ErrorCode orbis::dmem::setType(unsigned dmemIndex,
rx::AddressRange range, MemoryType type,
rx::EnumBitSet<QueryFlags> flags) {
if (dmemIndex >= std::size(g_dmemPools)) {
return ErrorCode::INVAL;
}
bool expectedPooled = (flags & QueryFlags::Pooled) == QueryFlags::Pooled;
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
auto it = dmem->query(range.beginAddress());
if (it == dmem->end() || !it->isAllocated() ||
it->isPooled() != expectedPooled) {
return ErrorCode::ACCES;
}
if (it->getMemoryType() == type && it.range().contains(range)) {
return {};
}
DirectMemoryAllocation allocation;
allocation.setMemoryType(type);
auto [_it, errc, _range] =
dmem->map(range.beginAddress(), range.size(), allocation,
AllocationFlags::Fixed, vmem::kPageSize);
return toErrorCode(errc);
}
std::optional<orbis::dmem::QueryResult>
orbis::dmem::query(unsigned dmemIndex, std::uint64_t dmemOffset,
rx::EnumBitSet<QueryFlags> flags) {
if (dmemIndex >= std::size(g_dmemPools)) {
return {};
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
auto it = dmem->lowerBound(dmemOffset);
if (flags & QueryFlags::LowerBound) {
while (it != dmem->end() && !it->isAllocated()) {
++it;
}
if (it == dmem->end()) {
return {};
}
} else if (it == dmem->end() || !it->isAllocated()) {
return {};
}
if (!(flags & QueryFlags::Pooled)) {
if (it->isPooled()) {
return {};
}
}
return QueryResult{.range = it.range(), .memoryType = it->getMemoryType()};
}
std::uint64_t orbis::dmem::getSize(unsigned dmemIndex) {
if (dmemIndex >= std::size(g_dmemPools)) {
return 0;
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
return dmem->dmemTotalSize - dmem->dmemReservedSize;
}
std::pair<rx::AddressRange, orbis::ErrorCode>
orbis::dmem::getAvailSize(unsigned dmemIndex, rx::AddressRange searchRange,
std::uint64_t alignment) {
if (dmemIndex >= std::size(g_dmemPools)) {
return {{}, ErrorCode::INVAL};
}
alignment = alignment == 0 ? kPageSize : alignment;
if (alignment % kPageSize) {
return {{}, ErrorCode::INVAL};
}
auto dmem = g_dmemPools[dmemIndex];
if (searchRange.endAddress() > dmem->dmemTotalSize) {
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
searchRange.endAddress(), dmem->dmemTotalSize);
return {{}, orbis::ErrorCode::INVAL};
}
if (!searchRange.isValid() &&
searchRange.beginAddress() != searchRange.endAddress()) {
ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(),
searchRange.endAddress());
return {{}, orbis::ErrorCode::INVAL};
}
std::lock_guard lock(*dmem);
if (searchRange.beginAddress() == searchRange.endAddress()) {
searchRange = rx::AddressRange::fromBeginEnd(searchRange.beginAddress(),
dmem->dmemTotalSize -
dmem->dmemReservedSize);
if (!searchRange.isValid()) {
ORBIS_LOG_ERROR(__FUNCTION__, "invalid range", searchRange.beginAddress(),
searchRange.endAddress());
return {{}, orbis::ErrorCode::INVAL};
}
}
if (searchRange.endAddress() > dmem->dmemTotalSize - dmem->dmemReservedSize) {
ORBIS_LOG_ERROR(__FUNCTION__, "out of direct memory size",
searchRange.endAddress(), dmem->dmemTotalSize,
dmem->dmemReservedSize);
return {{}, orbis::ErrorCode::INVAL};
}
auto it = dmem->lowerBound(searchRange.beginAddress());
rx::AddressRange result = rx::AddressRange::fromBeginEnd(
searchRange.beginAddress(), searchRange.beginAddress());
while (it != dmem->end() && it.beginAddress() < searchRange.endAddress()) {
if (!it->isAllocated()) {
auto viewRange = searchRange.intersection(it.range());
viewRange = rx::AddressRange::fromBeginEnd(
rx::alignUp(viewRange.beginAddress(), alignment),
viewRange.endAddress());
if (viewRange.isValid() &&
(!result.isValid() || viewRange.size() > result.size())) {
result = viewRange;
}
}
++it;
}
return {result, {}};
}
orbis::ErrorCode orbis::dmem::map(unsigned dmemIndex, rx::AddressRange range,
std::uint64_t offset,
rx::EnumBitSet<vmem::Protection> protection) {
if (dmemIndex >= std::size(g_dmemPools)) {
return ErrorCode::INVAL;
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
if (offset < 0 || offset + range.size() < offset ||
offset + range.size() > dmem->dmemTotalSize) {
return orbis::ErrorCode::INVAL;
}
if (offset + range.size() > dmem->dmemTotalSize - dmem->dmemReservedSize) {
return orbis::ErrorCode::ACCES;
}
auto allocationInfoIt = dmem->query(offset);
if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) {
if (allocationInfoIt != dmem->end()) {
dmemDump(
dmemIndex,
rx::format("map unallocated {:x}-{:x}, requested range {:x}-{:x}",
allocationInfoIt.beginAddress(),
allocationInfoIt.endAddress(), range.beginAddress(),
range.endAddress()));
} else {
dmemDump(dmemIndex, rx::format("map out of memory {:x}-{:x}",
range.beginAddress(), range.endAddress()));
}
return orbis::ErrorCode::ACCES;
}
if (allocationInfoIt->isPooled()) {
return orbis::ErrorCode::ACCES;
}
auto directRange = rx::AddressRange::fromBeginSize(offset, range.size())
.intersection(allocationInfoIt.range());
if (range.size() > directRange.size()) {
return orbis::ErrorCode::INVAL;
}
auto physicalRange =
rx::AddressRange::fromBeginSize(dmem->pmemOffset + offset, range.size());
return orbis::pmem::map(range.beginAddress(), physicalRange,
vmem::toCpuProtection(protection));
}
std::pair<std::uint64_t, orbis::ErrorCode>
orbis::dmem::getPmemOffset(unsigned dmemIndex, std::uint64_t dmemOffset) {
if (dmemIndex > std::size(g_dmemPools)) {
return {{}, ErrorCode::INVAL};
}
auto dmem = g_dmemPools[dmemIndex];
std::lock_guard lock(*dmem);
if (dmemOffset >= dmem->dmemTotalSize) {
return {{}, orbis::ErrorCode::INVAL};
}
if (dmemOffset >= dmem->dmemTotalSize - dmem->dmemReservedSize) {
return {{}, orbis::ErrorCode::ACCES};
}
auto allocationInfoIt = dmem->query(dmemOffset);
if (allocationInfoIt == dmem->end() || !allocationInfoIt->isAllocated()) {
return {{}, orbis::ErrorCode::ACCES};
}
return {dmem->pmemOffset + dmemOffset, {}};
}

49
kernel/orbis/src/fmem.cpp
View file

@ -1,12 +1,14 @@
#include "fmem.hpp"
#include "KernelAllocator.hpp"
#include "KernelObject.hpp"
#include "error.hpp"
#include "kernel/KernelObject.hpp"
#include "kernel/MemoryResource.hpp"
#include "pmem.hpp"
#include "rx/AddressRange.hpp"
#include "thread/Process.hpp"
#include "rx/debug.hpp"
#include "rx/print.hpp"
#include "vmem.hpp"
#include <cassert>
#include <rx/Mappable.hpp>
@ -30,43 +32,39 @@ struct FlexibleMemoryAllocation {
};
using FlexibleMemoryResource =
kernel::AllocableResource<FlexibleMemoryAllocation>;
kernel::AllocableResource<FlexibleMemoryAllocation, orbis::kallocator>;
static auto g_fmemInstance = orbis::createProcessLocalObject<
static auto g_fmemInstance = orbis::createGlobalObject<
kernel::LockableKernelObject<FlexibleMemoryResource>>();
orbis::ErrorCode orbis::fmem::initialize(Process *process, std::uint64_t size) {
orbis::ErrorCode orbis::fmem::initialize(std::uint64_t size) {
auto [range, errc] =
pmem::allocate(pmem::getSize() - 1, size, pmem::MemoryType::WbOnion,
kernel::AllocationFlags::Stack, vmem::kPageSize);
pmem::allocate(0, size, kernel::AllocationFlags::Stack, vmem::kPageSize);
if (errc != ErrorCode{}) {
return errc;
}
auto fmem = process->get(g_fmemInstance);
std::lock_guard lock(*fmem);
return toErrorCode(fmem->create(range));
rx::println("fmem: {:x}-{:x}", range.beginAddress(), range.endAddress());
std::lock_guard lock(*g_fmemInstance);
return toErrorCode(g_fmemInstance->create(range));
}
void orbis::fmem::destroy(Process *process) {
auto fmem = process->get(g_fmemInstance);
void orbis::fmem::destroy() {
std::lock_guard lock(*g_fmemInstance);
std::lock_guard lock(*fmem);
for (auto allocation : fmem->allocations) {
for (auto allocation : g_fmemInstance->allocations) {
pmem::deallocate(allocation);
}
fmem->destroy();
g_fmemInstance->destroy();
}
std::pair<rx::AddressRange, orbis::ErrorCode>
orbis::fmem::allocate(Process *process, std::uint64_t size) {
auto fmem = process->get(g_fmemInstance);
std::lock_guard lock(*fmem);
orbis::fmem::allocate(std::uint64_t size) {
std::lock_guard lock(*g_fmemInstance);
FlexibleMemoryAllocation allocation{.allocated = true};
auto [it, errc, range] = fmem->map(
auto [it, errc, range] = g_fmemInstance->map(
0, size, allocation, kernel::AllocationFlags::NoMerge, vmem::kPageSize);
if (errc != std::errc{}) {
@ -76,13 +74,12 @@ orbis::fmem::allocate(Process *process, std::uint64_t size) {
return {range, {}};
}
orbis::ErrorCode orbis::fmem::deallocate(Process *process,
rx::AddressRange range) {
orbis::ErrorCode orbis::fmem::deallocate(rx::AddressRange range) {
FlexibleMemoryAllocation allocation{};
auto fmem = process->get(g_fmemInstance);
std::lock_guard lock(*fmem);
auto [it, errc, _] = fmem->map(range.beginAddress(), range.size(), allocation,
AllocationFlags::Fixed, 1);
std::lock_guard lock(*g_fmemInstance);
auto [it, errc, _] =
g_fmemInstance->map(range.beginAddress(), range.size(), allocation,
AllocationFlags::Fixed, 1);
return toErrorCode(errc);
}

53
kernel/orbis/src/pmem.cpp
View file

@ -1,30 +1,32 @@
#include "pmem.hpp"
#include "IoDevice.hpp"
#include "KernelAllocator.hpp"
#include "KernelObject.hpp"
#include "error.hpp"
#include "error/ErrorCode.hpp"
#include "file.hpp"
#include "kernel/KernelObject.hpp"
#include "kernel/MemoryResource.hpp"
#include "rx/AddressRange.hpp"
#include "rx/Rc.hpp"
#include "rx/print.hpp"
#include "vmem.hpp"
#include <cassert>
#include <rx/Mappable.hpp>
struct PhysicalMemoryAllocation {
orbis::pmem::MemoryType type = orbis::pmem::MemoryType::Invalid;
bool allocated = false;
[[nodiscard]] bool isAllocated() const {
return type != orbis::pmem::MemoryType::Invalid;
}
[[nodiscard]] bool isAllocated() const { return allocated; }
[[nodiscard]] bool isRelated(const PhysicalMemoryAllocation &left,
rx::AddressRange, rx::AddressRange) const {
return type == left.type;
return allocated == left.allocated;
}
[[nodiscard]] PhysicalMemoryAllocation
merge(const PhysicalMemoryAllocation &other, rx::AddressRange,
rx::AddressRange) const {
assert(other.type == type);
assert(other.allocated == allocated);
return other;
}
@ -37,12 +39,22 @@ using MappableMemoryResource =
})>;
using PhysicalMemoryResource =
kernel::AllocableResource<PhysicalMemoryAllocation, MappableMemoryResource>;
kernel::AllocableResource<PhysicalMemoryAllocation, orbis::kallocator,
MappableMemoryResource>;
static auto g_pmemInstance = orbis::createGlobalObject<
kernel::LockableKernelObject<PhysicalMemoryResource>>();
struct PhysicalMemory : orbis::IoDevice {
orbis::File file;
PhysicalMemory() {
incRef(); // do not delete global object
file.device = this;
file.incRef(); // do not delete property
}
orbis::ErrorCode open(rx::Ref<orbis::File> *file, const char *path,
std::uint32_t flags, std::uint32_t mode,
orbis::Thread *thread) override {
@ -50,11 +62,12 @@ struct PhysicalMemory : orbis::IoDevice {
}
orbis::ErrorCode map(rx::AddressRange range, std::int64_t offset,
rx::EnumBitSet<rx::mem::Protection> protection,
orbis::Process *) override {
rx::EnumBitSet<orbis::vmem::Protection> protection,
orbis::File *, orbis::Process *) override {
return orbis::pmem::map(
range.beginAddress(),
rx::AddressRange::fromBeginSize(offset, range.size()), protection);
rx::AddressRange::fromBeginSize(offset, range.size()),
orbis::vmem::toCpuProtection(protection));
}
void serialize(rx::Serializer &s) const {}
@ -65,6 +78,8 @@ static auto g_phyMemory = orbis::createGlobalObject<PhysicalMemory>();
orbis::ErrorCode orbis::pmem::initialize(std::uint64_t size) {
std::lock_guard lock(*g_pmemInstance);
rx::println("pmem: {:x}", size);
return toErrorCode(
g_pmemInstance->create(rx::AddressRange::fromBeginSize(0, size)));
}
@ -74,11 +89,12 @@ void orbis::pmem::destroy() {
g_pmemInstance->destroy();
}
std::pair<rx::AddressRange, orbis::ErrorCode> orbis::pmem::allocate(
std::uint64_t addressHint, std::uint64_t size, MemoryType memoryType,
rx::EnumBitSet<AllocationFlags> flags, std::uint64_t alignment) {
std::pair<rx::AddressRange, orbis::ErrorCode>
orbis::pmem::allocate(std::uint64_t addressHint, std::uint64_t size,
rx::EnumBitSet<AllocationFlags> flags,
std::uint64_t alignment) {
std::lock_guard lock(*g_pmemInstance);
PhysicalMemoryAllocation allocation{.type = memoryType};
PhysicalMemoryAllocation allocation{.allocated = true};
auto [it, errc, range] =
g_pmemInstance->map(addressHint, size, allocation, flags, alignment);
@ -99,8 +115,7 @@ orbis::ErrorCode orbis::pmem::deallocate(rx::AddressRange range) {
return toErrorCode(errc);
}
std::optional<orbis::pmem::AllocatedMemory>
orbis::pmem::query(std::uint64_t address) {
std::optional<rx::AddressRange> orbis::pmem::query(std::uint64_t address) {
std::lock_guard lock(*g_pmemInstance);
auto result = g_pmemInstance->query(address);
@ -108,7 +123,7 @@ orbis::pmem::query(std::uint64_t address) {
return {};
}
return AllocatedMemory{.range = result.range(), .memoryType = result->type};
return result.range();
}
orbis::ErrorCode
@ -117,11 +132,11 @@ orbis::pmem::map(std::uint64_t virtualAddress, rx::AddressRange range,
auto virtualRange =
rx::AddressRange::fromBeginSize(virtualAddress, range.size());
auto errc = g_pmemInstance->mappable.map(virtualRange, range.beginAddress(),
protection, orbis::vmem::kPageSize);
protection, vmem::kPageSize);
return toErrorCode(errc);
}
std::size_t orbis::pmem::getSize() { return g_pmemInstance->size; }
orbis::IoDevice *orbis::pmem::getDevice() { return g_phyMemory.get(); }
orbis::File *orbis::pmem::getFile() { return &g_phyMemory->file; }

24
kernel/orbis/src/sys/sys_event.cpp
View file

@ -40,31 +40,33 @@ orbis::SysResult orbis::sys_kqueueex(Thread *thread, ptr<char> name,
return {};
}
static bool isReadEventTriggered(int hostFd) {
static bool isReadEventTriggered(const rx::Mappable &hostFd) {
#ifdef __linux
fd_set fds{};
FD_SET(hostFd, &fds);
FD_SET(hostFd.native_handle(), &fds);
timeval timeout{};
if (::select(hostFd + 1, &fds, nullptr, nullptr, &timeout) < 0) {
if (::select(hostFd.native_handle() + 1, &fds, nullptr, nullptr, &timeout) <
0) {
return false;
}
return FD_ISSET(hostFd, &fds);
return FD_ISSET(hostFd.native_handle(), &fds);
#else
#warning "Not implemented"
return false;
#endif
}
static bool isWriteEventTriggered(int hostFd) {
static bool isWriteEventTriggered(const rx::Mappable &hostFd) {
#ifdef __linux
fd_set fds{};
FD_SET(hostFd, &fds);
FD_SET(hostFd.native_handle(), &fds);
timeval timeout{};
if (::select(hostFd + 1, nullptr, &fds, nullptr, &timeout) < 0) {
if (::select(hostFd.native_handle() + 1, nullptr, &fds, nullptr, &timeout) <
0) {
return false;
}
return FD_ISSET(hostFd, &fds);
return FD_ISSET(hostFd.native_handle(), &fds);
#else
#warning "Not implemented"
return false;
@ -131,7 +133,7 @@ static SysResult keventChange(KQueue *kq, KEvent &change, Thread *thread) {
eventEmitter->subscribe(&*nodeIt);
nodeIt->triggered = true;
kq->cv.notify_all(kq->mtx);
} else if (note.file->hostFd < 0) {
} else if (!note.file->hostFd) {
ORBIS_LOG_ERROR("Unimplemented event emitter", change.ident);
}
} else if (change.filter == kEvFiltGraphicsCore ||
@ -303,7 +305,7 @@ orbis::SysResult orbis::sys_kevent(Thread *thread, sint fd,
if (!note.triggered) {
if (note.event.filter == kEvFiltRead) {
if (note.file->hostFd >= 0) {
if (note.file->hostFd) {
if (isReadEventTriggered(note.file->hostFd)) {
note.triggered = true;
} else {
@ -311,7 +313,7 @@ orbis::SysResult orbis::sys_kevent(Thread *thread, sint fd,
}
}
} else if (note.event.filter == kEvFiltWrite) {
if (note.file->hostFd >= 0) {
if (note.file->hostFd) {
if (isWriteEventTriggered(note.file->hostFd)) {
note.triggered = true;
} else {

215
kernel/orbis/src/sys/sys_sce.cpp
View file

@ -7,6 +7,9 @@
#include "module/ModuleInfoEx.hpp"
#include "orbis/time.hpp"
#include "osem.hpp"
#include "pmem.hpp"
#include "rx/AddressRange.hpp"
#include "rx/EnumBitSet.hpp"
#include "sys/sysproto.hpp"
#include "thread/Process.hpp"
#include "thread/ProcessOps.hpp"
@ -14,6 +17,7 @@
#include "ucontext.hpp"
#include "uio.hpp"
#include "utils/Logs.hpp"
#include "vmem.hpp"
#include <fcntl.h>
#include <ranges>
#include <utility>
@ -69,8 +73,12 @@ orbis::SysResult orbis::sys_netgetiflist(Thread *thread /* TODO */) {
return {};
}
orbis::SysResult orbis::sys_mtypeprotect(Thread *thread /* TODO */) {
return ErrorCode::NOSYS;
orbis::SysResult
orbis::sys_mtypeprotect(Thread *thread, uintptr_t addr, size_t len,
MemoryType type,
rx::EnumBitSet<vmem::Protection> prot) {
auto range = rx::AddressRange::fromBeginSize(addr, len);
return vmem::setTypeAndProtect(thread->tproc, range, type, prot);
}
orbis::SysResult orbis::sys_regmgr_call(Thread *thread, uint32_t op,
uint32_t id, ptr<void> result,
@ -393,19 +401,20 @@ orbis::SysResult orbis::sys_evf_cancel(Thread *thread, sint id, uint64_t value,
return {};
}
orbis::SysResult
orbis::sys_query_memory_protection(Thread *thread, ptr<void> address,
ptr<MemoryProtection> protection) {
if (auto query_memory_protection =
thread->tproc->ops->query_memory_protection) {
return query_memory_protection(thread, address, protection);
orbis::sys_query_memory_protection(Thread *thread, uintptr_t address,
ptr<vmem::MemoryProtection> protection) {
auto result = vmem::queryProtection(thread->tproc, address);
if (!result) {
return ErrorCode::ACCES;
}
return ErrorCode::NOSYS;
return uwrite(protection, *result);
}
namespace orbis {
struct BatchMapEntry {
ptr<char> start;
uintptr_t start;
off_t offset;
size_t length;
char protection;
@ -419,14 +428,14 @@ struct BatchMapEntry {
};
} // namespace orbis
orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, sint flags,
orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk,
rx::EnumBitSet<vmem::MapFlags> flags,
ptr<BatchMapEntry> entries,
sint entriesCount,
ptr<sint> processedCount) {
auto ops = thread->tproc->ops;
SysResult result = ErrorCode{};
ORBIS_LOG_ERROR(__FUNCTION__, unk, flags, entries, entriesCount,
processedCount);
ORBIS_LOG_ERROR(__FUNCTION__, unk, flags.toUnderlying(), entries,
entriesCount, processedCount);
int processed = 0;
for (int i = 0; i < entriesCount; ++i) {
@ -438,16 +447,29 @@ orbis::SysResult orbis::sys_batch_map(Thread *thread, sint unk, sint flags,
switch (_entry.operation) {
case 0:
result = ops->dmem_mmap(thread, _entry.start, _entry.length, _entry.type,
_entry.protection, flags, _entry.offset);
break;
result = sys_mmap_dmem(
thread, _entry.start, _entry.length,
static_cast<MemoryType>(_entry.type),
rx::EnumBitSet<vmem::Protection>::fromUnderlying(_entry.protection),
flags, _entry.offset);
case 1:
result = ops->munmap(thread, _entry.start, _entry.length);
result = sys_munmap(thread, _entry.start, _entry.length);
break;
case 2:
result =
ops->mprotect(thread, _entry.start, _entry.length, _entry.protection);
result = sys_mprotect(
thread, _entry.start, _entry.length,
rx::EnumBitSet<vmem::Protection>::fromUnderlying(_entry.protection));
break;
case 3:
result = sys_mmap(
thread, _entry.start, _entry.length,
rx::EnumBitSet<vmem::Protection>::fromUnderlying(_entry.protection),
vmem::MapFlags::Void | vmem::MapFlags::Anon | flags, -1, 0);
break;
case 4:
ORBIS_LOG_ERROR(__FUNCTION__, "unimplemented type protect");
break;
default:
result = ErrorCode::INVAL;
break;
@ -880,14 +902,22 @@ orbis::SysResult orbis::sys_budget_set(Thread *thread, sint budgetId) {
thread->tproc->budgetId = budgetId;
return {};
}
orbis::SysResult orbis::sys_virtual_query(Thread *thread, ptr<void> addr,
uint64_t unk, ptr<void> info,
orbis::SysResult orbis::sys_virtual_query(Thread *thread, uintptr_t addr,
uint64_t flags, ptr<void> info,
size_t infosz) {
if (auto virtual_query = thread->tproc->ops->virtual_query) {
return virtual_query(thread, addr, unk, info, infosz);
if (infosz != sizeof(vmem::QueryResult)) {
return ErrorCode::INVAL;
}
return ErrorCode::NOSYS;
auto result = vmem::query(thread->tproc, addr, flags & 1);
if (!result || result->start >= 0x800000000) {
return ErrorCode::ACCES;
}
return uwrite(ptr<vmem::QueryResult>(info), *result);
}
orbis::SysResult orbis::sys_mdbg_call(Thread *thread /* TODO */) { return {}; }
orbis::SysResult orbis::sys_obs_sblock_create(Thread *thread /* TODO */) {
@ -930,7 +960,7 @@ orbis::SysResult orbis::sys_is_in_sandbox(Thread *thread /* TODO */) {
}
orbis::SysResult orbis::sys_dmem_container(Thread *thread, uint id) {
ORBIS_LOG_NOTICE(__FUNCTION__, id);
thread->retval[0] = 1; // returns default direct memory device
thread->retval[0] = 0; // returns default direct memory device
if (id + 1)
return ErrorCode::PERM;
return {};
@ -955,24 +985,11 @@ orbis::SysResult orbis::sys_mname(Thread *thread, uint64_t addr, uint64_t len,
return result;
}
NamedMemoryRange range;
range.begin = addr & ~0x3fffull;
range.end = range.begin;
range.end += ((addr & 0x3fff) + 0x3fff + len) & ~0x3fffull;
std::lock_guard lock(thread->tproc->namedMemMutex);
auto [it, end] = thread->tproc->namedMem.equal_range<NamedMemoryRange>(range);
while (it != end) {
auto [addr2, end2] = it->first;
auto len2 = end2 - addr2;
ORBIS_LOG_NOTICE("sys_mname: removed overlapped", it->second, addr2, len2);
it = thread->tproc->namedMem.erase(it);
auto range = rx::AddressRange::fromBeginSize(addr, len);
if (auto error = vmem::setName(thread->tproc, range, _name);
error != ErrorCode{}) {
ORBIS_LOG_ERROR(__FUNCTION__, "failure:", static_cast<int>(error));
}
if (!thread->tproc->namedMem.try_emplace(range, _name).second)
std::abort();
if (!thread->tproc->namedMem.count(addr))
std::abort();
return {};
}
orbis::SysResult orbis::sys_dynlib_dlopen(Thread *thread /* TODO */) {
@ -1019,7 +1036,7 @@ orbis::SysResult orbis::sys_dynlib_get_info(Thread *thread,
ModuleInfo result = {};
result.size = sizeof(ModuleInfo);
std::strncpy(result.name, module->moduleName, sizeof(result.name));
std::strncpy(result.name, module->soName, sizeof(result.name));
std::memcpy(result.segments, module->segments,
sizeof(ModuleSegment) * module->segmentCount);
result.segmentCount = module->segmentCount;
@ -1485,14 +1502,56 @@ orbis::SysResult orbis::sys_get_cpu_usage_all(Thread *thread, uint32_t unk,
ORBIS_LOG_TODO(__FUNCTION__, unk, result);
return {};
}
orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, caddr_t addr, size_t len,
sint memoryType, sint prot, sint flags,
orbis::SysResult orbis::sys_mmap_dmem(Thread *thread, uintptr_t addr,
size_t len, MemoryType memoryType,
rx::EnumBitSet<vmem::Protection> prot,
rx::EnumBitSet<vmem::MapFlags> flags,
off_t directMemoryStart) {
if (auto dmem_mmap = thread->tproc->ops->dmem_mmap) {
return dmem_mmap(thread, addr, len, memoryType, prot, flags,
directMemoryStart);
auto callerAddress = getCallerAddress(thread);
auto alignment = dmem::kPageSize;
{
auto unpacked = unpackMapFlags(flags, dmem::kPageSize);
alignment = unpacked.first;
flags = unpacked.second;
}
return ErrorCode::NOSYS;
len = rx::alignUp(len, dmem::kPageSize);
rx::EnumBitSet<AllocationFlags> allocFlags{};
if (!prot) {
// HACK
// FIXME: implement protect for pid
prot = vmem::Protection::CpuRead | vmem::Protection::CpuWrite |
vmem::Protection::GpuRead | vmem::Protection::GpuWrite;
}
if (flags & vmem::MapFlags::Fixed) {
allocFlags = AllocationFlags::Fixed;
} else if (addr == 0) {
addr = 0xfe0000000;
}
if (flags & vmem::MapFlags::NoOverwrite) {
allocFlags |= AllocationFlags::NoOverwrite;
}
auto name =
callerAddress ? rx::format("anon:{:012x}", callerAddress) : "<dmem unk>";
auto [range, errc] =
vmem::mapDirect(thread->tproc, addr,
rx::AddressRange::fromBeginSize(directMemoryStart, len),
prot, allocFlags, name, alignment, memoryType);
if (errc != ErrorCode{}) {
return errc;
}
thread->retval[0] = range.beginAddress();
return {};
}
orbis::SysResult orbis::sys_physhm_open(Thread *thread /* TODO */) {
return ErrorCode::NOSYS;
@ -1694,31 +1753,53 @@ orbis::SysResult orbis::sys_process_terminate(Thread *thread /* TODO */) {
return ErrorCode::NOSYS;
}
orbis::SysResult orbis::sys_blockpool_open(Thread *thread) {
if (auto blockpool_open = thread->tproc->ops->blockpool_open) {
rx::Ref<File> file;
auto result = blockpool_open(thread, &file);
if (result.isError()) {
return result;
}
if (!g_context->blockpool) {
return ErrorCode::NOSYS;
}
thread->retval[0] = thread->tproc->fileDescriptors.insert(file);
return {};
auto fd = thread->tproc->fileDescriptors.insert(g_context->blockpool);
if (fd == thread->tproc->fileDescriptors.npos) {
return ErrorCode::MFILE;
}
return ErrorCode::NOSYS;
thread->retval[0] = fd;
return {};
}
orbis::SysResult orbis::sys_blockpool_map(Thread *thread, caddr_t addr,
size_t len, sint prot, sint flags) {
if (auto blockpool_map = thread->tproc->ops->blockpool_map) {
return blockpool_map(thread, addr, len, prot, flags);
orbis::SysResult
orbis::sys_blockpool_map(Thread *thread, uintptr_t addr, size_t len,
MemoryType type, rx::EnumBitSet<vmem::Protection> prot,
rx::EnumBitSet<vmem::MapFlags> flags) {
rx::EnumBitSet<AllocationFlags> allocFlags{};
if (flags & vmem::MapFlags::Fixed) {
allocFlags |= AllocationFlags::Fixed;
}
return ErrorCode::NOSYS;
if (flags & vmem::MapFlags::NoOverwrite) {
allocFlags |= AllocationFlags::NoOverwrite;
}
if (flags & vmem::MapFlags::NoCoalesce) {
allocFlags |= AllocationFlags::NoMerge;
}
auto [range, errc] = vmem::commitPooled(
thread->tproc, rx::AddressRange::fromBeginSize(addr, len), type, prot);
if (errc != ErrorCode{}) {
return errc;
}
thread->retval[0] = range.beginAddress();
return {};
}
orbis::SysResult orbis::sys_blockpool_unmap(Thread *thread, caddr_t addr,
orbis::SysResult orbis::sys_blockpool_unmap(Thread *thread, uintptr_t addr,
size_t len, sint flags) {
if (auto blockpool_unmap = thread->tproc->ops->blockpool_unmap) {
return blockpool_unmap(thread, addr, len);
if (flags != 0) {
return ErrorCode::INVAL;
}
return ErrorCode::NOSYS;
return vmem::decommitPooled(thread->tproc,
rx::AddressRange::fromBeginSize(addr, len));
}
orbis::SysResult
orbis::sys_dynlib_get_info_for_libdbg(Thread *thread /* TODO */) {

6
kernel/orbis/src/sys/sys_sysctl.cpp
View file

@ -1,4 +1,5 @@
#include "KernelContext.hpp"
#include "rx/print.hpp"
#include "sys/sysproto.hpp"
#include "thread/Process.hpp"
#include "thread/Thread.hpp"
@ -749,9 +750,8 @@ SysResult kern_sysctl(Thread *thread, ptr<sint> name, uint namelen,
return ErrorCode::INVAL;
}
std::printf("Reporting stack at %p\n", thread->stackEnd);
*(ptr<void> *)old = thread->stackEnd;
return {};
rx::println(stderr, "Reporting stack at {:x}", thread->stackEnd);
return uwrite(ptr<uint64_t>(old), thread->stackEnd);
}
case sysctl_kern::smp_cpus:

219
kernel/orbis/src/sys/sys_vm_mmap.cpp
View file

@ -1,8 +1,16 @@
#include "KernelContext.hpp"
#include "error.hpp"
#include "rx/AddressRange.hpp"
#include "rx/align.hpp"
#include "rx/debug.hpp"
#include "rx/format.hpp"
#include "rx/print.hpp"
#include "sys/sysproto.hpp"
#include "thread/Process.hpp"
#include "thread/ProcessOps.hpp"
#include "thread/Thread.hpp"
#include "utils/Logs.hpp"
#include "vmem.hpp"
orbis::SysResult orbis::sys_sbrk(Thread *, sint) {
return ErrorCode::OPNOTSUPP;
@ -11,94 +19,179 @@ orbis::SysResult orbis::sys_sstk(Thread *, sint) {
return ErrorCode::OPNOTSUPP;
}
orbis::SysResult orbis::sys_mmap(Thread *thread, caddr_t addr, size_t len,
sint prot, sint flags, sint fd, off_t pos) {
if (auto impl = thread->tproc->ops->mmap) {
return impl(thread, addr, len, prot, flags, fd, pos);
orbis::SysResult orbis::sys_mmap(Thread *thread, uintptr_t addr, size_t len,
rx::EnumBitSet<vmem::Protection> prot,
rx::EnumBitSet<vmem::MapFlags> flags, sint fd,
off_t pos) {
std::uint64_t callerAddress = getCallerAddress(thread);
auto shift = addr & (vmem::kPageSize - 1);
addr = rx::alignDown(addr, vmem::kPageSize);
rx::EnumBitSet<AllocationFlags> allocFlags{};
rx::EnumBitSet<vmem::BlockFlags> blockFlags{};
rx::EnumBitSet<vmem::BlockFlagsEx> blockFlagsEx{};
std::uint64_t alignment = vmem::kPageSize;
{
auto unpacked = unpackMapFlags(flags, vmem::kPageSize);
alignment = unpacked.first;
flags = unpacked.second;
}
return ErrorCode::NOSYS;
len = rx::alignUp(len, vmem::kPageSize);
if (flags & vmem::MapFlags::Stack) {
allocFlags |= AllocationFlags::Stack;
}
if (flags & vmem::MapFlags::Fixed) {
allocFlags |= AllocationFlags::Fixed;
}
if (flags & vmem::MapFlags::NoOverwrite) {
allocFlags |= AllocationFlags::NoOverwrite;
}
if (flags & vmem::MapFlags::NoCoalesce) {
allocFlags |= AllocationFlags::NoMerge;
}
if (flags & vmem::MapFlags::Shared) {
blockFlagsEx |= vmem::BlockFlagsEx::Shared;
if (flags & vmem::MapFlags::Private) {
return ErrorCode::INVAL;
}
}
if (flags & vmem::MapFlags::Private) {
blockFlagsEx |= vmem::BlockFlagsEx::Private;
}
if (addr == 0) {
addr = flags & vmem::MapFlags::System ? 0xfc0000000 : 0x200000000;
}
if (flags & vmem::MapFlags::Void) {
flags |= vmem::MapFlags::Anon;
if (fd != -1 || pos != 0) {
return ErrorCode::INVAL;
}
}
if (flags & vmem::MapFlags::Stack) {
flags |= vmem::MapFlags::Anon;
blockFlags |= vmem::BlockFlags::Stack;
if (fd != -1 || pos != 0) {
return ErrorCode::INVAL;
}
if ((prot & (vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) !=
(vmem::Protection::CpuRead | vmem::Protection::CpuWrite)) {
return ErrorCode::INVAL;
}
}
auto name = callerAddress ? rx::format("anon:{:012x}", callerAddress) : "";
if (flags & vmem::MapFlags::Anon) {
if (fd != -1 || pos != 0) {
return ErrorCode::INVAL;
}
if (prot & (vmem::Protection::GpuRead | vmem::Protection::GpuWrite)) {
return ErrorCode::INVAL;
}
auto [range, errc] = vmem::mapFlex(thread->tproc, len, prot, addr,
allocFlags, blockFlags, name, alignment);
if (errc != orbis::ErrorCode{}) {
return errc;
}
thread->retval[0] = range.beginAddress() + shift;
return {};
}
auto file = thread->tproc->fileDescriptors.get(fd);
if (file == nullptr) {
return ErrorCode::BADF;
}
if (!file->device->blockFlags) {
blockFlags |= vmem::BlockFlags::FlexibleMemory;
}
prot &= ~vmem::Protection::CpuExec;
auto [range, errc] =
vmem::mapFile(thread->tproc, addr, len, allocFlags, prot, blockFlags,
blockFlagsEx, file.get(), pos, name, alignment);
if (errc != ErrorCode{}) {
return errc;
}
thread->retval[0] = range.beginAddress() + shift;
return {};
}
orbis::SysResult orbis::sys_freebsd6_mmap(Thread *thread, caddr_t addr,
size_t len, sint prot, sint flags,
orbis::SysResult orbis::sys_freebsd6_mmap(Thread *thread, uintptr_t addr,
size_t len,
rx::EnumBitSet<vmem::Protection> prot,
rx::EnumBitSet<vmem::MapFlags> flags,
sint fd, sint, off_t pos) {
return sys_mmap(thread, addr, len, prot, flags, fd, pos);
}
orbis::SysResult orbis::sys_msync(Thread *thread, ptr<void> addr, size_t len,
orbis::SysResult orbis::sys_msync(Thread *thread, uintptr_t addr, size_t len,
sint flags) {
if (auto impl = thread->tproc->ops->msync) {
return impl(thread, addr, len, flags);
}
return ErrorCode::NOSYS;
ORBIS_LOG_TODO(__FUNCTION__, addr, len, flags);
return {};
}
orbis::SysResult orbis::sys_munmap(Thread *thread, ptr<void> addr, size_t len) {
if (auto impl = thread->tproc->ops->munmap) {
return impl(thread, addr, len);
}
orbis::SysResult orbis::sys_munmap(Thread *thread, uintptr_t addr, size_t len) {
auto range = rx::AddressRange::fromBeginSize(addr, len);
return ErrorCode::NOSYS;
return vmem::unmap(thread->tproc, range);
}
orbis::SysResult orbis::sys_mprotect(Thread *thread, ptr<const void> addr,
size_t len, sint prot) {
if (auto impl = thread->tproc->ops->mprotect) {
return impl(thread, addr, len, prot);
}
return ErrorCode::NOSYS;
orbis::SysResult orbis::sys_mprotect(Thread *thread, uintptr_t addr, size_t len,
rx::EnumBitSet<vmem::Protection> prot) {
auto range = rx::AddressRange::fromBeginSize(addr, len);
return vmem::protect(thread->tproc, range, prot);
}
orbis::SysResult orbis::sys_minherit(Thread *thread, ptr<void> addr, size_t len,
orbis::SysResult orbis::sys_minherit(Thread *thread, uintptr_t addr, size_t len,
sint inherit) {
if (auto impl = thread->tproc->ops->minherit) {
return impl(thread, addr, len, inherit);
}
ORBIS_LOG_TODO(__FUNCTION__, addr, len, inherit);
return ErrorCode::NOSYS;
}
orbis::SysResult orbis::sys_madvise(Thread *thread, ptr<void> addr, size_t len,
orbis::SysResult orbis::sys_madvise(Thread *thread, uintptr_t addr, size_t len,
sint behav) {
if (auto impl = thread->tproc->ops->madvise) {
return impl(thread, addr, len, behav);
}
ORBIS_LOG_TODO(__FUNCTION__, addr, len, behav);
return ErrorCode::NOSYS;
}
orbis::SysResult orbis::sys_mincore(Thread *thread, ptr<const void> addr,
size_t len, ptr<char> vec) {
if (auto impl = thread->tproc->ops->mincore) {
return impl(thread, addr, len, vec);
}
orbis::SysResult orbis::sys_mincore(Thread *thread, uintptr_t addr, size_t len,
ptr<char> vec) {
ORBIS_LOG_TODO(__FUNCTION__, addr, len, vec);
return ErrorCode::NOSYS;
}
orbis::SysResult orbis::sys_mlock(Thread *thread, ptr<const void> addr,
size_t len) {
if (auto impl = thread->tproc->ops->mlock) {
return impl(thread, addr, len);
}
return ErrorCode::NOSYS;
orbis::SysResult orbis::sys_mlock(Thread *thread, uintptr_t addr, size_t len) {
ORBIS_LOG_TODO(__FUNCTION__, addr, len);
return {};
}
orbis::SysResult orbis::sys_mlockall(Thread *thread, sint how) {
if (auto impl = thread->tproc->ops->mlockall) {
return impl(thread, how);
}
return ErrorCode::NOSYS;
ORBIS_LOG_TODO(__FUNCTION__, how);
return {};
}
orbis::SysResult orbis::sys_munlockall(Thread *thread) {
if (auto impl = thread->tproc->ops->munlockall) {
return impl(thread);
}
return ErrorCode::NOSYS;
ORBIS_LOG_TODO(__FUNCTION__);
return {};
}
orbis::SysResult orbis::sys_munlock(Thread *thread, ptr<const void> addr,
orbis::SysResult orbis::sys_munlock(Thread *thread, uintptr_t addr,
size_t len) {
if (auto impl = thread->tproc->ops->munlock) {
return impl(thread, addr, len);
}
return ErrorCode::NOSYS;
ORBIS_LOG_TODO(__FUNCTION__, addr, len);
return {};
}

94
kernel/orbis/src/sysvec.cpp
View file

@ -1,10 +1,15 @@
#include "orbis-config.hpp"
#include "rx/EnumBitSet.hpp"
#include "rx/format.hpp"
#include "sys/syscall.hpp"
#include "sys/sysentry.hpp"
#include "sys/sysproto.hpp"
#include "thread/Process.hpp"
#include "thread/Thread.hpp"
#include <algorithm>
#include <type_traits>
#include <unordered_map>
#include <utility>
enum { PSL_C = 0x1 };
@ -101,6 +106,72 @@ struct WrapImpl<Fn> {
sysent result;
result.narg = sizeof...(Args);
result.call = &WrapImpl::call;
result.format = [](uint64_t *values) -> std::string {
std::string result = getSysentName(&WrapImpl::call);
result += "(";
auto formatArg =
[&]<std::size_t I, typename T>(std::integral_constant<std::size_t, I>,
T value, std::uint64_t raw) {
if (I != 0) {
result += ", ";
}
using type = std::remove_cvref_t<T>;
if constexpr (std::is_same_v<type, bool>) {
if (value) {
result += "true";
} else {
result += "false";
}
} else if constexpr (std::is_integral_v<type>) {
if (value > 9) {
result += rx::format("{:#x}", value);
} else {
result += rx::format("{}", value);
}
} else if constexpr (std::is_pointer_v<type>) {
result += rx::format("{}", (void *)value);
// using pointee = std::remove_cvref_t<std::remove_pointer_t<type>>;
// if constexpr (requires(rx::format_parse_context &ctx) {
// rx::formatter<pointee>().parse(ctx);
// }) {
// if (value) {
// pointee kernelValue;
// auto errc = orbis::uread(kernelValue, value);
// if (errc == ErrorCode{}) {
// result += rx::format("={}", kernelValue);
// } else {
// result += rx::format("={}", errc);
// }
// }
// }
} else if constexpr (requires(rx::format_parse_context &ctx) {
rx::formatter<type>().parse(ctx);
}) {
result += rx::format("{}", value);
} else {
if (raw > 9) {
result += rx::format("{:#x}", raw);
} else {
result += rx::format("{}", raw);
}
}
};
auto formatArgs = [&]<std::size_t... I>(std::index_sequence<I...>,
uint64_t *values) {
(formatArg(std::integral_constant<std::size_t, I>{},
makeArg<Args>(values[I]), values[I]),
...);
};
formatArgs(std::make_index_sequence<sizeof...(Args)>{}, values);
result += ")";
return result;
};
return result;
}
@ -113,7 +184,28 @@ private:
template <std::size_t... I>
static SysResult callImpl(Thread *thread, uint64_t *args,
std::index_sequence<I...>) {
return Fn(thread, Args(args[I])...);
return Fn(thread, makeArg<Args>(args[I])...);
}
template <typename T>
static T makeArg(uint64_t raw)
requires requires { std::bit_cast<T>(raw); }
{
return std::bit_cast<T>(raw);
}
template <typename T>
static T makeArg(uint64_t raw)
requires requires { T(raw); } && (!requires { std::bit_cast<T>(raw); })
{
return T(raw);
}
template <typename T>
static T makeArg(uint64_t raw)
requires requires { T(T::fromUnderlying(raw)); }
{
return T::fromUnderlying(raw);
}
};
} // namespace detail

6
kernel/orbis/src/thread/Process.cpp
View file

@ -177,3 +177,9 @@ orbis::Process *orbis::createProcess(Process *parentProcess, pid_t pid) {
g_processList->list = result;
return &result->object;
}
orbis::Budget *orbis::Process::getBudget() const {
auto result = g_context->budgets.get(budgetId);
return result != nullptr ? result.get() : nullptr;
}

32
kernel/orbis/src/thread/Thread.cpp
View file

@ -36,3 +36,35 @@ orbis::Thread *orbis::createThread(Process *process, std::string_view name) {
return result;
}
uintptr_t orbis::getCallerAddress(Thread *thread) {
if (!thread->context) {
return 0;
}
auto rbp = readRegister(thread->context, RegisterId::rbp);
std::uint64_t result = 0;
while (rbp < 0x8000'0000'0000) {
auto framePtr = std::bit_cast<ptr<uint64_t>>(rbp);
std::uint64_t nextFrame;
std::uint64_t retAddress;
if (orbis::uread(retAddress, framePtr + 1) != orbis::ErrorCode{}) {
break;
}
if (orbis::uread(nextFrame, framePtr) != orbis::ErrorCode{}) {
break;
}
if (!thread->tproc->libkernelRange.contains(retAddress)) {
result = retAddress;
break;
}
rbp = nextFrame;
}
return result;
}

1018
kernel/orbis/src/vmem.cpp
View file

File diff suppressed because it is too large Load diff