ISO: Add ISO reader code and ISO device

Adds infrastructure to read files/data from ISOs.

Also adds classes extending fs::device_base and fs::file_base in order
to allow reading files through fs namespace functions. This approach
should allow ISO data to be read by the emulator with the least changes
to existing code.

Utilities/File.h

@@ -155,7 +155,7 @@ namespace fs
 	// Virtual device
 	struct device_base
 	{
-		const std::string fs_prefix;
+		std::string fs_prefix;
 
 		device_base();
 		virtual ~device_base();

rpcs3/Emu/CMakeLists.txt

@@ -125,6 +125,7 @@ target_sources(rpcs3_emu PRIVATE
     ../Loader/PSF.cpp
     ../Loader/PUP.cpp
     ../Loader/TAR.cpp
+    ../Loader/ISO.cpp
     ../Loader/TROPUSR.cpp
     ../Loader/TRP.cpp
 )

rpcs3/Loader/ISO.cpp

@@ -0,0 +1,600 @@
+#include "stdafx.h"
+
+#include "ISO.h"
+
+#include <codecvt>
+#include <algorithm>
+#include <cmath>
+#include <filesystem>
+
+// TODO: replace with file check for iso!
+bool is_file_iso(const std::string& path)
+{
+	if (fs::is_dir(path)) return false;
+
+	return is_file_iso(fs::file(path));
+}
+
+bool is_file_iso(const fs::file& file)
+{
+	if (!file) return false;
+	if (file.size() < 32768 + 6) return false;
+
+	file.seek(32768);
+
+	char magic[5];
+	file.read_at(32768 + 1, magic, 5);
+
+	return magic[0] == 'C' && magic[1] == 'D'
+		&& magic[2] == '0' && magic[3] == '0'
+		&& magic[4] == '1';
+}
+
+const int ISO_BLOCK_SIZE = 2048;
+
+template<typename T>
+inline T read_both_endian_int(fs::file& file)
+{
+	T out;
+
+	if (std::endian::little == std::endian::native)
+	{
+		out = file.read<T>();
+		file.seek(sizeof(T), fs::seek_cur);
+	}
+	else
+	{
+		file.seek(sizeof(T), fs::seek_cur);
+		out = file.read<T>();
+	}
+
+	return out;
+}
+
+// assumed that directory_entry is at file head
+std::optional<iso_fs_metadata> iso_read_directory_entry(fs::file& file, bool names_in_ucs2 = false)
+{
+	auto start_pos = file.pos();
+	u8 entry_length = file.read<u8>();
+
+	if (entry_length == 0) return std::nullopt;
+
+	file.seek(1, fs::seek_cur);
+	u32 start_sector = read_both_endian_int<u32>(file);
+	u32 file_size = read_both_endian_int<u32>(file);
+
+	std::tm file_date = {};
+	file_date.tm_year = file.read<u8>();
+	file_date.tm_mon = file.read<u8>() - 1;
+	file_date.tm_mday = file.read<u8>();
+	file_date.tm_hour = file.read<u8>();
+	file_date.tm_min = file.read<u8>();
+	file_date.tm_sec = file.read<u8>();
+	s16 timezone_value = file.read<u8>();
+	s16 timezone_offset = (timezone_value - 50) * 15 * 60;
+
+	std::time_t date_time = std::mktime(&file_date) + timezone_offset;
+
+	u8 flags = file.read<u8>();
+
+	// 2nd flag bit indicates whether a given fs node is a directory
+	bool is_directory = flags & 0b00000010;
+	bool has_more_extents = flags & 0b10000000;
+
+	file.seek(6, fs::seek_cur);
+
+	u8 file_name_length = file.read<u8>();
+
+	std::string file_name;
+	file.read(file_name, file_name_length);
+
+	if (file_name_length == 1 && file_name[0] == 0)
+	{
+		file_name = ".";
+	}
+	else if (file_name == "\1")
+	{
+		file_name = "..";
+	}
+	else if (names_in_ucs2) // for strings in joliet descriptor
+	{
+		std::string new_file_name = "";
+		int read = 0;
+		const u8* raw_str = reinterpret_cast<const u8*>(file_name.c_str());
+		while(read < file_name_length)
+		{
+			// characters are stored in big endian format.
+			const u16 upper = raw_str[read];
+			const u8 lower = raw_str[read + 1];
+
+			const u16 code_point = (upper << 8) + lower;
+
+			std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> convert;
+			new_file_name += convert.to_bytes(code_point);
+
+			read += 2;
+		}
+
+		file_name = new_file_name;
+	}
+
+	if (file_name.ends_with(";1"))
+	{
+		file_name.erase(file_name.end() - 2, file_name.end());
+	}
+
+	if (file_name_length > 1 && file_name.ends_with("."))
+	{
+		file_name.pop_back();
+	}
+
+	// skip the rest of the entry.
+	file.seek(entry_length + start_pos);
+
+	return iso_fs_metadata
+	{
+		.name = file_name,
+		.time = date_time,
+		.is_directory = is_directory,
+		.has_multiple_extents = has_more_extents,
+		.extents =
+		{
+			iso_extent_info
+			{
+				.start = start_sector,
+				.size = file_size
+			}
+		}
+	};
+}
+
+void iso_form_hierarchy(fs::file& file, iso_fs_node& node,
+	bool use_ucs2_decoding = false, std::string parent_path = "")
+{
+	if (!node.metadata.is_directory) return;
+
+	std::vector<int> multi_extent_node_indices;
+
+	// assuming the directory spans a single extent
+	const auto& directory_extent = node.metadata.extents[0];
+
+	file.seek(directory_extent.start * ISO_BLOCK_SIZE);
+
+	u64 end_pos = directory_extent.size + (directory_extent.start * ISO_BLOCK_SIZE);
+
+	while(file.pos() < end_pos)
+	{
+		auto entry = iso_read_directory_entry(file, use_ucs2_decoding);
+		if (!entry)
+		{
+			float block_size = ISO_BLOCK_SIZE;
+			float t = std::floor(file.pos() / block_size);
+			u64 new_sector = t+1;
+			file.seek(new_sector * ISO_BLOCK_SIZE);
+			continue;
+		}
+
+		bool extent_added = false;
+
+		// find previous extent and merge into it, otherwise we push this node's index
+		for (int index : multi_extent_node_indices)
+		{
+			auto& selected_node = node.children.at(index);
+			if (selected_node->metadata.name.compare(entry->name) == 0)
+			{
+				// merge into selected_node
+				selected_node->metadata.extents.push_back(entry->extents[0]);
+
+				extent_added = true;
+			}
+		}
+
+		if (extent_added) continue;
+
+		if (entry->has_multiple_extents)
+		{
+			// haven't pushed entry to node.children yet so node.children::size() == entry_index
+			multi_extent_node_indices.push_back(node.children.size());
+		}
+
+		node.children.push_back(std::make_unique<iso_fs_node>(iso_fs_node{
+			.metadata = *entry
+		}));
+	}
+
+	for (auto& child_node : node.children)
+	{
+		if (child_node->metadata.name != "." && child_node->metadata.name != "..")
+		{
+			iso_form_hierarchy(file, *child_node, use_ucs2_decoding, parent_path + "/" + node.metadata.name);
+		}
+	}
+}
+
+u64 iso_fs_metadata::size() const
+{
+	u64 total_size = 0;
+	for (const auto& extent : extents)
+	{
+		total_size += extent.size;
+	}
+
+	return total_size;
+}
+
+iso_archive::iso_archive(const std::string& path)
+{
+	m_path = path;
+	m_file = fs::file(path);
+
+	if (!is_file_iso(m_file))
+	{
+		// not iso... TODO: throw something??
+		return;
+	}
+
+	u8 descriptor_type = -2;
+	bool use_ucs2_decoding = false;
+
+	do
+	{
+		auto descriptor_start = m_file.pos();
+
+		descriptor_type = m_file.read<u8>();
+
+		// 1 = primary vol descriptor, 2 = joliet SVD
+		if (descriptor_type == 1 || descriptor_type == 2)
+		{
+			use_ucs2_decoding = descriptor_type == 2;
+
+			// skip the rest of descriptor's data
+			m_file.seek(155, fs::seek_cur);
+
+			m_root = iso_fs_node
+			{
+				.metadata = iso_read_directory_entry(m_file, use_ucs2_decoding).value(),
+			};
+
+			m_file.seek(descriptor_start);
+		}
+
+		m_file.seek(descriptor_start + ISO_BLOCK_SIZE);
+	}
+	while(descriptor_type != 255);
+
+	iso_form_hierarchy(m_file, m_root, use_ucs2_decoding);
+}
+
+iso_fs_node* iso_archive::retrieve(const std::string& passed_path)
+{
+	std::string path = std::filesystem::path(passed_path).lexically_normal().string();
+
+	size_t start = 0;
+	size_t end = path.find_first_of(fs::delim);
+
+	auto dir_entry = &m_root;
+
+	do
+	{
+		if (end == std::string::npos)
+		{
+			end = path.size();
+		}
+
+		auto path_component = path.substr(start, end-start);
+
+		bool found = false;
+		for (const auto& entry : dir_entry->children)
+		{
+			if (entry->metadata.name.compare(path_component) == 0)
+			{
+				dir_entry = entry.get();
+
+				start = end + 1;
+				end = path.find_first_of(fs::delim, start);
+				found = true;
+				break;
+			}
+		}
+
+		if (!found)
+		{
+			return nullptr;
+		}
+	}
+	while(start < path.size());
+
+	return dir_entry;
+}
+
+bool iso_archive::exists(const std::string& path)
+{
+	return retrieve(path) != nullptr;
+}
+
+bool iso_archive::is_file(const std::string& path)
+{
+	auto file_node = retrieve(path);
+	if (!file_node) return false;
+
+	return !file_node->metadata.is_directory;
+}
+
+iso_file iso_archive::open(const std::string& path)
+{
+	return iso_file(fs::file(m_path), *retrieve(path));
+}
+
+iso_file::iso_file(fs::file&& iso_handle, const iso_fs_node& node)
+	: m_file(std::move(iso_handle)), m_meta(node.metadata), m_pos(0)
+{
+	m_file.seek(ISO_BLOCK_SIZE * node.metadata.extents[0].start);
+}
+
+fs::stat_t iso_file::get_stat()
+{
+	return fs::stat_t
+	{
+		.is_directory = false,
+		.is_symlink = false,
+		.is_writable = false,
+		.size = size(),
+		.atime = m_meta.time,
+		.mtime = m_meta.time,
+		.ctime = m_meta.time
+	};
+}
+
+bool iso_file::trunc(u64)
+{
+	fs::g_tls_error = fs::error::readonly;
+	return false;
+}
+
+std::pair<u64, iso_extent_info> iso_file::get_extent_pos(u64 pos) const
+{
+	auto it = m_meta.extents.begin();
+
+	while(pos >= it->size && it < m_meta.extents.end() - 1)
+	{
+		pos -= it->size;
+
+		it++;
+	}
+
+	return {pos, *it};
+}
+
+// assumed valid and in bounds.
+u64 iso_file::file_offset(u64 pos) const
+{
+	auto [local_pos, extent] = get_extent_pos(pos);
+
+	return (extent.start * ISO_BLOCK_SIZE) + local_pos;
+}
+
+u64 iso_file::local_extent_remaining(u64 pos) const
+{
+	auto [local_pos, extent] = get_extent_pos(pos);
+
+	return extent.size - local_pos;
+}
+
+u64 iso_file::local_extent_size(u64 pos) const
+{
+	return get_extent_pos(pos).second.size;
+}
+
+u64 iso_file::read(void* buffer, u64 size)
+{
+	auto r = read_at(m_pos, buffer, size);
+	m_pos += r;
+	return r;
+}
+
+u64 iso_file::read_at(u64 offset, void* buffer, u64 size)
+{
+	const u64 bad_res = -1;
+	u64 local_remaining = local_extent_remaining(offset);
+
+	u64 total_read = m_file.read_at(file_offset(offset), buffer, std::min(size, local_remaining));
+	if (total_read == bad_res) return -1;
+
+	auto total_size = this->size();
+
+	if (size > total_read && (offset + total_read) < total_size)
+	{
+		u64 second_total_read = read_at(offset + total_read,
+			static_cast<char*>(buffer) + total_read,
+			size - total_read
+		);
+
+		if (second_total_read == bad_res) return -1;
+
+		return total_read + second_total_read;
+	}
+
+	return total_read;
+}
+
+u64 iso_file::write(const void*, u64)
+{
+	fs::g_tls_error = fs::error::readonly;
+	return 0;
+}
+
+u64 iso_file::seek(s64 offset, fs::seek_mode whence)
+{
+	const s64 total_size = size();
+	const s64 new_pos =
+		whence == fs::seek_set ? offset :
+		whence == fs::seek_cur ? offset + m_pos :
+		whence == fs::seek_end ? offset + total_size : -1;
+
+	if (new_pos < 0)
+	{
+		fs::g_tls_error = fs::error::inval;
+		return -1;
+	}
+
+	const u64 bad_res = -1;
+
+	u64 result = m_file.seek(file_offset(m_pos));
+	if (result == bad_res) return -1;
+
+	m_pos = new_pos;
+	return m_pos;
+}
+
+u64 iso_file::size()
+{
+	u64 extent_sizes = 0;
+	for (const auto& extent : m_meta.extents)
+	{
+		extent_sizes += extent.size;
+	}
+
+	return extent_sizes;
+}
+
+void iso_file::release()
+{
+	m_file.release();
+}
+
+bool iso_dir::read(fs::dir_entry& entry)
+{
+	if (m_pos < m_node.children.size())
+	{
+		auto& selected = m_node.children[m_pos].get()->metadata;
+		u64 size = selected.size();
+
+		entry.name = selected.name;
+		entry.atime = selected.time;
+		entry.mtime = selected.time;
+		entry.ctime = selected.time;
+		entry.is_directory = selected.is_directory;
+		entry.is_symlink = false;
+		entry.is_writable = false;
+		entry.size = size;
+
+		m_pos++;
+
+		return true;
+	}
+
+	return false;
+}
+
+bool iso_device::stat(const std::string& path, fs::stat_t& info)
+{
+	auto relative_path = std::filesystem::relative(std::filesystem::path(path),
+		std::filesystem::path(fs_prefix)).string();
+
+	auto node = m_archive.retrieve(relative_path);
+	if (!node)
+	{
+		fs::g_tls_error = fs::error::noent;
+		return false;
+	}
+
+	auto& meta = node->metadata;
+	u64 size = meta.size();
+
+	info = fs::stat_t
+	{
+		.is_directory = meta.is_directory,
+		.is_symlink = false,
+		.is_writable = false,
+		.size = size,
+		.atime = meta.time,
+		.mtime = meta.time,
+		.ctime = meta.time
+	};
+
+	return true;
+}
+
+bool iso_device::statfs(const std::string& path, fs::device_stat& info)
+{
+	auto relative_path = std::filesystem::relative(std::filesystem::path(path),
+		std::filesystem::path(fs_prefix)).string();
+
+	auto node = m_archive.retrieve(relative_path);
+	if (!node)
+	{
+		fs::g_tls_error = fs::error::noent;
+		return false;
+	}
+
+	auto& meta = node->metadata;
+	u64 size = meta.size();
+
+	info = fs::device_stat
+	{
+		.block_size=size,
+		.total_size=size,
+		.total_free=0,
+		.avail_free=0
+	};
+
+	return false;
+}
+
+std::unique_ptr<fs::file_base> iso_device::open(const std::string& path, bs_t<fs::open_mode> mode)
+{
+	auto relative_path = std::filesystem::relative(std::filesystem::path(path),
+		std::filesystem::path(fs_prefix)).string();
+
+	auto node = m_archive.retrieve(relative_path);
+	if (!node)
+	{
+		fs::g_tls_error = fs::error::noent;
+		return nullptr;
+	}
+
+	if (node->metadata.is_directory)
+	{
+		fs::g_tls_error = fs::error::isdir;
+		return nullptr;
+	}
+
+	return std::make_unique<iso_file>(fs::file(iso_path), *node);
+}
+
+std::unique_ptr<fs::dir_base> iso_device::open_dir(const std::string& path)
+{
+	auto relative_path = std::filesystem::relative(std::filesystem::path(path),
+		std::filesystem::path(fs_prefix)).string();
+
+	auto node = m_archive.retrieve(relative_path);
+	if (!node)
+	{
+		fs::g_tls_error = fs::error::noent;
+		return nullptr;
+	}
+
+	if (!node->metadata.is_directory)
+	{
+		// fs::dir::open -> ::readdir should return ENOTDIR when path is
+		// pointing to a file instead of a folder, which translates to error::unknown.
+		// doing the same here.
+		fs::g_tls_error = fs::error::unknown;
+	}
+
+	return std::make_unique<iso_dir>(*node);
+}
+
+void iso_dir::rewind()
+{
+	m_pos = 0;
+}
+
+void load_iso(const std::string& path)
+{
+	fs::set_virtual_device("iso_overlay_fs_dev",
+		stx::shared_ptr<iso_device>());
+
+	fs::set_virtual_device("iso_overlay_fs_dev",
+		stx::make_shared<iso_device>(path));
+}

rpcs3/Loader/ISO.h

@@ -0,0 +1,112 @@
+#pragma once
+
+#include "Utilities/File.h"
+#include "util/types.hpp"
+
+bool is_file_iso(const std::string& path);
+bool is_file_iso(const fs::file& path);
+
+void load_iso(const std::string& path);
+
+struct iso_extent_info
+{
+	u64 start;
+	u64 size;
+};
+
+struct iso_fs_metadata
+{
+	std::string name;
+	s64 time;
+	bool is_directory;
+	bool has_multiple_extents;
+	std::vector<iso_extent_info> extents;
+
+	u64 size() const;
+};
+
+struct iso_fs_node
+{
+	iso_fs_metadata metadata;
+	std::vector<std::unique_ptr<iso_fs_node>> children;
+};
+
+class iso_file : public fs::file_base
+{
+	fs::file m_file;
+	iso_fs_metadata m_meta;
+	u64 m_pos;
+
+	std::pair<u64, iso_extent_info> get_extent_pos(u64 pos) const;
+	u64 file_offset(u64 pos) const;
+	u64 local_extent_remaining(u64 pos) const;
+	u64 local_extent_size(u64 pos) const;
+
+	public:
+		iso_file(fs::file&& iso_handle, const iso_fs_node& node);
+
+		fs::stat_t get_stat() override;
+		bool trunc(u64 length) override;
+		u64 read(void* buffer, u64 size) override;
+		u64 read_at(u64 offset, void* buffer, u64 size) override;
+		u64 write(const void* buffer, u64 size) override;
+		u64 seek(s64 offset, fs::seek_mode whence) override;
+		u64 size() override;
+
+		void release() override;
+};
+
+class iso_dir : public fs::dir_base
+{
+	const iso_fs_node& m_node;
+	u64 m_pos;
+
+	public:
+		iso_dir(const iso_fs_node& node)
+			: m_node(node), m_pos(0)
+		{}
+
+		bool read(fs::dir_entry&) override;
+		void rewind() override;
+};
+
+// represents the .iso file itself.
+class iso_archive
+{
+	std::string m_path;
+	iso_fs_node m_root;
+	fs::file m_file;
+
+	public:
+		iso_archive(const std::string& path);
+
+		iso_fs_node* retrieve(const std::string& path);
+		bool exists(const std::string& path);
+		bool is_file(const std::string& path);
+
+		iso_file open(const std::string& path);
+};
+
+class iso_device : public fs::device_base
+{
+	iso_archive m_archive;
+	std::string iso_path;
+
+	public:
+		inline static std::string virtual_device_name = "/vfsv0_virtual_iso_overlay_fs_dev";
+
+		iso_device(const std::string& iso_path, const std::string& device_name = virtual_device_name)
+			: m_archive(iso_path), iso_path(iso_path)
+		{
+			fs_prefix = device_name;
+		}
+		~iso_device() override = default;
+
+		const std::string& get_loaded_iso() const { return iso_path; }
+
+		bool stat(const std::string& path, fs::stat_t& info) override;
+		bool statfs(const std::string& path, fs::device_stat& info) override;
+
+		std::unique_ptr<fs::file_base> open(const std::string& path, bs_t<fs::open_mode> mode) override;
+		std::unique_ptr<fs::dir_base> open_dir(const std::string& path) override;
+};

rpcs3/emucore.vcxproj

@@ -540,6 +540,7 @@
     <ClCompile Include="Loader\PSF.cpp" />
     <ClCompile Include="Loader\PUP.cpp" />
     <ClCompile Include="Loader\TAR.cpp" />
+    <ClCompile Include="Loader\ISO.cpp" />
     <ClCompile Include="Loader\mself.cpp" />
     <ClCompile Include="Loader\TROPUSR.cpp" />
     <ClCompile Include="Loader\TRP.cpp" />
@@ -1021,6 +1022,7 @@
     <ClInclude Include="Loader\PSF.h" />
     <ClInclude Include="Loader\PUP.h" />
     <ClInclude Include="Loader\TAR.h" />
+    <ClInclude Include="Loader\ISO.h" />
     <ClInclude Include="Loader\TROPUSR.h" />
     <ClInclude Include="Loader\TRP.h" />
     <ClInclude Include="rpcs3_version.h" />
@@ -1096,4 +1098,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>