mirror of
https://github.com/RPCSX/rpcsx.git
synced 2026-04-04 14:08:37 +00:00
PKG: Pack multiple PKGs into fast+efficient install (#13147)
This commit is contained in:
Elad Ashkenazi
GitHub
parent
7423abb136
commit
7c2d6f8a23
4 changed files with 364 additions and 266 deletions
|
|
@ -13,6 +13,8 @@
|
|||
#include "util/sysinfo.hpp"
|
||||
#include "Loader/PSF.h"
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
LOG_CHANNEL(pkg_log, "PKG");
|
||||
|
||||
package_reader::package_reader(const std::string& path)
|
||||
|
|
@ -689,34 +691,174 @@ package_error package_reader::check_target_app_version() const
|
|||
return package_error::app_version;
|
||||
}
|
||||
|
||||
fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false);
|
||||
|
||||
usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic_t<double>& sync, thread_key thread_data_key, atomic_t<usz>& entry_indexer, std::vector<PKGEntry>& entries)
|
||||
bool package_reader::fill_data(std::map<std::string, install_entry*>& all_install_entries)
|
||||
{
|
||||
if (!m_is_valid)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
m_install_entries.clear();
|
||||
m_install_path.clear();
|
||||
m_bootable_file_path.clear();
|
||||
m_entry_indexer = 0;
|
||||
m_written_bytes = 0;
|
||||
|
||||
// Get full path and create the directory
|
||||
std::string dir = rpcs3::utils::get_hdd0_dir();
|
||||
|
||||
// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
|
||||
switch (m_metadata.content_type)
|
||||
{
|
||||
case PKG_CONTENT_TYPE_THEME:
|
||||
dir += "theme/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_WIDGET:
|
||||
dir += "widget/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_LICENSE:
|
||||
dir += "home/" + Emu.GetUsr() + "/exdata/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_VSH_MODULE:
|
||||
dir += "vsh/modules/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_PSN_AVATAR:
|
||||
dir += "home/" + Emu.GetUsr() + "/psn_avatar/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_VMC:
|
||||
dir += "tmp/vmc/";
|
||||
break;
|
||||
// TODO: Find out if other content types are installed elsewhere
|
||||
default:
|
||||
dir += "game/";
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO: Verify whether other content types require appending title ID
|
||||
if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE)
|
||||
dir += m_install_dir + '/';
|
||||
|
||||
// If false, an existing directory is being overwritten: cannot cancel the operation
|
||||
m_was_null = !fs::is_dir(dir);
|
||||
|
||||
if (!fs::create_path(dir))
|
||||
{
|
||||
pkg_log.error("Could not create the installation directory %s", dir);
|
||||
return false;
|
||||
}
|
||||
|
||||
m_install_path = dir;
|
||||
|
||||
if (!decrypt_data())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
usz num_failures = 0;
|
||||
|
||||
for (usz entry_index = entry_indexer++; num_failures == 0 && entry_index < entries.size(); entry_index = entry_indexer++)
|
||||
std::vector<PKGEntry> entries(m_header.file_count);
|
||||
|
||||
std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));
|
||||
|
||||
// Create directories first
|
||||
for (const auto& entry : entries)
|
||||
{
|
||||
const auto& entry = ::at32(entries, entry_index);
|
||||
|
||||
if ((entry.type & 0xff) == PKG_FILE_ENTRY_FOLDER || (entry.type & 0xff) == 0x12u)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.name_size > PKG_MAX_FILENAME_SIZE)
|
||||
{
|
||||
num_failures++;
|
||||
pkg_log.error("PKG name size is too big (0x%x)", entry.name_size);
|
||||
break;
|
||||
}
|
||||
|
||||
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
|
||||
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());
|
||||
|
||||
const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
|
||||
std::string path = dir + vfs::escape(name);
|
||||
|
||||
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
|
||||
|
||||
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
|
||||
|
||||
switch (const u8 entry_type = entry.type & 0xff)
|
||||
{
|
||||
case PKG_FILE_ENTRY_FOLDER:
|
||||
case 0x12:
|
||||
{
|
||||
if (fs::is_dir(path))
|
||||
{
|
||||
pkg_log.warning("Reused existing directory %s", path);
|
||||
}
|
||||
else if (fs::create_path(path))
|
||||
{
|
||||
pkg_log.notice("Created directory %s", path);
|
||||
}
|
||||
else
|
||||
{
|
||||
num_failures++;
|
||||
pkg_log.error("Failed to create directory %s", path);
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
const std::string true_path = std::filesystem::weakly_canonical(std::filesystem::u8path(path)).string();
|
||||
auto map_ptr = &*all_install_entries.try_emplace(true_path).first;
|
||||
|
||||
m_install_entries.push_back({ map_ptr, name, entry.file_offset, entry.file_size, entry.type, entry.pad });
|
||||
|
||||
if (map_ptr->second && !(entry.type & PKG_FILE_ENTRY_OVERWRITE))
|
||||
{
|
||||
// Cannot override
|
||||
continue;
|
||||
}
|
||||
|
||||
// Link
|
||||
map_ptr->second = &m_install_entries.back();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (num_failures != 0)
|
||||
{
|
||||
pkg_log.error("Package installation failed: %s", dir);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false);
|
||||
|
||||
usz package_reader::extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries)
|
||||
{
|
||||
usz num_failures = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
const usz maybe_index = m_entry_indexer++;
|
||||
|
||||
if (maybe_index >= m_install_entries.size())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
const install_entry& entry = ::at32(m_install_entries, maybe_index);
|
||||
|
||||
if (!entry.is_dominating())
|
||||
{
|
||||
// Overwritten by another entry
|
||||
m_written_bytes += entry.file_size;
|
||||
continue;
|
||||
}
|
||||
|
||||
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
|
||||
|
||||
std::span<const char> data_span = decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
|
||||
|
||||
const std::string name{data_span.data(), entry.name_size};
|
||||
const std::string path = dir + vfs::escape(name);
|
||||
const std::string& path = entry.weak_reference->first;
|
||||
const std::string& name = entry.name;
|
||||
|
||||
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
|
||||
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
|
||||
|
|
@ -761,7 +903,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
|
|||
{
|
||||
const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos);
|
||||
|
||||
data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
|
||||
const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
|
||||
|
||||
if (data_span.size() != block_size)
|
||||
{
|
||||
|
|
@ -777,18 +919,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
|
|||
break;
|
||||
}
|
||||
|
||||
if (sync.fetch_add((block_size + 0.0) / m_header.data_size) < 0.)
|
||||
{
|
||||
// Cancel the installation
|
||||
num_failures++;
|
||||
|
||||
if (was_null)
|
||||
{
|
||||
pkg_log.error("Package installation cancelled: %s", dir);
|
||||
out.close();
|
||||
return num_failures;
|
||||
}
|
||||
}
|
||||
m_written_bytes += block_size;
|
||||
}
|
||||
|
||||
if (is_buffered)
|
||||
|
|
@ -844,156 +975,57 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
|
|||
return num_failures;
|
||||
}
|
||||
|
||||
bool package_reader::extract_data(atomic_t<double>& sync)
|
||||
bool package_reader::extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths)
|
||||
{
|
||||
if (!m_is_valid)
|
||||
std::map<std::string, install_entry*> all_install_entries;
|
||||
|
||||
for (auto& reader : readers)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get full path and create the directory
|
||||
std::string dir = rpcs3::utils::get_hdd0_dir();
|
||||
|
||||
// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
|
||||
switch (m_metadata.content_type)
|
||||
{
|
||||
case PKG_CONTENT_TYPE_THEME:
|
||||
dir += "theme/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_WIDGET:
|
||||
dir += "widget/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_LICENSE:
|
||||
dir += "home/" + Emu.GetUsr() + "/exdata/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_VSH_MODULE:
|
||||
dir += "vsh/modules/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_PSN_AVATAR:
|
||||
dir += "home/" + Emu.GetUsr() + "/psn_avatar/";
|
||||
break;
|
||||
case PKG_CONTENT_TYPE_VMC:
|
||||
dir += "tmp/vmc/";
|
||||
break;
|
||||
// TODO: Find out if other content types are installed elsewhere
|
||||
default:
|
||||
dir += "game/";
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO: Verify whether other content types require appending title ID
|
||||
if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE)
|
||||
dir += m_install_dir + '/';
|
||||
|
||||
// If false, an existing directory is being overwritten: cannot cancel the operation
|
||||
const bool was_null = !fs::is_dir(dir);
|
||||
|
||||
if (!fs::create_path(dir))
|
||||
{
|
||||
pkg_log.error("Could not create the installation directory %s", dir);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!decrypt_data())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
atomic_t<usz> num_failures = 0;
|
||||
|
||||
std::vector<PKGEntry> entries(m_header.file_count);
|
||||
|
||||
std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));
|
||||
|
||||
// Create directories first
|
||||
for (const auto& entry : entries)
|
||||
{
|
||||
if (entry.name_size > PKG_MAX_FILENAME_SIZE)
|
||||
if (!reader.fill_data(all_install_entries))
|
||||
{
|
||||
num_failures++;
|
||||
pkg_log.error("PKG name size is too big (0x%x)", entry.name_size);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
usz num_failures = 0;
|
||||
|
||||
for (auto& reader : readers)
|
||||
{
|
||||
reader.m_bufs.resize(std::min<usz>(utils::get_thread_count(), reader.m_install_entries.size()));
|
||||
|
||||
atomic_t<usz> thread_indexer = 0;
|
||||
|
||||
named_thread_group workers("PKG Installer "sv, std::max<usz>(reader.m_bufs.size(), 1) - 1, [&]()
|
||||
{
|
||||
num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
|
||||
});
|
||||
|
||||
num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
|
||||
workers.join();
|
||||
|
||||
reader.m_bufs.clear();
|
||||
reader.m_bufs.shrink_to_fit();
|
||||
|
||||
if (num_failures)
|
||||
{
|
||||
if (reader.m_was_null)
|
||||
{
|
||||
fs::remove_all(reader.m_install_path, true);
|
||||
}
|
||||
|
||||
pkg_log.success("Package failed to install ('%s')", reader.m_install_path);
|
||||
break;
|
||||
}
|
||||
|
||||
switch (const u8 entry_type = entry.type & 0xff)
|
||||
else if (reader.get_progress(1) != 1)
|
||||
{
|
||||
case PKG_FILE_ENTRY_FOLDER:
|
||||
case 0x12:
|
||||
{
|
||||
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
|
||||
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());
|
||||
|
||||
const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
|
||||
const std::string path = dir + vfs::escape(name);
|
||||
|
||||
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
|
||||
|
||||
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
|
||||
|
||||
if (fs::is_dir(path))
|
||||
{
|
||||
pkg_log.warning("Reused existing directory %s", path);
|
||||
}
|
||||
else if (fs::create_path(path))
|
||||
{
|
||||
pkg_log.notice("Created directory %s", path);
|
||||
}
|
||||
else
|
||||
{
|
||||
num_failures++;
|
||||
pkg_log.error("Failed to create directory %s", path);
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
continue;
|
||||
}
|
||||
pkg_log.warning("Missing %d bytes from PKG total files size.", reader.m_header.data_size - reader.m_written_bytes);
|
||||
reader.m_written_bytes = reader.m_header.data_size; // Mark as completed anyway
|
||||
}
|
||||
|
||||
// May be empty
|
||||
bootable_paths.emplace_back(std::move(reader.m_bootable_file_path));
|
||||
}
|
||||
|
||||
if (num_failures != 0)
|
||||
{
|
||||
if (was_null)
|
||||
{
|
||||
fs::remove_all(dir, true);
|
||||
}
|
||||
|
||||
pkg_log.error("Package installation failed: %s", dir);
|
||||
return false;
|
||||
}
|
||||
|
||||
atomic_t<usz> entry_indexer = 0;
|
||||
atomic_t<usz> thread_indexer = 0;
|
||||
|
||||
m_bufs.resize(std::min<usz>(1 /*utils::get_thread_count()*/, entries.size()));
|
||||
|
||||
named_thread_group workers("PKG Installer "sv, std::max<usz>(m_bufs.size(), 1) - 1, [&]()
|
||||
{
|
||||
num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries);
|
||||
});
|
||||
|
||||
num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries);
|
||||
|
||||
workers.join();
|
||||
|
||||
if (num_failures == 0)
|
||||
{
|
||||
pkg_log.success("Package successfully installed to %s", dir);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (was_null)
|
||||
{
|
||||
fs::remove_all(dir, true);
|
||||
}
|
||||
|
||||
pkg_log.error("Package installation failed: %s", dir);
|
||||
}
|
||||
|
||||
m_bufs.clear();
|
||||
return num_failures == 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#include <span>
|
||||
#include <deque>
|
||||
|
||||
// Constants
|
||||
enum : u32
|
||||
|
|
@ -305,24 +306,57 @@ enum class package_error
|
|||
|
||||
class package_reader
|
||||
{
|
||||
struct thread_key
|
||||
{
|
||||
const usz unique_num = umax;
|
||||
};
|
||||
|
||||
struct install_entry
|
||||
{
|
||||
typename std::map<std::string, install_entry*>::value_type* weak_reference{};
|
||||
std::string name;
|
||||
|
||||
u64 file_offset{};
|
||||
u64 file_size{};
|
||||
u32 type{};
|
||||
u32 pad{};
|
||||
|
||||
// Check if the entry is the same one registered in entries to install
|
||||
bool is_dominating() const
|
||||
{
|
||||
return weak_reference->second == this;
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
package_reader(const std::string& path);
|
||||
~package_reader();
|
||||
|
||||
bool is_valid() const { return m_is_valid; }
|
||||
package_error check_target_app_version() const;
|
||||
bool extract_data(atomic_t<double>& sync);
|
||||
static bool extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths);
|
||||
psf::registry get_psf() const { return m_psf; }
|
||||
|
||||
std::string try_get_bootable_file_path_if_created_new() const
|
||||
int get_progress(int maximum = 100) const
|
||||
{
|
||||
return m_bootable_file_path;
|
||||
const usz wr = m_written_bytes;
|
||||
|
||||
return wr >= m_header.data_size ? maximum : ::narrow<int>(wr * maximum / m_header.data_size);
|
||||
}
|
||||
|
||||
struct thread_key
|
||||
void abort_extract()
|
||||
{
|
||||
const usz unique_num = umax;
|
||||
};
|
||||
m_entry_indexer.fetch_op([this](usz& v)
|
||||
{
|
||||
if (v < m_install_entries.size())
|
||||
{
|
||||
v = m_install_entries.size();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
bool read_header();
|
||||
|
|
@ -331,11 +365,18 @@ private:
|
|||
bool decrypt_data();
|
||||
void archive_seek(s64 new_offset, const fs::seek_mode damode = fs::seek_set);
|
||||
u64 archive_read(void* data_ptr, u64 num_bytes);
|
||||
bool fill_data(std::map<std::string, install_entry*>& all_install_entries);
|
||||
std::span<const char> archive_read_block(u64 offset, void* data_ptr, u64 num_bytes);
|
||||
std::span<const char> decrypt(u64 offset, u64 size, const uchar* key, thread_key thread_data_key = {0});
|
||||
usz extract_worker(const std::string& dir, bool was_null, atomic_t<double>& sync, thread_key thread_data_key, atomic_t<usz>& entry_indexer, std::vector<PKGEntry>& entries);
|
||||
usz extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries);
|
||||
|
||||
const usz BUF_SIZE = 8192 * 1024; // 8 MB
|
||||
std::deque<install_entry> m_install_entries;
|
||||
std::string m_install_path;
|
||||
atomic_t<usz> m_entry_indexer = 0;
|
||||
atomic_t<usz> m_written_bytes = 0;
|
||||
bool m_was_null = false;
|
||||
|
||||
static constexpr usz BUF_SIZE = 8192 * 1024; // 8 MB
|
||||
|
||||
bool m_is_valid = false;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue