feat(libSceNpManager): retain the game presence callback and validate content restrictions offline

CMakeLists.txt

@@ -87,6 +87,10 @@ if(BUILD_TESTING)
     set_tests_properties(request_serializer PROPERTIES TIMEOUT 20)
     find_package(Python3 QUIET COMPONENTS Interpreter)
     if(Python3_Interpreter_FOUND)
+        add_test(NAME shader_cache_native_paths COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND})
+        set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20)
         add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
         add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}
@@ -115,6 +119,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_wrapped_declarations.py)
         add_test(NAME conventions_ci_gate COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_ci_gate.py)
+        add_test(NAME conventions_doc_links COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_doc_links.py)
         add_test(NAME test_registration COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_registration.py)
         add_test(NAME check_skips COMMAND ${Python3_EXECUTABLE}

core/libs/CMakeLists.txt

@@ -58,6 +58,21 @@ if(MINGW)
     foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static)
         string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
     endforeach()
+
+    if(NOT EXISTS "${CMAKE_RANLIB}")
+        message(FATAL_ERROR "Stable import libraries require ranlib")
+    endif()
+    list(LENGTH CMAKE_CXX_CREATE_SHARED_LIBRARY linkCommandCount)
+    string(FIND "${CMAKE_CXX_CREATE_SHARED_LIBRARY}" "-Wl,--out-implib,<TARGET_IMPLIB> " implibOption)
+    if(NOT linkCommandCount EQUAL 1 OR implibOption EQUAL -1)
+        message(FATAL_ERROR "Expected one shared library link command that writes -Wl,--out-implib,<TARGET_IMPLIB>: ${CMAKE_CXX_CREATE_SHARED_LIBRARY}")
+    endif()
+    string(REPLACE "-Wl,--out-implib,<TARGET_IMPLIB> " "-Wl,--out-implib,<TARGET_IMPLIB>.new " linkCommand "${CMAKE_CXX_CREATE_SHARED_LIBRARY}")
+    set(CMAKE_CXX_CREATE_SHARED_LIBRARY
+            "\"${CMAKE_COMMAND}\" -E env SOURCE_DATE_EPOCH=0 ${linkCommand}"
+            "\"${CMAKE_RANLIB}\" -D <TARGET_IMPLIB>.new"
+            "\"${CMAKE_COMMAND}\" -E copy_if_different <TARGET_IMPLIB>.new <TARGET_IMPLIB>"
+    )
 endif()
 
 set(LIBC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/prx/libc)
@@ -87,6 +102,14 @@ set_target_properties(libc PROPERTIES
 
 set(LIBC_UNPATCHED ${UNPATCHED_DIR}/libc.prx)
 set(LIBC_PATCHED ${PATCHED_DIR}/libc.prx)
+set(LIBC_EXPORT_LIST ${CMAKE_CURRENT_BINARY_DIR}/libc.exports)
+
+add_custom_command(
+        OUTPUT ${LIBC_EXPORT_LIST}
+        COMMAND $<TARGET_FILE:nid_patcher> --write-export-list ${LIBC_UNPATCHED} ${LIBC_EXPORT_LIST}
+        DEPENDS nid_patcher libc ${LIBC_UNPATCHED}
+        COMMENT "Listing libc exports"
+)
 
 add_custom_command(
         OUTPUT ${LIBC_PATCHED}
@@ -146,8 +169,8 @@ foreach(LIB_DIR ${LIBS_DIRS})
         add_custom_command(
                 OUTPUT ${PATCHED_FILE}
                 COMMAND ${CMAKE_COMMAND} -E copy ${UNPATCHED_FILE} ${PATCHED_FILE}
-                COMMAND $<TARGET_FILE:nid_patcher> ${LIB_NAME} --preserve-exports "${LIBC_UNPATCHED}" "${PATCHED_FILE}"
-                DEPENDS nid_patcher ${LIB_NAME} ${UNPATCHED_FILE} ${LIBC_UNPATCHED}
+                COMMAND $<TARGET_FILE:nid_patcher> ${LIB_NAME} --preserve-export-list "${LIBC_EXPORT_LIST}" "${PATCHED_FILE}"
+                DEPENDS nid_patcher ${LIB_NAME} ${UNPATCHED_FILE} ${LIBC_EXPORT_LIST}
                 COMMENT "Patching NIDs for ${LIB_NAME}"
         )
 

core/libs/SceTypes.hpp

@@ -1314,7 +1314,20 @@ struct Http2AsyncOption {
 
 struct NpTitleId { char data[13]; char pad[3]; };
 struct NpTitleSecret { std::uint8_t data[128]; };
-struct NpContentRestriction { std::uint8_t opaque[128]; };
+struct NpAgeRestriction {
+    char countryCode[4];
+    std::int8_t age;
+    std::uint8_t padding[3];
+};
+struct NpContentRestriction {
+    std::uint64_t size;
+    std::int8_t defaultAgeRestriction;
+    std::uint8_t padding[3];
+    std::int32_t ageRestrictionCount;
+    const NpAgeRestriction* ageRestriction;
+};
+static_assert(sizeof(NpAgeRestriction) == 8);
+static_assert(sizeof(NpContentRestriction) == 0x18);
 struct NpOnlineId { char data[17]; char pad[3]; };
 
 struct SharePlayConnectionInfoA {
@@ -1750,32 +1763,43 @@ struct JpegEncOutputInfo {
 };
 
 struct JpegDecCreateParam {
-    std::uint32_t size;
+    std::uint32_t this_size;
     std::uint32_t attribute;
+    std::uint32_t max_image_width;
 };
 
 struct JpegDecParseParam {
     const void* jpeg_mem_addr;
     std::uint32_t jpeg_mem_size;
-    std::uint32_t reserved0;
+    std::uint16_t decode_mode;
+    std::uint16_t down_scale;
 };
 
 struct JpegDecDecodeParam {
     const void* jpeg_mem_addr;
     void* image_mem_addr;
+    void* coefficient_mem_addr;
     std::uint32_t jpeg_mem_size;
     std::uint32_t image_mem_size;
+    std::uint32_t coefficient_mem_size;
+    std::uint16_t decode_mode;
+    std::uint16_t down_scale;
     std::uint16_t pixel_format;
-    std::uint16_t reserved0;
+    std::uint16_t alpha_value;
     std::uint32_t image_pitch;
 };
 
 struct JpegDecImageInfo {
     std::uint32_t image_width;
     std::uint32_t image_height;
     std::uint16_t color_space;
-    std::uint16_t bit_depth;
-    std::uint32_t image_flag;
+    std::uint16_t component_number;
+    std::uint8_t component_id[4];
+    std::uint8_t sampling_factor[4];
+    std::uint32_t coefficient_mem_size;
+    std::uint32_t suitable_csc_attribute;
+    std::uint32_t output_image_width;
+    std::uint32_t output_image_height;
 };
 
 struct PlayGoInitParams {

core/libs/nid/include/nid/ExportExclusions.hpp

@@ -8,6 +8,8 @@ namespace Nid {
 
 std::string NormalizeExportName(const std::string& name);
 std::unordered_set<std::string> ReadExportExclusions(const std::string& path);
+std::unordered_set<std::string> ReadExportList(const std::string& path);
+bool WriteExportList(const std::string& referencePath, const std::string& path);
 
 }
 

core/libs/nid/src/ExportExclusions.cpp

@@ -4,10 +4,13 @@
 #include <nid/MachONidPatcher.hpp>
 #include <nid/NidPatcherUtils.hpp>
 #include <nid/NativePath.hpp>
+#include <algorithm>
 #include <bit>
 #include <cstring>
 #include <fstream>
+#include <iterator>
 #include <stdexcept>
+#include <vector>
 
 namespace Nid {
 
@@ -113,6 +116,24 @@ std::unordered_set<std::string> ReadElfExports(const std::vector<std::uint8_t>&
     return result;
 }
 
+std::string ReadText(const std::string& path, std::ifstream& input) {
+    std::string text{std::istreambuf_iterator<char>(input), {}};
+    if (input.bad()) throw std::runtime_error("cannot read export list: " + path);
+    return text;
+}
+
+std::string FormatExportList(const std::unordered_set<std::string>& exports) {
+    std::vector<std::string> names(exports.begin(), exports.end());
+    std::ranges::sort(names);
+    std::string text;
+    for (const auto& name : names) {
+        if (name.find('\n') != std::string::npos) throw std::runtime_error("export name contains a line break: " + name);
+        text += name;
+        text += '\n';
+    }
+    return text;
+}
+
 }
 
 std::string NormalizeExportName(const std::string& name) {
@@ -153,4 +174,31 @@ std::unordered_set<std::string> ReadExportExclusions(const std::string& path) {
     return result;
 }
 
+std::unordered_set<std::string> ReadExportList(const std::string& path) {
+    std::ifstream input(path, std::ios::binary);
+    if (!input) throw std::runtime_error("cannot open export list: " + path);
+    const auto text = ReadText(path, input);
+    if (text.empty() || text.back() != '\n') throw std::runtime_error("malformed export list: " + path);
+    std::unordered_set<std::string> result;
+    for (std::size_t start = 0; start < text.size();) {
+        const auto end = text.find('\n', start);
+        if (end == start) throw std::runtime_error("empty name in export list: " + path);
+        auto name = text.substr(start, end - start);
+        if (!result.insert(name).second) throw std::runtime_error("duplicate name in export list: " + name);
+        start = end + 1;
+    }
+    return result;
+}
+
+bool WriteExportList(const std::string& referencePath, const std::string& path) {
+    const auto text = FormatExportList(ReadExportExclusions(referencePath));
+    if (std::ifstream existing(path, std::ios::binary); existing && ReadText(path, existing) == text) return false;
+    std::ofstream output(path, std::ios::binary | std::ios::trunc);
+    if (!output) throw std::runtime_error("cannot write export list: " + path);
+    output.write(text.data(), static_cast<std::streamsize>(text.size()));
+    output.close();
+    if (!output) throw std::runtime_error("cannot write export list: " + path);
+    return true;
+}
+
 }

core/libs/nid/src/main.cpp

@@ -25,21 +25,36 @@ void _writeFile(const std::string& path, const std::vector<std::uint8_t>& data)
 
 int Run(int argc, char* argv[]) {
     if (argc < 3) {
-        std::cerr << "Usage: nid_patcher <library_name> [--preserve-exports <unpatched-library>] <file> [file2 ...]\n";
+        std::cerr << "Usage: nid_patcher <library_name> [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
+                     "       nid_patcher --write-export-list <unpatched-library> <export-list>\n";
         return 1;
     }
 
+    if (std::string(argv[1]) == "--write-export-list") {
+        if (argc != 4) {
+            std::cerr << "FAIL: --write-export-list requires a reference library and an export list\n";
+            return 2;
+        }
+        try {
+            std::cout << (Nid::WriteExportList(argv[2], argv[3]) ? "WROTE: " : "UNCHANGED: ") << argv[3] << "\n";
+        } catch (const std::exception& e) {
+            std::cerr << "FAIL: " << argv[3] << ": " << e.what() << "\n";
+            return 2;
+        }
+        return 0;
+    }
+
     const std::string libraryName = argv[1];
     std::unordered_set<std::string> excludedExports;
     std::vector<std::string> paths;
     try {
         bool hasReference = false;
         for (int i = 2; i < argc; ++i) {
             const std::string argument = argv[i];
-            if (argument == "--preserve-exports") {
-                if (hasReference || i + 1 == argc) throw std::runtime_error("--preserve-exports requires exactly one reference library");
+            if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
+                if (hasReference || i + 1 == argc) throw std::runtime_error(argument + " requires exactly one reference");
                 hasReference = true;
-                excludedExports = Nid::ReadExportExclusions(argv[++i]);
+                excludedExports = argument == "--preserve-exports" ? Nid::ReadExportExclusions(argv[++i]) : Nid::ReadExportList(argv[++i]);
             } else {
                 if (argument.starts_with("--")) throw std::runtime_error("unknown option: " + argument);
                 paths.push_back(argument);

core/libs/nid/tests/test_macho_nids.py

@@ -1,6 +1,7 @@
 """Check nid_patcher on small byte-built Mach-O dylibs: the export trie, the chained imports and what it refuses."""
 
 import hashlib
+import os
 import struct
 import subprocess
 import sys
@@ -211,6 +212,53 @@ def check_preserved(patcher, directory):
     assert exports_of(path.read_bytes()) == {"_puts_nid_postfix": (0, 0x100), "_" + nid("exit"): (0, 0x108)}, exports_of(path.read_bytes())
 
 
+def write_export_list(patcher, reference, listing):
+    return subprocess.run([str(patcher), "--write-export-list", str(reference), str(listing)], capture_output=True, text=True, timeout=30)
+
+
+def check_export_list(patcher, directory):
+    reference = Path(directory) / "listed.dylib"
+    listing = Path(directory) / "listed.exports"
+    reference.write_bytes(build_dylib([("_puts_nid_postfix", 0, 0x100), ("_exit", 0, 0x108)], []))
+    result = write_export_list(patcher, reference, listing)
+    assert result.returncode == 0, (result.stdout, result.stderr)
+    assert listing.read_bytes() == b"exit\nputs\n", listing.read_bytes()
+
+    os.utime(listing, ns=(1_000_000_000_000_000_000, 1_000_000_000_000_000_000))
+    reference.write_bytes(build_dylib([("_exit", 0, 0x110), ("_puts_nid_postfix", 0, 0x118)], []))
+    result = write_export_list(patcher, reference, listing)
+    assert result.returncode == 0, (result.stdout, result.stderr)
+    assert listing.stat().st_mtime_ns == 1_000_000_000_000_000_000, "an unchanged export list was rewritten"
+
+    reference.write_bytes(build_dylib([("_puts", 0, 0x100)], []))
+    result = write_export_list(patcher, reference, listing)
+    assert result.returncode == 0, (result.stdout, result.stderr)
+    assert listing.read_bytes() == b"puts\n", listing.read_bytes()
+    assert listing.stat().st_mtime_ns != 1_000_000_000_000_000_000, "a changed export list kept its time"
+
+    path = Path(directory) / "listed-preserve.dylib"
+    path.write_bytes(build_dylib([("_puts_nid_postfix", 0, 0x100), ("_exit_nid_postfix", 0, 0x108)], []))
+    result = subprocess.run([str(patcher), "libtest", "--preserve-export-list", str(listing), str(path)], capture_output=True, text=True, timeout=30)
+    assert result.returncode == 0, (result.stdout, result.stderr)
+    assert exports_of(path.read_bytes()) == {"_puts_nid_postfix": (0, 0x100), "_" + nid("exit"): (0, 0x108)}, exports_of(path.read_bytes())
+
+    malformed_lists = [
+        ("unterminated", b"puts", "malformed export list"),
+        ("blank", b"puts\n\nexit\n", "empty name in export list"),
+        ("twice", b"puts\nputs\n", "duplicate name in export list"),
+        ("empty", b"", "malformed export list"),
+    ]
+    for name, content, message in malformed_lists:
+        malformed = Path(directory) / (name + ".exports")
+        malformed.write_bytes(content)
+        target = Path(directory) / (name + ".dylib")
+        target.write_bytes(build_dylib([("_puts_nid_postfix", 0, 0x100)], []))
+        result = subprocess.run([str(patcher), "libtest", "--preserve-export-list", str(malformed), str(target)], capture_output=True, text=True, timeout=30)
+        assert result.returncode != 0 and message in result.stdout + result.stderr, (name, result.stdout, result.stderr)
+    result = subprocess.run([str(patcher), "libtest", "--preserve-exports", str(reference), "--preserve-export-list", str(listing), str(path)], capture_output=True, text=True, timeout=30)
+    assert result.returncode != 0 and "exactly one reference" in result.stdout + result.stderr, (result.stdout, result.stderr)
+
+
 def check_refusals(patcher, directory):
     growing = [("_ab", 0, 0x100), ("_cd", 0, 0x108)]
     result, _ = patch(patcher, directory, "signed.dylib", build_dylib(growing, [], signed=True))
@@ -227,6 +275,7 @@ def main():
         check_shrinking(patcher, directory)
         check_growing(patcher, directory)
         check_preserved(patcher, directory)
+        check_export_list(patcher, directory)
         check_refusals(patcher, directory)
     print("NID patcher Mach-O tests passed")
 

core/libs/prx/libSceAgc/DcbState/src/Init.cpp

@@ -7,21 +7,18 @@
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 
-static constexpr int SCE_AGC_ERROR_INVALID_ARGUMENT = static_cast<int>(0x80890001);
-
 extern "C" {
 
-int APS5_VABI sceAgcInit_0090(std::uint32_t* state, std::uint32_t version) {
-    Agc::Command::CheckAddress(reinterpret_cast<std::uintptr_t>(state), alignof(std::uint32_t), __func__);
-    Agc::Command::Require(version < 14, __func__, "unsupported AGC initialization version");
-    return 0;
-}
-
 int APS5_VABI sceAgcInit(std::uint32_t version) {
-    if (version >= 14) return SCE_AGC_ERROR_INVALID_ARGUMENT;
+    if (version >= 14) return 0;
     Agc::Command::GetRegisterDefaults(version, false, __func__);
     Agc::Command::GetRegisterDefaults(version, true, __func__);
     return 0;
 }
 
+int APS5_VABI sceAgcInit_0090(std::uint32_t* state, std::uint32_t version) {
+    Agc::Command::CheckAddress(reinterpret_cast<std::uintptr_t>(state), alignof(std::uint32_t), __func__);
+    return sceAgcInit(version);
+}
+
 }

core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp

@@ -72,7 +72,7 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader*
         throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32");
     }
 
-    AgcDriver::ShaderPreparationScope transaction;
+    AgcDriver::ShaderPreparationScope transaction({ps != nullptr ? gs : nullptr, ps});
     std::array<ShaderRegister, 32> values{};
     auto* output = regs;
     regs = values.data();

core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp

@@ -22,7 +22,7 @@ extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegist
 
     // The simple vertex/pixel pipeline combines two primitive context registers
     // and 32 interpolant registers. Publish only after both helpers succeed.
-    AgcDriver::ShaderPreparationScope transaction;
+    AgcDriver::ShaderPreparationScope transaction({vertex, pixel});
     std::array<ShaderRegister, 34> contextValues{};
     std::array<ShaderRegister, 3> primitiveValues{};
     sceAgcCreatePrimState(contextValues.data(), primitiveValues.data(), nullptr, vertex, primitiveType);

core/libs/prx/libSceAgc/Shader/src/PrimState.cpp

@@ -8,6 +8,7 @@
 
 #include "SceShaders.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp"
 #include "prx/libSceAgc/Shader/include/ShaderUtils.hpp"
 #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp"
 
@@ -56,7 +57,9 @@ int APS5_VABI sceAgcCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_
     const std::array<const Shader*, 2> stages{hs, gs};
     const bool patch = prim_type == static_cast<std::uint32_t>(ShaderRegs::PrimitiveType::Patch);
     if (patch == ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_HS_BIT) != 0)) {
+        AgcDriver::ShaderPreparationScope transaction({hs, gs});
         AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues);
+        transaction.Commit();
     }
     if (cx_regs != nullptr) std::copy(contextValues.begin(), contextValues.end(), cx_regs);
     if (uc_regs != nullptr) std::copy(primitiveValues.begin(), primitiveValues.end(), uc_regs);