feat(shader): publish and load prebuilt resource plans

CMakeLists.txt

@@ -260,3 +260,4 @@ add_subdirectory(core/Decoder/Jpeg)
 add_subdirectory(core/Decoder/Png)
 add_subdirectory(core/libs)
 add_subdirectory(core/relinker)
+add_subdirectory(tools/shader-archive EXCLUDE_FROM_ALL)

core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp

@@ -1989,7 +1989,11 @@ int runPlanLoadingProcess(std::string_view mode) {
         std::cout << "a process with another switch value rebuilt all " << count << " plans\n";
         return 0;
     }
-    if (mode == "--load-plan") {
+    const bool prebuilt = mode == "--load-plan-pack" || mode == "--load-plan-loose";
+    if (prebuilt || mode == "--load-plan-pack-corrupt") {
+        require(builds == 0u && totals.planHits == (prebuilt ? 0u : count) && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "prebuilt plans did not preserve front-end reuse" + counts);
+        require(totals.prebuiltPlanHits == (prebuilt ? count : 0u) && totals.prebuiltPlanMisses == (prebuilt ? 0u : count) && totals.prebuiltPlanRejected == (prebuilt ? 0u : count), "prebuilt plan counters or rejection fallback differ");
+    } else if (mode == "--load-plan") {
         require(builds == 0u && totals.planHits == count && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "the second process ran the front end" + counts);
     } else if (mode == "--load-plan-off") {
         require(builds == count && totals.planHits == 0u && totals.planMisses == 0u && totals.planWrites == 0u && totals.hits == count && totals.misses == 0u, "APS5_NO_PERF_DISK_PLANS=1 used the stored plans or missed the stored variants" + counts);
@@ -2012,6 +2016,8 @@ int runPlanLoadingProcess(std::string_view mode) {
         std::cout << "second process prepared " << count << " shaders from stored plans without running the front end\n";
     } else if (mode == "--load-plan-off") {
         std::cout << "APS5_NO_PERF_DISK_PLANS=1 ran the front end for all " << count << " shaders with the same output\n";
+    } else if (prebuilt || mode == "--load-plan-pack-corrupt") {
+        std::cout << "prebuilt resource plans preserved the first process's outputs without front-end builds\n";
     } else {
         std::cout << "a damaged plan entry was rejected, rebuilt and stored again with the same output\n";
     }
@@ -2053,6 +2059,55 @@ void verifyPlanAcrossProcesses(const char* self) {
     };
     run("--load-plan");
     run("--load-plan-off");
+    const auto prebuiltRoot = ShaderDiskCache::EntryDirectory().parent_path() / "prebuilt";
+    std::filesystem::create_directories(prebuiltRoot);
+    std::vector<std::pair<std::filesystem::path, std::vector<std::byte>>> packedPlans;
+    for (std::size_t index = 0; index < fixtures.size(); ++index) {
+        std::vector<std::byte> key, entry;
+        ShaderDiskCache::BuildPlanKey(fixtures[index].Request(true), 32u, key);
+        require(ShaderDiskCache::IsPlanKey(key), "the plan key lost its entry kind");
+        require(ReadWholeFile(planEntryPath(fixtures[index]), entry), "cannot read plan pack input");
+        const auto pack = prebuiltRoot / ("plan-" + std::to_string(index) + ".pack");
+        auto bytes = packBytes({{key, entry}});
+        require(WriteFileAtomically(pack, bytes), "cannot write plan pack fixture");
+        packedPlans.emplace_back(pack, std::move(bytes));
+    }
+    const auto runWithoutWritablePlans = [&](const char* mode) {
+        std::vector<std::pair<std::filesystem::path, std::vector<std::byte>>> originals;
+        for (auto& fixture : fixtures) {
+            const auto path = planEntryPath(fixture);
+            std::vector<std::byte> bytes;
+            require(ReadWholeFile(path, bytes), "cannot preserve writable plan fixture");
+            originals.emplace_back(path, std::move(bytes));
+        }
+        const auto restore = [&] {
+            for (const auto& [path, bytes] : originals) require(WriteFileAtomically(path, bytes), "cannot restore writable plan fixture");
+        };
+        try {
+            for (const auto& [path, bytes] : originals) require(std::filesystem::remove(path), "cannot remove writable plan fixture");
+            run(mode);
+        } catch (...) {
+            restore();
+            throw;
+        }
+        restore();
+    };
+    runWithoutWritablePlans("--load-plan-pack");
+    for (auto& [path, bytes] : packedPlans) {
+        bytes.back() ^= std::byte{1};
+        require(WriteFileAtomically(path, bytes), "cannot corrupt plan pack fixture");
+    }
+    run("--load-plan-pack-corrupt");
+    for (const auto& [path, bytes] : packedPlans) std::filesystem::remove(path);
+    std::vector<std::filesystem::path> loosePlans;
+    for (auto& fixture : fixtures) {
+        const auto source = planEntryPath(fixture);
+        const auto loose = prebuiltRoot / source.filename();
+        std::filesystem::copy_file(source, loose);
+        loosePlans.push_back(loose);
+    }
+    runWithoutWritablePlans("--load-plan-loose");
+    for (const auto& path : loosePlans) std::filesystem::remove(path);
     std::vector<std::byte> entry;
     const auto damaged = planEntryPath(fixtures.front());
     require(ReadWholeFile(damaged, entry), "cannot read a stored plan entry");

core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp

@@ -52,6 +52,11 @@ struct ResourceCapture {
 // the target); the vertex stages' input validation is repeated per capture because it reads V#
 // fields the key does not cover. Null for a request that bypasses the cache (useCache false).
 struct CompiledVariant;
+struct OfflineCompileTimings {
+    double frontendSeconds = 0;
+    double backendSeconds = 0;
+};
+[[nodiscard]] std::shared_ptr<const CompiledVariant> CompileShaderUncached(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, OfflineCompileTimings* timings = nullptr);
 struct SourceHandle {
     std::shared_ptr<SourceEntry> source;
     std::shared_ptr<const CompiledVariant> artifact;

core/shader/recompiler/Recompiler.cpp

@@ -1363,6 +1363,20 @@ std::shared_ptr<const SourceHandle> PrepareShader(const RecompileRequest& reques
     return PrepareShader(request, nullptr);
 }
 
+std::shared_ptr<const CompiledVariant> CompileShaderUncached(const RecompileRequest& request, ShaderPreparationContext* preparation, OfflineCompileTimings* timings) {
+    return recompileReporting(request, [&] {
+        static_cast<void>(RequestInputInfo(request));
+        using Clock = std::chrono::steady_clock;
+        const auto start = timings ? Clock::now() : Clock::time_point{};
+        auto program = PrepareResourceProgram(request, preparation);
+        const auto middle = timings ? Clock::now() : Clock::time_point{};
+        if (timings) timings->frontendSeconds = std::chrono::duration<double>(middle - start).count();
+        auto variant = std::make_shared<const CompiledVariant>(compileVariant(request, std::move(program)));
+        if (timings) timings->backendSeconds = std::chrono::duration<double>(Clock::now() - middle).count();
+        return variant;
+    });
+}
+
 std::shared_ptr<const SourceHandle> PrepareShader(const RecompileRequest& request, ShaderPreparationContext* preparation) {
     return recompileReporting(request, [&]() -> std::shared_ptr<const SourceHandle> {
         static_cast<void>(RequestInputInfo(request));

core/shader/recompiler/ShaderCachePack.cpp

@@ -5,6 +5,7 @@
 #include <chrono>
 #include <cstring>
 #include <limits>
+#include <type_traits>
 #ifdef _WIN32
 #ifndef NOMINMAX
 #define NOMINMAX
@@ -95,7 +96,8 @@ void CachePack::Validate() {
     records = checked;
 }
 
-LoadStatus CachePack::Load(std::span<const std::byte> key, CompiledVariant& variant, PackLoadTimings& timings) const {
+template<typename TValue>
+LoadStatus CachePack::LoadValue(std::span<const std::byte> key, TValue& value, PackLoadTimings& timings) const {
     const auto start = Clock::now();
     const auto hash = PackKeyHash(key);
     const auto first = std::lower_bound(records.begin(), records.end(), hash, [](const auto& record, const auto& value) { return record.hash < value; });
@@ -104,12 +106,25 @@ LoadStatus CachePack::Load(std::span<const std::byte> key, CompiledVariant& vari
     for (auto it = first; it != records.end() && it->hash == hash; ++it) {
         const auto data = file.subspan(it->offset, it->size);
         const auto decode = Clock::now();
-        const auto result = DecodeEntry(data, key, variant);
+        const auto result = [&] {
+            if constexpr (std::is_same_v<TValue, CompiledVariant>) return DecodeEntry(data, key, value);
+            else return DecodePlanEntry(data, key, value);
+        }();
         timings.decodeNanoseconds += elapsed(decode);
         timings.bytes += data.size();
         if (result == LoadStatus::Loaded) return result;
         if (result == LoadStatus::Rejected) status = result;
     }
     return status;
 }
+LoadStatus CachePack::Load(std::span<const std::byte> key, CompiledVariant& variant, PackLoadTimings& timings) const {
+    if (IsPlanKey(key)) return LoadStatus::KeyMismatch;
+    return LoadValue(key, variant, timings);
+}
+
+LoadStatus CachePack::LoadPlan(std::span<const std::byte> key, IrResourcePlan& plan, PackLoadTimings& timings) const {
+    if (!IsPlanKey(key)) return LoadStatus::KeyMismatch;
+    return LoadValue(key, plan, timings);
+}
+
 }

core/shader/recompiler/ShaderCachePack.hpp

@@ -26,6 +26,7 @@ struct PackLoadTimings {
 };
 
 [[nodiscard]] std::array<std::uint64_t, 2> PackKeyHash(std::span<const std::byte> key);
+[[nodiscard]] std::span<const std::byte> EntryKey(std::span<const std::byte> file);
 [[nodiscard]] std::vector<std::filesystem::path> PrebuiltCachePaths(const std::filesystem::path& cacheRoot);
 
 class CachePack {
@@ -38,9 +39,12 @@ class CachePack {
     [[nodiscard]] const std::string& Error() const { return error; }
     [[nodiscard]] std::size_t Count() const { return records.size(); }
     [[nodiscard]] LoadStatus Load(std::span<const std::byte> key, CompiledVariant& variant, PackLoadTimings& timings) const;
+    [[nodiscard]] LoadStatus LoadPlan(std::span<const std::byte> key, IrResourcePlan& plan, PackLoadTimings& timings) const;
 
 private:
     void Validate();
+    template<typename TValue>
+    LoadStatus LoadValue(std::span<const std::byte> key, TValue& value, PackLoadTimings& timings) const;
     std::span<const std::byte> file;
     std::span<const PackRecord> records;
     std::string error;