perf(agc): wait at submit only for the preparations of the programs a submission binds

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp

@@ -11,6 +11,7 @@
 #include <map>
 #include <memory>
 #include <set>
+#include <span>
 #include <thread>
 #include <unordered_map>
 #include <unordered_set>
@@ -54,6 +55,8 @@ struct QueueWorker {
     std::thread thread;
 };
 
+std::vector<std::uint64_t> ReferencedPrograms(std::span<const std::uint32_t> commands, const std::map<std::size_t, std::vector<std::uint32_t>>& registerLists);
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -115,7 +115,7 @@ std::shared_ptr<const ShaderSnapshot> RegisteredProgram(const ShaderRegistry& re
 void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const std::shared_ptr<const ShaderSnapshot>& snapshot);
 void SubmitRegistrationPreparation(std::function<void()> task, const PreparedShaders* prepared = nullptr);
 void PrioritizeRegistrationPreparation(PreparedShaders& prepared);
-void AwaitRegistrationPreparation();
+void AwaitReferencedPreparations(const ShaderRegistry& registry, std::span<const std::uint64_t> addresses);
 void SubmitLibraryPrebuild(std::function<void()> task);
 void CloseRegistrationPreparation();
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp

@@ -10,6 +10,8 @@
 #include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
 #include "ThreadOwned.hpp"
 #include "prx/libc/include/HostThreadLocal.hpp"
+#include <algorithm>
+#include <array>
 #include <bit>
 #include <cmath>
 #include <cstdio>
@@ -160,6 +162,41 @@ bool Driver::copySegment(Submission& submission, const std::uint32_t* guest, std
     return false;
 }
 
+std::vector<std::uint64_t> ReferencedPrograms(std::span<const std::uint32_t> commands, const std::map<std::size_t, std::vector<std::uint32_t>>& registerLists) {
+    constexpr std::array<std::uint32_t, 6> programRegisters{0x008u, 0x088u, 0x0c8u, 0x108u, 0x148u, 0x20cu};
+    std::array<std::uint32_t, programRegisters.size() * 2> values{};
+    std::array<bool, programRegisters.size() * 2> written{};
+    std::vector<std::uint64_t> addresses;
+    const auto write = [&](std::uint32_t offset, std::uint32_t value) {
+        const auto base = std::ranges::find_if(programRegisters, [&](std::uint32_t program) { return offset - program < 2u; });
+        if (base == programRegisters.end()) return;
+        const auto slot = static_cast<std::size_t>(base - programRegisters.begin()) * 2u + (offset - *base);
+        values[slot] = value;
+        written[slot] = true;
+        const auto low = slot & ~std::size_t{1};
+        if (written[low] && written[low + 1]) addresses.push_back((static_cast<std::uint64_t>(values[low]) << 8u) | (static_cast<std::uint64_t>(values[low + 1] & 0xffu) << 40u));
+    };
+    for (std::size_t cursor = 0; cursor < commands.size(); cursor += Pm4::PacketWords(commands[cursor])) {
+        const auto header = commands[cursor];
+        if ((header >> 30u) != 3u) continue;
+        const auto opcode = (header >> 8u) & 0xffu;
+        const auto words = std::min<std::size_t>(Pm4::PacketWords(header), commands.size() - cursor);
+        if (opcode == 0x76u && words >= 3) {
+            const auto packet = commands.subspan(cursor, words);
+            if (Pm4::IsTagMarker(packet)) continue;
+            const auto offset = packet[1] & ~0x70000000u;
+            for (std::size_t i = 2; i < words; ++i) write(offset + static_cast<std::uint32_t>(i - 2), packet[i]);
+        } else if (opcode == 0x63u) {
+            const auto list = registerLists.find(cursor);
+            if (list == registerLists.end()) continue;
+            for (std::size_t i = 0; i + 1 < list->second.size(); i += 2) write(list->second[i] & ~0x70000000u, list->second[i + 1]);
+        }
+    }
+    std::ranges::sort(addresses);
+    addresses.erase(std::unique(addresses.begin(), addresses.end()), addresses.end());
+    return addresses;
+}
+
 void Driver::readRegisterLists(Submission& submission) {
     submission.registerLists.clear();
     for (std::size_t cursor = 0; cursor < submission.commands.size(); cursor += Pm4::PacketWords(submission.commands[cursor])) {
@@ -343,7 +380,12 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
     if (APS5_ENABLE_TIMING_LOG) submission.validatedAt = std::chrono::steady_clock::now();
     waitForFlipRoom(submission);
     holdFlipBehindWorker(submission);
-    AwaitRegistrationPreparation();
+    std::shared_ptr<const ShaderRegistry> registry;
+    {
+        std::lock_guard lock(mutex);
+        registry = shaders;
+    }
+    if (registry != nullptr) AwaitReferencedPreparations(*registry, ReferencedPrograms(submission.commands, submission.registerLists));
     if (APS5_ENABLE_TIMING_LOG) submission.roomReadyAt = std::chrono::steady_clock::now();
     static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr;
     if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast<const void*>(descriptor.addr));

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -975,10 +975,8 @@ class RegistrationPreparePool {
         {
             std::unique_lock lock(mutex);
             if (threads != 0 && !closed) {
-                const auto sequence = ++submitted;
-                if (prepared != nullptr && prepared->demanded.load()) tasks.push_front({std::move(task), prepared, sequence});
-                else tasks.push_back({std::move(task), prepared, sequence});
-                unfinished.insert(sequence);
+                if (prepared != nullptr && prepared->demanded.load()) tasks.push_front({std::move(task), prepared});
+                else tasks.push_back({std::move(task), prepared});
                 lock.unlock();
                 ready.notify_one();
                 return;
@@ -995,12 +993,6 @@ class RegistrationPreparePool {
         std::rotate(tasks.begin(), found, std::next(found));
     }
 
-    void AwaitSubmitted() {
-        std::unique_lock lock(mutex);
-        const auto ticket = submitted;
-        idle.wait(lock, [&] { return unfinished.empty() || *unfinished.begin() > ticket; });
-    }
-
     void Close() {
         std::unique_lock lock(mutex);
         closed = true;
@@ -1034,14 +1026,12 @@ class RegistrationPreparePool {
                 pool.tasks.pop_front();
                 ++pool.running;
             }
-            const auto sequence = task.sequence;
             task.function();
             task = {};
             bool drained = false;
             {
                 std::lock_guard lock(pool.mutex);
                 --pool.running;
-                pool.unfinished.erase(sequence);
                 drained = pool.tasks.empty() && pool.running == 0;
                 pool.idle.notify_all();
             }
@@ -1055,7 +1045,6 @@ class RegistrationPreparePool {
     struct Task {
         std::function<void()> function;
         const PreparedShaders* prepared = nullptr;
-        std::uint64_t sequence = 0;
     };
 
     std::mutex mutex;
@@ -1064,8 +1053,6 @@ class RegistrationPreparePool {
     std::deque<Task> tasks;
     unsigned threads = 0;
     unsigned running = 0;
-    std::uint64_t submitted = 0;
-    std::set<std::uint64_t> unfinished;
     bool closed = false;
 };
 
@@ -1111,8 +1098,20 @@ void PrioritizeRegistrationPreparation(PreparedShaders& prepared) {
     if (auto* pool = RegistrationPreparePool::Existing()) pool->Prioritize(prepared);
 }
 
-void AwaitRegistrationPreparation() {
-    if (auto* pool = RegistrationPreparePool::Existing()) pool->AwaitSubmitted();
+void AwaitReferencedPreparations(const ShaderRegistry& registry, std::span<const std::uint64_t> addresses) {
+    for (const auto address : addresses) {
+        auto found = registry.upper_bound(address);
+        if (found == registry.begin()) continue;
+        --found;
+        for (const auto& snapshot : found->second) {
+            if (address - snapshot->codeAddress >= snapshot->code.size() * sizeof(std::uint32_t)) continue;
+            auto& prepared = *snapshot->prepared;
+            std::unique_lock lock(prepared.mutex);
+            if (!prepared.pending) continue;
+            PrioritizeRegistrationPreparation(prepared);
+            prepared.settled.wait(lock, [&] { return !prepared.pending; });
+        }
+    }
 }
 
 void SubmitLibraryPrebuild(std::function<void()> task) {

core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp

@@ -2,6 +2,7 @@
 #include "Optimization/ResourceProgram.hpp"
 #include "ControlFlow/RequestSerializer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
@@ -13,6 +14,7 @@
 #include <spirv/unified1/spirv.hpp>
 #include <algorithm>
 #include <array>
+#include <map>
 #include <cstddef>
 #include <cstring>
 #include <chrono>
@@ -313,29 +315,67 @@ std::exception_ptr InjectedFailure() {
     }
 }
 
-void AwaitedRegistrationPreparation() {
+void ReferencedRegistrationPreparation() {
+    using AgcDriver::DriverDetail::ShaderSnapshot;
+    alignas(256) static const std::array<std::uint32_t, 1> boundCode{0xbf810000u};
+    alignas(256) static const std::array<std::uint32_t, 1> unrelatedCode{0xbf810000u};
+    const auto bound = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(boundCode.data()));
+    const auto unrelated = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(unrelatedCode.data()));
+    const auto boundSnapshot = std::make_shared<ShaderSnapshot>(ShaderSnapshot{bound, 0, 0, {boundCode.begin(), boundCode.end()}, {}});
+    const auto unrelatedSnapshot = std::make_shared<ShaderSnapshot>(ShaderSnapshot{unrelated, 0, 0, {unrelatedCode.begin(), unrelatedCode.end()}, {}});
+    AgcDriver::DriverDetail::ShaderRegistry registry;
+    registry[bound].push_back(boundSnapshot);
+    registry[unrelated].push_back(unrelatedSnapshot);
+    const std::vector<std::uint32_t> commands{0xc0027600u, 0x20cu, static_cast<std::uint32_t>(bound >> 8u), static_cast<std::uint32_t>(bound >> 40u)};
+    Require(AgcDriver::DriverDetail::ReferencedPrograms(commands, {}) == std::vector<std::uint64_t>{bound}, "a compute program bound with SET_SH_REG was not found in the submission");
+    const std::vector<std::uint32_t> indirect{0xc0036300u, 0, 0, 0, 2};
+    const std::map<std::size_t, std::vector<std::uint32_t>> lists{{0, {0xc8u, static_cast<std::uint32_t>(unrelated >> 8u), 0xc9u, static_cast<std::uint32_t>(unrelated >> 40u)}}};
+    Require(AgcDriver::DriverDetail::ReferencedPrograms(indirect, lists) == std::vector<std::uint64_t>{unrelated}, "a vertex program bound through an indirect register list was not found in the submission");
+    const std::vector<std::uint32_t> lowOnly{0xc0017600u, 0x008u, static_cast<std::uint32_t>(bound >> 8u)};
+    Require(AgcDriver::DriverDetail::ReferencedPrograms(lowOnly, {}).empty(), "a program address without its high register was resolved");
+    boundSnapshot->prepared->pending = true;
+    unrelatedSnapshot->prepared->pending = true;
+    std::atomic<bool> unrelatedRan{false};
+    AgcDriver::DriverDetail::SubmitRegistrationPreparation([unrelatedSnapshot, &unrelatedRan] {
+        std::this_thread::sleep_for(std::chrono::milliseconds(1500));
+        unrelatedRan.store(true, std::memory_order_release);
+        {
+            std::lock_guard lock(unrelatedSnapshot->prepared->mutex);
+            unrelatedSnapshot->prepared->pending = false;
+        }
+        unrelatedSnapshot->prepared->settled.notify_all();
+    }, unrelatedSnapshot->prepared.get());
+    auto settle = std::async(std::launch::async, SettleLater, std::ref(*boundSnapshot->prepared), std::vector<AgcDriver::DriverDetail::PreparedShaders::Entry>{}, nullptr);
+    AgcDriver::DriverDetail::AwaitReferencedPreparations(registry, AgcDriver::DriverDetail::ReferencedPrograms(commands, {}));
+    bool boundPending = false;
+    {
+        std::lock_guard lock(boundSnapshot->prepared->mutex);
+        boundPending = boundSnapshot->prepared->pending;
+    }
+    const bool unrelatedDone = unrelatedRan.load(std::memory_order_acquire);
+    settle.get();
+    Require(!boundPending, "a submission did not wait for the preparation of a program its commands bind");
+    Require(!unrelatedDone, "a slow preparation of a program the commands do not bind held the submission back");
+    const std::array<std::uint64_t, 1> unrelatedAddress{unrelated};
+    AgcDriver::DriverDetail::AwaitReferencedPreparations(registry, unrelatedAddress);
+    Require(unrelatedRan.load(std::memory_order_acquire), "a submission binding the slow program did not wait for its preparation");
+}
+
+void UnrelatedPreparationAtSubmit() {
     std::atomic<bool> ran{false};
     AgcDriver::DriverDetail::SubmitRegistrationPreparation([&] {
-        std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        std::this_thread::sleep_for(std::chrono::milliseconds(1500));
         ran.store(true, std::memory_order_release);
     });
-    AgcDriver::DriverDetail::AwaitRegistrationPreparation();
-    Require(ran.load(std::memory_order_acquire), "a submission did not wait for the registration preparation queued before it");
-    std::atomic<bool> slow{false};
-    std::atomic<bool> fast{false};
-    AgcDriver::DriverDetail::SubmitRegistrationPreparation([&] {
-        std::this_thread::sleep_for(std::chrono::milliseconds(300));
-        slow.store(true, std::memory_order_release);
-    });
-    auto waiter = std::async(std::launch::async, [&] {
-        AgcDriver::DriverDetail::AwaitRegistrationPreparation();
-        return slow.load(std::memory_order_acquire);
-    });
-    std::this_thread::sleep_for(std::chrono::milliseconds(50));
-    AgcDriver::DriverDetail::SubmitRegistrationPreparation([&] { fast.store(true, std::memory_order_release); });
-    Require(waiter.get(), "a submission stopped waiting for an earlier preparation once a later one finished");
-    AgcDriver::DriverDetail::AwaitRegistrationPreparation();
-    Require(fast.load(std::memory_order_acquire), "the later preparation did not run");
+    std::vector<std::uint32_t> commands{0xc0001000u, 0};
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    const auto began = std::chrono::steady_clock::now();
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "a submission behind an unrelated preparation failed");
+    const auto waited = std::chrono::steady_clock::now() - began;
+    const bool done = ran.load(std::memory_order_acquire);
+    AgcDriverWaitIdle_nid_postfix();
+    while (!ran.load(std::memory_order_acquire)) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    Require(!done && waited < std::chrono::milliseconds(1000), "a submission waited for a preparation its commands do not use");
 }
 
 void PendingRegistrationPreparation(AgcDriver::VulkanDevice& device) {
@@ -809,11 +849,12 @@ int main(int argc, char** argv) {
         RegisteredVertexWorkgroup(device->Target());
         FailureCapture(*device);
         PendingRegistrationPreparation(*device);
-        AwaitedRegistrationPreparation();
+        ReferencedRegistrationPreparation();
         Require(argc != 2 || std::string_view(argv[1]) != "--fail-before-registration", "injected failure before registration");
         PrepareMultisampledStorage(*device);
         device.reset();
         SharedCodeHeaders();
+        UnrelatedPreparationAtSubmit();
         RegistrationWithoutSpecials();
         Registration(argc == 2);
         std::cout << "prepared shader and transactional registration tests passed\n";

docs/dev/debt/libSceAgcDriver.md

@@ -160,4 +160,4 @@
 - [Depth surfaces created over storage writes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver): a depth surface created without HTILE copies its depth plane from the storage image at the plane's address, a pending one first, when that image has the plane's extent, one level, a 2D view or a 2D array view (its layer 0 is copied) and a 16-bit format for a 16-bit plane or a 32-bit format for a 32-bit plane. A multisampled surface, a surface the device stores in another format, a surface with HTILE and a surface without such an image start from `DB_DEPTH_CLEAR`, and the stencil plane from `DB_STENCIL_CLEAR`; CPU writes to the plane's memory are not read. Not measured on the console, where the plane is that memory.
 - [Flip packets raise the graphics end-of-pipe event](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) (libSceAgcDriver) - a flip delivers the queue's end-of-pipe event (`sceAgcDriverAddEqEvent`) once the work recorded before it completes, as a `RELEASE_MEM` with `INT_SEL` does. Minecraft (PPSA17221) registers the graphics event, never requests an interrupt in its releases, and waits on the event after every flip while holding its render lock, so without this it never presents a second frame. Whether the console raises the event from the flip itself or from an interrupt its flip sequence contains is not measured; a title that waits only on release interrupts sees one extra event per flip, with `data` counting it.
 - [Host import windows and budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): a registered range larger than 64 MiB is imported one 64 MiB window at a time for buffer and fill accesses; an access across window edges imports the cells it touches, grown to the union of the range's imports it overlaps. An address-based build (`AcquireRegistered`) imports the range whole and replaces its windows. Live imports stay under a budget: `APS5_HOST_IMPORT_MIB`, else the larger of 6 GiB and half the physical memory, at most the largest host heap less 2 GiB (imports and type-3 mirrors share it; #167's rule). A new import beyond it retires the least recently used imports not used in the last 256 lookups; one that would not fit even then is refused before anything is retired, and imports made when a range is mapped never evict. A refused import is tried again at its next use. The 64 MiB window, the 256-lookup protection and the 2 GiB reserve are choices, not measured.
-- [Registration preparation at submit](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) - `Driver::Submit` waits for every registration-time shader preparation queued before the call, not only those of the shaders the commands use, so a submission behind a long queue of unrelated preparations waits for all of them on the submitting thread. Waiting there instead of in the queue worker keeps the GPU side of a submission short: a title that sees a submission take seconds treats the GPU as hung (Goat Simulator 3, PPSA04159, then stops submitting and its garbage collector fails a 10 s timeout). On the console shaders are compiled ahead of time, so neither wait exists; how long a title tolerates a slow submit call is not known.
+- [Registration preparation at submit](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) - `Driver::Submit` waits, on the submitting thread, for the registration-time preparations of the programs its commands bind: the program address registers (`SPI_SHADER_PGM_LO/HI` of the pixel, geometry, export, hull and local stages, `COMPUTE_PGM_LO/HI`) written by `SET_SH_REG` packets and indirect SH register lists in the submission. A program bound by an earlier submission, a program address whose low or high register the submission does not write, or one reached another way (an indirect draw patching a program register, an address loaded by the shader) is not waited for there; its first draw or dispatch waits for it in the queue worker instead (`AwaitRegisteredPreparation`, which moves it to the front of the queue), so that submission's GPU side can again take as long as the preparation. Programs the commands write but never draw (a register overwritten before the draw, a predicated or skipped range) are still waited for. Waiting at submit keeps the GPU side short: a title that sees a submission take seconds treats the GPU as hung (Goat Simulator 3, PPSA04159, then stops submitting and its garbage collector fails a 10 s timeout). On the console shaders are compiled ahead of time, so neither wait exists; how long a title tolerates a slow submit call is not known.