diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 7173257e786..f415b21e769 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -862,6 +862,18 @@ if(BUILD_TESTING) endif() endforeach() set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30) + add_test(NAME agc_driver_implicit_eop COMMAND agc_driver_tests --implicit-eop-only) + set_tests_properties(agc_driver_implicit_eop PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_IMPLICIT_SUBMISSION_EOP=1" + SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 20) + add_test(NAME agc_driver_completion_order COMMAND agc_driver_recorder_tests --completion-labels-only) + set_tests_properties(agc_driver_completion_order PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" + SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 20) + if(WIN32) + set_tests_properties(agc_driver_implicit_eop agc_driver_completion_order PROPERTIES + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE) set_tests_properties(agc_driver_storage_cache_budget PROPERTIES TIMEOUT 30) add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 0b92d3e8df2..b06863b6291 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -78,7 +78,7 @@ class Driver { static void readRegisterLists(Submission& submission); void waitForFlipRoom(const Submission& submission); void reserveOutputs(Submission& submission); - void executeRewindTail(const Submission& stalled); + void executeRewindTail(const Submission& stalled, bool& streamRequestedEop); void enqueue(Submission submission); void noteHeldAtSubmit(Submission& submission, std::size_t cursor); static void forgetUnfinishedWrites(QueueWorker& worker, const Submission& submission); @@ -223,7 +223,7 @@ class Driver { static void timed(double WorkerProfile::*bucket, TWork&& work); FrameTiming* frameTiming(); FrameTiming* includeTimingSubmission(const Submission& submission, bool firstSegment); - void execute(const Submission& submission); + void execute(const Submission& submission, bool& streamRequestedEop); void markCompleted(std::uint64_t serial); static const std::atomic*& workerQueued(); void reapCompletionLabels(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 8bc41e88897..c8827188683 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -29,7 +29,7 @@ void Driver::timed(double WorkerProfile::*bucket, TWork&& work) { } } -void Driver::execute(const Submission& submission) { +void Driver::execute(const Submission& submission, bool& streamRequestedEop) { auto* submissionTiming = includeTimingSubmission(submission, true); if (submission.suspend) { PerformanceContext timingContext(submissionTiming); @@ -169,6 +169,7 @@ void Driver::execute(const Submission& submission) { bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false; const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump); + streamRequestedEop |= endOfPipeInterrupt; traceLabel(packet, submission.queue); timing.Mark("prepare_memory"); @@ -310,7 +311,7 @@ void Driver::execute(const Submission& submission) { if (localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); } endTiming.Finish(); - if (submission.rewindTail != nullptr) executeRewindTail(submission); + if (submission.rewindTail != nullptr) executeRewindTail(submission, streamRequestedEop); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index 9fa87efc4cd..54021a62bde 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -6,6 +6,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "prx/libSceAgcDriver/Eq/include/Event.hpp" #include namespace AgcDriver::DriverDetail { @@ -135,13 +136,27 @@ void Driver::run(std::uint32_t id) noexcept { worker.queued.fetch_sub(1, std::memory_order_acq_rel); if (profile && submission.enqueuedAt != std::chrono::steady_clock::time_point{}) costs.dequeueNs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - submission.enqueuedAt).count()); } + bool streamRequestedEop = false; { struct Running { std::atomic& count; explicit Running(std::atomic& count) : count(count) { count.fetch_add(1, std::memory_order_acq_rel); } ~Running() { count.fetch_sub(1, std::memory_order_acq_rel); } } running{runningWorkers}; - execute(submission); + execute(submission, streamRequestedEop); + } + static const bool implicitEop = std::getenv("APS5_IMPLICIT_SUBMISSION_EOP") != nullptr; + if (implicitEop && !submission.suspend && !streamRequestedEop) { + bool deferred = false; + { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.Load()) { + deferred = localDevice->AfterRecordedWork([id] { AgcDriverDeliverEopInterrupt(id); }, id == 0); + if (deferred) localDevice->SubmitRecorded(id == 0); + } + } + if (!deferred) AgcDriverDeliverEopInterrupt(id); } if (traceGpu) std::fprintf(stderr, "[gpu] %.1f done serial=%llu queue=0x%x\n", TraceMs(), static_cast(submission.serial), id); const auto completeStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index 3df687bf658..9fc8ab51a3b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -154,7 +154,7 @@ void Driver::reserveOutputs(Submission& submission) { } } -void Driver::executeRewindTail(const Submission& stalled) { +void Driver::executeRewindTail(const Submission& stalled, bool& streamRequestedEop) { PerformanceTimer timing("Driver.Rewind"); std::atomic_ref control(*const_cast(stalled.rewindTail - 1)); if ((control.load(std::memory_order_acquire) & 0x80000000u) == 0) { @@ -191,7 +191,7 @@ void Driver::executeRewindTail(const Submission& stalled) { } if (APS5_ENABLE_TIMING_LOG) tail.enqueuedAt = tail.dequeuedAt = tail.orderedAt = std::chrono::steady_clock::now(); timing.Finish(); - execute(tail); + execute(tail, streamRequestedEop); } void Driver::Submit(const Packet* packet, std::uint32_t queue) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 47871020c81..60b4677a2e8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -3,6 +3,8 @@ #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Eq/include/Event.hpp" +#include "prx/libkernel/Equeue/Equeue.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" @@ -51,6 +53,8 @@ namespace { extern "C" { +int APS5_VABI sceKernelCreateEqueue(KernelEqueue* eq, const char* name); +int APS5_VABI sceKernelDeleteEqueue(KernelEqueue eq); int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*); int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t); int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t); @@ -413,6 +417,33 @@ void afterRecordedWorkTests(const Device& device, Recorder& recorder) { Require(seen == 1, "an action ran before the completion label recorded ahead of it"); } +void afterRecordedWorkEventTests(const Device& device, Recorder& recorder) { + recorder.Sync(); + KernelEqueue eq = 0; + Require(sceKernelCreateEqueue(&eq, "AGC recorded work") == 0, "cannot create recorded-work event queue"); + auto owner = EqueuePin_nid_postfix(eq); + int tag = 0; + Require(sceAgcDriverAddEqEvent(eq, 0, &tag) == 0, "cannot register recorded-work event"); + alignas(64) static std::uint32_t memory[16]{}; + const auto base = reinterpret_cast(memory); + const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + recorder.NotePendingWrite(0x52000, 0x100); + recorder.AfterCompletions(base, value, 6, 0, false); + std::uint32_t seen = 0; + Require(recorder.AfterRecordedWork([&] { seen = memory[0]; AgcDriverDeliverEopInterrupt(0); }), "recorded work did not retain its event"); + recorder.Submit(); + KernelEvent event{}; + Require(owner->GetTriggeredEvents(&event, 1) == 0 && seen == 0, "recorded-work event arrived before retirement"); + device.WaitQueue(); + Require(owner->GetTriggeredEvents(&event, 1) == 0, "recorded-work event arrived before reaping"); + recorder.Sync(); + Require(owner->GetTriggeredEvents(&event, 1) == 1 && event.filter == -14 && event.ident == 0 && event.udata == &tag && event.data == 1 && seen == 1, "recorded-work event was lost, duplicated or preceded its completion label"); + Require(owner->GetTriggeredEvents(&event, 1) == 0, "recorded-work event did not clear"); + Require(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "cannot unregister recorded-work event"); + owner.reset(); + Require(sceKernelDeleteEqueue(eq) == 0, "cannot delete recorded-work event queue"); +} + void batchStampTests(Recorder& recorder) { Require(recorder.Idle(), "batch stamps: the recorder is busy"); double previousEnd = 0; @@ -3601,6 +3632,7 @@ int main(int argc, char** argv) { if (argc == 2 && std::string_view(argv[1]) == "--completion-labels-only") { completionCountTests(device, recorder); afterRecordedWorkTests(device, recorder); + afterRecordedWorkEventTests(device, recorder); directCompletionLabelTests(recorder); labelTests(recorder); lateLabelTests(recorder); diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index 3c9db0b0cea..1550f049ce2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #ifdef _WIN32 @@ -170,7 +171,7 @@ void testSubmissions() { AgcDriverWaitIdle_nid_postfix(); } -void testEndOfPipeInterrupts() { +void testEndOfPipeInterrupts(bool implicitEop = false) { KernelEqueue eq = 0; check(sceKernelCreateEqueue(&eq, "AGC test") == 0, "event queue creation failed"); auto owner = EqueuePin_nid_postfix(eq); @@ -195,14 +196,18 @@ void testEndOfPipeInterrupts() { words[2] = 0; check(sceAgcDriverSubmitDcb(&packet) == 0, "plain release submit failed"); AgcDriverWaitIdle_nid_postfix(); - check(owner->GetTriggeredEvents(events.data(), 2) == 0, "release without INT_SEL raised an interrupt"); + const auto plainEvents = owner->GetTriggeredEvents(events.data(), 2); + check(plainEvents == (implicitEop ? 1 : 0), "plain release has the wrong submission completion count"); + if (implicitEop) check(events[0].ident == 0 && events[0].data == 1, "plain release completion has the wrong queue or count"); alignas(8) static volatile std::uint64_t label = 0; const auto labelAddress = reinterpret_cast(&label); words = {0xc0064900, 0x528, (3u << 29u) | (3u << 24u) | (1u << 16u), static_cast(labelAddress), static_cast(static_cast(labelAddress) >> 32u), 0x89abcdefu, 0x01234567u, 0}; check(sceAgcDriverSubmitDcb(&packet) == 0, "send-data release submit failed"); AgcDriverWaitIdle_nid_postfix(); check(label != 0, "send-data release did not write its label"); - check(owner->GetTriggeredEvents(events.data(), 2) == 0, "release with INT_SEL send data after write confirm raised an interrupt"); + const auto sendDataEvents = owner->GetTriggeredEvents(events.data(), 2); + check(sendDataEvents == (implicitEop ? 1 : 0), "send-data release has the wrong submission completion count"); + if (implicitEop) check(events[0].ident == 0 && events[0].data == 1, "send-data release completion has the wrong queue or count"); words = {0xc0064900, 0, 1u << 24u, 0, 0, 0, 0, 0}; check(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "graphics event deletion failed"); expectFailure([&] { sceAgcDriverDeleteEqEvent(eq, 0); }); @@ -251,6 +256,61 @@ std::vector commands(const std::array&... packe return words; } +void testImplicitSubmissionCompletions() { + KernelEqueue eq = 0; + check(sceKernelCreateEqueue(&eq, "AGC submission completions") == 0, "event queue creation failed"); + auto owner = EqueuePin_nid_postfix(eq); + int graphicsTag = 0, computeTag = 0; + check(sceAgcDriverAddEqEvent(eq, 0, &graphicsTag) == 0, "graphics event registration failed"); + check(sceAgcDriverAddEqEvent(eq, 0x20, &computeTag) == 0, "compute event registration failed"); + std::array events{}; + const auto expectCompletion = [&](std::uintptr_t queue, void* tag, std::intptr_t count) { + check(owner->GetTriggeredEvents(events.data(), 2) == 1, "submission completion was missing or reached another queue"); + check(events[0].filter == -14 && events[0].ident == queue && events[0].udata == tag && events[0].data == count, "submission completion has the wrong encoding or count"); + check(owner->GetTriggeredEvents(events.data(), 2) == 0, "submission completion did not clear"); + }; + alignas(64) static volatile std::uint32_t gate = 0, started = 0, finished = 0; + submit(0, commands(writeData(&started, 1), waitEqual(&gate, 1), writeData(&finished, 1))); + waitFor(&started, 1, "graphics submission never reached its wait"); + check(owner->GetTriggeredEvents(events.data(), 2) == 0 && finished == 0, "a blocked submission signaled completion early"); + submit(0x20, commands(writeData(&gate, 1))); + AgcDriverWaitIdle_nid_postfix(); + check(finished == 1, "graphics submission did not finish after its dependency"); + check(owner->GetTriggeredEvents(events.data(), 2) == 2, "independent queue completions were lost"); + for (const auto& event : events) { + check(event.filter == -14 && event.data == 1, "queue completion was duplicated"); + check((event.ident == 0 && event.udata == &graphicsTag) || (event.ident == 0x20 && event.udata == &computeTag), "queue completion reached the wrong registration"); + } + for (int i = 0; i < 3; ++i) submit(0, {0xc0001000, 0}); + AgcDriverWaitIdle_nid_postfix(); + expectCompletion(0, &graphicsTag, 3); + Packet empty{}; + check(sceAgcDriverSubmitDcb(&empty) == 0, "empty submit failed"); + AgcDriverWaitIdle_nid_postfix(); + expectCompletion(0, &graphicsTag, 1); + const std::vector rewindThenInterrupt{0xc0005900, 0x80000000u, 0xc0064900, 0, 1u << 24u, 0, 0, 0, 0, 0}; + submit(0, rewindThenInterrupt); + AgcDriverWaitIdle_nid_postfix(); + expectCompletion(0, &graphicsTag, 1); + const std::vector interruptThenRewind{0xc0064900, 0, 1u << 24u, 0, 0, 0, 0, 0, 0xc0005900, 0x80000000u, 0xc0001000, 0}; + submit(0, interruptThenRewind); + AgcDriverWaitIdle_nid_postfix(); + expectCompletion(0, &graphicsTag, 1); + submit(0, {0xc0001000, 0}); + AgcDriverWaitIdle_nid_postfix(); + expectCompletion(0, &graphicsTag, 1); + AgcDriverSuspendPoint_nid_postfix(); + AgcDriverWaitIdle_nid_postfix(); + check(owner->GetTriggeredEvents(events.data(), 2) == 0, "suspend boundary raised a submission interrupt"); + check(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "graphics event deletion failed"); + submit(0, {0xc0001000, 0}); + AgcDriverWaitIdle_nid_postfix(); + check(owner->GetTriggeredEvents(events.data(), 2) == 0, "a removed registration received completion"); + check(sceAgcDriverDeleteEqEvent(eq, 0x20) == 0, "compute event deletion failed"); + owner.reset(); + check(sceKernelDeleteEqueue(eq) == 0, "event queue deletion failed"); +} + std::array endOfPipeLabel(volatile std::uint32_t* address, std::uint32_t value) { const auto target = reinterpret_cast(address); return {0xc0064900, 0x514, (1u << 29u) | (2u << 24u), static_cast(target), static_cast(static_cast(target) >> 32u), value, 0, 0}; @@ -553,10 +613,18 @@ void testWorkerFailure() { } -int main() { +int main(int argc, char** argv) { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (argc == 2 && std::string_view(argv[1]) == "--implicit-eop-only") { + testImplicitSubmissionCompletions(); + testEndOfPipeInterrupts(true); + testEndOfPipeLabelsWithoutWork(); + LibcRunShutdown_nid_postfix(); + std::puts("AGC implicit submission completion tests passed"); + return 0; + } alignas(256) std::array rawCode{}; rawCode.fill(0xbf800000); rawCode[0] = 0xbe8003ff; diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index f4ab1a4c8ec..f3967b9885e 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -1,5 +1,7 @@ # Project technical debt +- [Implicit submission EOP events](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp) (libSceAgcDriver) are opt-in with `APS5_IMPLICIT_SUBMISSION_EOP=1`. A submission without an executed interrupt-requesting packet raises one event after recorded GPU work completes; suspend boundaries raise none. This unblocks a graphics-event wait observed in Minecraft Preview, but implicit interrupts per submission have not been verified on console hardware. The shared recorder can delay an event behind work from another queue. Existing explicit interrupts retain their packet semantics. + ### Build - Building on Windows requires a specific version of mingw - MinGW-w64 GCC 15.2.0 (`winlibs-gcc15`, `x86_64-ucrt-posix-seh`)