fix(agc): skip unused pixel stages with retained export formats

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1628,6 +1628,7 @@ if(BUILD_TESTING)
     add_test(NAME agc_driver_draw_cache_reuse_verify COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_verify PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_DRAW_DATA_HITS=1" SKIP_RETURN_CODE 77)
     set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:")
+    set_property(TEST agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off APPEND PROPERTY ENVIRONMENT "APS5_VERIFY_DRAW_RECIPE=1" "APS5_VERIFY_REGISTER_KEYS=1")
     if(WIN32)
         set_tests_properties(agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -127,7 +127,7 @@ constexpr std::uint32_t DepthControlMask = ~0x007007f0u;
 // EXEC_ON_HIER_FAIL / EXEC_ON_NOOP / EXEC_IF_OVERLAPPED (bits 9, 10, 17) only force the pixel shader
 // to run, which it always does here.
 constexpr std::uint32_t ShaderControlMask = ~(0x0000f870u | 0x00020600u | 0x00010000u);
-constexpr std::uint32_t PixelStageRunsMask = 0x00020747u;
+constexpr std::uint32_t PixelStageRunsMask = 0x000207c7u;
 constexpr std::uint32_t AlphaToCoverageMask = ~0x0001ff00u;
 constexpr std::uint32_t AlphaToCoverageUndithered = 0x0000aa01u;
 constexpr std::uint32_t AlphaToCoverageDithered = 0x00018701u;
@@ -1414,10 +1414,11 @@ bool PixelProgramSkipped(const QueueState& queue) {
     const auto& cx = queue.context;
     const auto targetMask = find(cx, 0x8e);
     const auto shaderMask = find(cx, 0x8f);
-    const auto zFormat = find(cx, 0x1c4);
     const auto shaderControl = find(cx, 0x203);
-    if (targetMask == cx.end() || shaderMask == cx.end() || zFormat == cx.end() || shaderControl == cx.end()) return false;
-    return (targetMask->second & shaderMask->second) == 0 && zFormat->second == 0 && (shaderControl->second & PixelStageRunsMask) == 0;
+    if (targetMask == cx.end() || shaderMask == cx.end() || shaderControl == cx.end()) return false;
+    const auto alphaToMask = find(cx, 0x2dc);
+    if (alphaToMask != cx.end() && (alphaToMask->second & 1u) != 0) return false;
+    return (targetMask->second & shaderMask->second) == 0 && (shaderControl->second & PixelStageRunsMask) == 0;
 }
 
 std::string NullPixelProgramRejection(const QueueState& queue) {

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -187,6 +187,48 @@ void inactiveVertexPointSizeTests() {
     rejected(queue);
 }
 
+void retainedPixelStageTests() {
+    using namespace AgcDriver::Graphics;
+    auto queue = makeState();
+    queue.context[0x8e] = queue.context[0x8f] = queue.context[0x203] = 0;
+    for (const auto format : {0u, 1u, 2u, 3u, 9u}) {
+        queue.context[0x1c4] = format;
+        Require(PixelProgramSkipped(queue), "a retained export format forced an unused pixel program to run");
+        Require(DrawRejection(queue, true).empty(), "an unused pixel program with a retained export format was rejected");
+        for (const auto control : {1u, 2u, 4u, 0x40u, 0x80u, 0x100u, 0x200u, 0x400u, 0x20000u}) {
+            queue.context[0x203] = control;
+            Require(!PixelProgramSkipped(queue), "an enabled export or execution flag was skipped");
+        }
+        queue.context[0x203] = 0;
+        queue.context[0x8e] = 0xf0000000u;
+        queue.context[0x8f] = 0x0f000000u;
+        Require(PixelProgramSkipped(queue), "disjoint color masks ran an unused pixel program");
+        queue.context[0x8f] = 0xf0000000u;
+        Require(!PixelProgramSkipped(queue), "an enabled high render target lost its pixel program");
+        queue.context[0x8e] = queue.context[0x8f] = 0;
+    }
+    queue.context[0x1b3] = queue.context[0x1b4] = 2;
+    for (const auto [format, control] : {std::pair{1u, 1u}, std::pair{2u, 2u}, std::pair{9u, 0x100u}, std::pair{9u, 0x103u}}) {
+        queue.context[0x1c4] = format;
+        queue.context[0x203] = control;
+        const auto state = DecodeState(queue);
+        Require(DrawRejection(queue, true).empty(), "a valid pixel export combination was rejected");
+        const auto pixel = DecodePixelStageInfo(queue.context, ExportMappings(state), PixelProgramSkipped(queue));
+        Require(pixel.depthExportEnable == ((control & 1u) != 0) && pixel.stencilRefExportEnable == ((control & 2u) != 0) && pixel.sampleMaskExportEnable == ((control & 0x100u) != 0), "decoding lost an enabled pixel export");
+    }
+    queue.context[0x203] = 0;
+    for (const auto alpha : {0xaa00u, 0xaa01u}) {
+        queue.context[0x2dc] = alpha;
+        Require(PixelProgramSkipped(queue) == ((alpha & 1u) == 0), "alpha-to-coverage enable was not respected");
+    }
+    queue.context[0x2dc] = 0xaa00u;
+    for (const auto offset : {0x8eu, 0x8fu, 0x203u}) {
+        auto missing = queue;
+        missing.context.erase(offset);
+        Require(!PixelProgramSkipped(missing), "incomplete execution state skipped a bound pixel program");
+    }
+}
+
 void hdrComponentSwapTests() {
     using namespace AgcDriver::Graphics;
     for (const auto number : {0u, 1u, 4u, 7u}) {
@@ -460,7 +502,18 @@ void stateTests() {
     Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program with Z_EXPORT_ENABLE was skipped");
     queue.context[0x203] = 0;
     queue.context[0x1c4] = 1;
-    Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program exporting depth was skipped");
+    Require(AgcDriver::Graphics::PixelProgramSkipped(queue), "a retained Z export format ran a pixel program with depth exports disabled");
+    Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "a depth-only draw with a retained Z export format was rejected");
+    for (const auto control : {1u, 2u, 4u, 0x40u, 0x80u, 0x100u, 0x200u, 0x400u, 0x20000u}) {
+        queue.context[0x203] = control;
+        Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "an enabled pixel export or execution flag was ignored with a retained Z format");
+    }
+    queue.context[0x203] = 0;
+    queue.context[0x2dc] = 0xaa01;
+    Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "alpha-to-coverage skipped the pixel program");
+    queue.context[0x203] = 0x800;
+    Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "the Vulkan alpha-to-coverage state lost its pixel program with DB's override set");
+    queue.context[0x203] = queue.context[0x2dc] = 0;
     queue.context[0x1c4] = 0;
     queue.context[0x8e] = 0xf;
     Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program writing color was skipped");
@@ -1454,6 +1507,25 @@ void DepthStencilTests() {
     const auto rejection = AgcDriver::Graphics::DrawRejection(queue, false);
     Require(rejection.empty(), "precheck rejected a stencil draw with a surface: " + rejection);
     Require(state.depth && state.depth->address == 0x10000 && state.depth->stencilAddress == 0x20000 && state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT && state.depth->clearStencil == 7 && state.depth->htileAddress == 0x10000030000ull && !state.depth->htileStencil, "depth surface decode changed");
+    for (const auto clear : {0u, 1u, 2u, 3u}) {
+        auto retained = queue;
+        retained.context[0x8e] = retained.context[0x8f] = retained.context[0x203] = 0;
+        retained.context[0x000] = clear;
+        retained.context[0x200] = 0x00700777u;
+        auto unbound = retained;
+        unbound.shader[0x008] = unbound.shader[0x009] = 0;
+        const auto expected = AgcDriver::Graphics::DecodeState(unbound);
+        for (const auto format : {1u, 2u, 3u, 9u}) {
+            retained.context[0x1c4] = format;
+            Require(AgcDriver::Graphics::PixelProgramSkipped(retained), "a retained format ran the pixel program during a depth/stencil pass");
+            const auto actual = AgcDriver::Graphics::DecodeState(retained);
+            Require(actual.depth && expected.depth && actual.depth->address == expected.depth->address && actual.depth->stencilAddress == expected.depth->stencilAddress && actual.depth->format == expected.depth->format && actual.depth->clearDepth == expected.depth->clearDepth && actual.depth->clearStencil == expected.depth->clearStencil, "null substitution changed the depth/stencil surface");
+            Require(actual.depthTest == expected.depthTest && actual.depthWrite == expected.depthWrite && actual.depthCompare == expected.depthCompare && actual.depthBoundsTest == expected.depthBoundsTest && actual.stencilTest == expected.stencilTest, "null substitution changed fixed-function depth/stencil tests");
+            Require(std::memcmp(&actual.stencilFront, &expected.stencilFront, sizeof(VkStencilOpState)) == 0 && std::memcmp(&actual.stencilBack, &expected.stencilBack, sizeof(VkStencilOpState)) == 0, "null substitution changed stencil operations");
+            Require(actual.viewport.minDepth == expected.viewport.minDepth && actual.viewport.maxDepth == expected.viewport.maxDepth && actual.renderExtent.width == expected.renderExtent.width && actual.renderExtent.height == expected.renderExtent.height, "null substitution changed depth coverage");
+            Require(AgcDriver::Graphics::DrawRejection(retained, true).empty(), "a retained-format depth/stencil pass was rejected");
+        }
+    }
     {
         queue.context[0x011] &= ~(1u << 29u);
         const auto stencilTiled = AgcDriver::Graphics::DecodeState(queue);
@@ -6595,6 +6667,7 @@ int main(int argc, char** argv) {
         RunGuestLeaseWaitTests();
         ResourceContentKeyTests(false);
         stateTests();
+        retainedPixelStageTests();
         pipelineLibrarySetTests();
         hdrComponentSwapTests();
         depthMaintenanceTests();

core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheReuse.cpp

@@ -182,6 +182,40 @@ void StageDataHitsRefreshTheirWords(bool libraries) {
     Require(BrightestColorByte() == 255, mode + ": the draw of constant 1.0 wrote " + std::to_string(BrightestColorByte()) + ", expected 255");
 }
 
+void RetainedPixelCacheTransitions() {
+    auto commands = Draws(32.0f, VertexCode.data(), 1, PixelCode.data(), 1, {});
+    Append(commands, 0x76, {0x00c, std::bit_cast<std::uint32_t>(1.0f)});
+    const auto draw = [&] {
+        for (unsigned i = 0; i < 3; ++i) Append(commands, 0x2d, {3, 2});
+    };
+    const auto skipped = [&] {
+        Append(commands, 0x69, {0x8e, 0, 0});
+        Append(commands, 0x69, {0x203, 0});
+        Append(commands, 0x69, {0x2dc, 0xaa00});
+        draw();
+    };
+    Append(commands, 0x69, {0x1c4, 1});
+    skipped();
+    Append(commands, 0x69, {0x8e, 0xf, 0xf});
+    draw();
+    skipped();
+    Append(commands, 0x69, {0x203, 0x400});
+    draw();
+    skipped();
+    Append(commands, 0x69, {0x8e, 0xf, 0xf});
+    Append(commands, 0x69, {0x2dc, 0xaa01});
+    draw();
+    skipped();
+    const auto before = AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters();
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "retained pixel cache transitions failed to submit");
+    AgcDriverWaitIdle_nid_postfix();
+    const auto after = AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters();
+    Require(after.lookups - before.lookups == 21 && after.absent - before.absent == 8 && after.inserts - before.inserts == 4 && after.hits - before.hits == 13, "color, execution or alpha-to-coverage transitions reused the wrong cached draw");
+    if (std::getenv("APS5_VERIFY_DRAW_RECIPE") != nullptr) Require(after.verifyDecodes > before.verifyDecodes, "pixel cache transitions did not verify any cached decode");
+    Require(after.verifyDecodeMismatches == before.verifyDecodeMismatches && after.facadeMismatches == before.facadeMismatches, "cached pixel transitions disagreed with fresh decoding");
+}
+
 void SightingsKeepCollidingKeys() {
     DrawKeySightings sightings(DrawKeySightings::Ways);
     Require(!sightings.SeenBefore(0) && sightings.SeenBefore(0), "draw key 0 was not admitted at its second sight");
@@ -207,6 +241,7 @@ int main() {
         ThirdDrawHitsTheCache(libraries);
         UserDataDrawsReuseTheirShape(libraries);
         StageDataHitsRefreshTheirWords(libraries);
+        RetainedPixelCacheTransitions();
         AgcDriverShutdown_nid_postfix();
         std::cout << "A repeated draw reused the cached draw " << (libraries ? "with" : "without") << " graphics pipeline libraries\n";
         return 0;

core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp

@@ -394,6 +394,87 @@ void PrebuildPolicyReadback(AgcDriver::VulkanDevice& device, bool disabled) {
     Require(std::ranges::all_of(pixels, [](const auto value) { return value == 0xffffffffu; }), "prebuild policy changed native prepared-shader readback");
 }
 
+void RetainedPixelMatchesNullProgram(AgcDriver::VulkanDevice& device) {
+    using namespace AgcDriver::DriverDetail;
+    Fixture front;
+    Fixture retained;
+    front.Initialize(2u);
+    retained.Initialize(1u);
+    const auto null = std::make_shared<ShaderSnapshot>(PrepareNullPixelProgram(device));
+    ShaderRegistry registry;
+    for (const auto& snapshot : {front.snapshot, retained.snapshot, null}) registry.emplace(snapshot->codeAddress, std::vector<std::shared_ptr<const ShaderSnapshot>>{snapshot});
+    AgcDriver::QueueState queue{};
+    queue.context[0x8e] = queue.context[0x8f] = queue.context[0x203] = 0;
+    queue.context[0x2dc] = 0xaa00;
+    queue.context[0x1c4] = 1;
+    front.Bind(queue, 0xc8u, 0x8bu);
+    queue.shader[0x8bu] = 0;
+    retained.Bind(queue, 0x008u, 0x00bu);
+    queue.shader[0x00bu] = (31u << 1u) | (1u << 27u);
+    for (std::uint32_t i = 0; i < 30; ++i) queue.shader[0x00cu + i] = 0xdead0000u + i;
+    queue.context[0x1b6] = queue.context[0x1b3] = queue.context[0x1b4] = queue.context[0x1c5] = 0xffffffffu;
+    auto captured = queue;
+    captured.shader[0x00bu] = 20u << 1u;
+    DrawDecode capturedDraw{};
+    capturedDraw.state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex;
+    DecodeGraphicsPrograms(capturedDraw, captured, registry, false, true);
+    Require(capturedDraw.programs.back().snapshot == null && capturedDraw.programs.back().userData.empty(), "a retained pixel program with a valid 20-word resource count consumed unrelated userdata");
+    for (const auto wave : {64u, 32u}) {
+        DrawDecode draw{};
+        draw.state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex;
+        draw.state.stages.vertexWaveSize = 32u;
+        draw.state.stages.fragmentWaveSize = wave;
+        std::vector<AgcDriver::Graphics::RegisterRead> reads;
+        AgcDriver::Graphics::RegisterReadLog() = &reads;
+        try {
+            DecodeGraphicsPrograms(draw, queue, registry, false, true);
+            draw.pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, {}, AgcDriver::Graphics::PixelProgramSkipped(queue));
+        } catch (...) {
+            AgcDriver::Graphics::RegisterReadLog() = nullptr;
+            throw;
+        }
+        AgcDriver::Graphics::RegisterReadLog() = nullptr;
+        Require(draw.roles.size() == 2 && draw.roles.back() == ShaderRecompiler::ProgramRole::Fragment, "null substitution changed stage linking");
+        Require(draw.programs.back().snapshot == null && draw.programs.back().userData.empty(), "an unused retained pixel program consumed unrelated userdata");
+        Require(draw.pixel.interpolatorCount == 0 && !draw.pixel.depthExportEnable && !draw.pixel.stencilRefExportEnable && !draw.pixel.sampleMaskExportEnable && !draw.pixel.pixelKillEnable, "the substituted pixel program inherited stale inputs or exports");
+        for (const auto mode : draw.pixel.targetOutputMode) Require(mode == 0, "the substituted pixel program inherited stale color formats");
+        for (const auto read : reads) {
+            Require(AgcDriver::Graphics::DrawKeyCovers(read), "a pixel substitution dependency is absent from the draw key");
+            Require(read.bank != AgcDriver::Graphics::RegisterBank::Shader || read.offset < 0x00bu || read.offset > 0x02bu, "the substituted program read fragment resources or userdata");
+        }
+        const auto stages = PrepareGraphicsStages(draw, device.Target());
+        Require(stages.size() == 2 && stages.back().entry.handle != nullptr, "a retained-format null program was not prepared");
+        std::vector<ShaderRecompiler::LinkedProgram> linked;
+        std::vector<ShaderRecompiler::MemoryRegion> memory;
+        for (std::size_t i = 0; i < draw.programs.size(); ++i) {
+            const auto& program = draw.programs[i];
+            linked.push_back({draw.roles[i], program.binary, program.userDataBase, program.firstUserSgpr, program.userData});
+            memory.insert(memory.end(), program.memory.begin(), program.memory.end());
+        }
+        const auto& fragment = draw.programs.back();
+        const auto offset = stages.back().pushOffset;
+        const auto room = AgcDriver::Graphics::PipelinePushSlotBytes - offset % AgcDriver::Graphics::PipelinePushSlotBytes;
+        ShaderRecompiler::RecompileRequest request{fragment.binary, {wave, 0, {}, {}, draw.pixel, {}, memory, RegisteredFloatMode(*null)}, device.Target(), {0, 0, offset, room}, ShaderRecompiler::GraphicsCompileContext{0, linked, {}, {}, {}}};
+        static_cast<void>(InvocationFor(*null, fragment.codeOffset, request));
+        const auto handle = SourceHandleFor(*null, fragment.codeOffset, request);
+        const auto entries = null->prepared->entries.size();
+        static_cast<void>(InvocationFor(*null, fragment.codeOffset, request));
+        Require(null->prepared->entries.size() == entries && SourceHandleFor(*null, fragment.codeOffset, request) == handle, "a repeated null program did not reuse its prepared variant");
+    }
+    queue.shader[0x00bu] = 2;
+    queue.shader[0x00cu] = 0x12345678u;
+    for (const auto trigger : {0u, 1u, 2u}) {
+        auto active = queue;
+        if (trigger == 0) active.context[0x8e] = active.context[0x8f] = 0xf;
+        if (trigger == 1) active.context[0x203] = 0x400;
+        if (trigger == 2) active.context[0x2dc] = 0xaa01;
+        DrawDecode draw{};
+        draw.state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex;
+        DecodeGraphicsPrograms(draw, active, registry, false, true);
+        Require(draw.programs.back().snapshot == retained.snapshot && draw.programs.back().userData == std::vector<std::uint32_t>{0x12345678u}, "an enabled consumer did not restore the retained pixel program");
+    }
+}
+
 void NullPixelMatchesRegistration(AgcDriver::VulkanDevice& device) {
     using namespace AgcDriver::DriverDetail;
     Fixture front;
@@ -449,6 +530,7 @@ int main(int argc, char** argv) {
             return 0;
         }
         const auto dump = argc == 2 ? std::filesystem::path(argv[1]) : std::filesystem::path{};
+        RetainedPixelMatchesNullProgram(*device);
         NullPixelMatchesRegistration(*device);
         Check(*device, AgcDriver::Graphics::ShaderPath::Vertex, dump);
         Check(*device, AgcDriver::Graphics::ShaderPath::Geometry, dump);