fix(shader): keep host interpolation across entry EXEC copies

core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp

@@ -1,6 +1,7 @@
 #include "IntermediateRepresentation/IrBuilder.hpp"
 #include "Optimization/ConstantFolder.hpp"
 #include "Optimization/DeadCodeEliminator.hpp"
+#include <array>
 #include <cstdint>
 #include <cstdio>
 #include <exception>
@@ -87,6 +88,71 @@ bool vertexProjection(std::uint32_t low, std::uint32_t hostSubgroup, std::uint32
     return retainedLane;
 }
 
+enum class QuadMask {
+    Expanded,
+    Intersected,
+    Nested,
+    SeparateHalves,
+    RealOnly,
+    Partial,
+    WrongHalf,
+    Divergent,
+};
+
+bool quadProjection(std::uint32_t wave, QuadMask shape, bool expected, IrShaderStage stage = IrShaderStage::Pixel) {
+    IrProgram program;
+    program.Resources().stage = stage;
+    program.SetWaveSize(wave);
+    auto& block = program.CreateBlock();
+    program.SetEntryBlock(block);
+    program.BlockOrder().push_back(&block);
+    IrBuilder builder(program);
+    builder.SetInsertionPoint(block);
+    auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {});
+    if (shape == QuadMask::Divergent) {
+        auto& taken = program.CreateBlock();
+        auto& other = program.CreateBlock();
+        program.BlockOrder().push_back(&taken);
+        program.BlockOrder().push_back(&other);
+        builder.BranchConditional(builder.ULessThan(lane, builder.Constant(16u)), taken, other);
+        builder.SetInsertionPoint(other);
+        static_cast<void>(builder.Emit(IrOpcode::Return, IrType::Void, {}));
+        builder.SetInsertionPoint(taken);
+    }
+    const auto ballot = [&](IrValue& condition) {
+        auto& mask = builder.Emit(IrOpcode::Ballot, IrType::U32x4, {&condition});
+        return std::array<IrValue*, 2>{&builder.CompositeExtract(mask, 0u), wave == 64u ? &builder.CompositeExtract(mask, 1u) : &builder.Constant(0u)};
+    };
+    const auto expand = [&](const std::array<IrValue*, 2>& words) {
+        auto& wide = builder.ConstructU64(*words[0], *words[1]);
+        auto& mask = builder.Emit(IrOpcode::WqmU64, IrType::U64, {&wide});
+        return std::array<IrValue*, 2>{&builder.CompositeExtract(mask, 0u), &builder.CompositeExtract(mask, 1u)};
+    };
+    auto& helper = builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast<std::uint32_t>(StageInputKind::HelperInvocation)), &builder.Constant(0u)});
+    const auto real = ballot(builder.IEqual(helper, builder.Constant(0u)));
+    const auto quads = expand(real);
+    const auto host = expand(ballot(builder.ConstantBool(true)));
+    auto words = quads;
+    if (shape == QuadMask::Intersected || shape == QuadMask::Nested || shape == QuadMask::SeparateHalves || shape == QuadMask::RealOnly) {
+        const auto highHost = shape == QuadMask::SeparateHalves ? expand(ballot(builder.ConstantBool(true))) : host;
+        const auto source = shape == QuadMask::Nested || shape == QuadMask::RealOnly ? real : quads;
+        words = {&builder.BitwiseAnd(*source[0], *host[0]), &builder.BitwiseAnd(*source[1], *highHost[1])};
+        if (shape == QuadMask::Nested) words = expand(words);
+    }
+    if (shape == QuadMask::Partial) {
+        words = {&builder.BitwiseAnd(*words[0], builder.Constant(0x0000ffffu)), &builder.Constant(0u)};
+    }
+    if (shape == QuadMask::WrongHalf) words[1] = words[0];
+    IrValue& word = wave == 64u ? builder.Select(builder.ULessThan(lane, builder.Constant(32u)), *words[0], *words[1]) : *words[0];
+    auto& shifted = builder.ShiftRightLogical(word, builder.BitwiseAnd(lane, builder.Constant(31u)));
+    auto& test = builder.INotEqual(builder.BitwiseAnd(shifted, builder.Constant(1u)), builder.Constant(0u));
+    auto& user = builder.Emit(IrOpcode::Reference, IrType::Void, {&test});
+    static_cast<void>(builder.Emit(IrOpcode::Return, IrType::Void, {}));
+    ConstantFolder{}.Fold(program);
+    const auto* result = user.Argument(0)->Resolve();
+    return expected ? result->HasImmediate() && result->ImmediateBool() : !result->HasImmediate();
+}
+
 bool run(const char* name, bool passed) {
     if (!passed) std::fprintf(stderr, "%s\n", name);
     return passed;
@@ -97,6 +163,17 @@ bool run(const char* name, bool passed) {
 int main() {
     try {
         bool passed = true;
+        for (const auto wave : {32u, 64u}) {
+            passed &= run("a whole-quad entry mask covers pixel helpers", quadProjection(wave, QuadMask::Expanded, true));
+            passed &= run("intersecting whole-quad entry masks preserves pixel helpers", quadProjection(wave, QuadMask::Intersected, true));
+            passed &= run("a nested whole-quad entry mask covers pixel helpers", quadProjection(wave, QuadMask::Nested, true));
+            passed &= run("independent low and high whole-quad masks cover pixel helpers", quadProjection(wave, QuadMask::SeparateHalves, true));
+            passed &= run("an unexpanded entry mask must keep the helper bit test", quadProjection(wave, QuadMask::RealOnly, false));
+            passed &= run("a partial whole-quad mask must keep the lane bit test", quadProjection(wave, QuadMask::Partial, false));
+            passed &= run("a divergent whole-quad ballot must keep the lane bit test", quadProjection(wave, QuadMask::Divergent, false));
+            passed &= run("whole-quad entry proofs do not apply to other stages", quadProjection(wave, QuadMask::Intersected, false, IrShaderStage::Vertex));
+        }
+        passed &= run("a low word cannot prove the high-half helper bits", quadProjection(64u, QuadMask::WrongHalf, false));
         passed &= run("an all-zero word must fold the lane bit test to false", foldsTo(0u, false, true));
         passed &= run("an all-one word must fold the lane bit test to true", foldsTo(0xffffffffu, true, true));
         passed &= run("a mixed word must keep the lane bit test", foldsTo(0x0000ffffu, false, true));

core/libs/prx/libSceAgcDriver/tests/execution/PixelExecInterpolation.cpp

@@ -0,0 +1,159 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 48;
+alignas(256) std::array<float, Width * Height * 4> Pixels{};
+constexpr std::array<std::array<float, 3>, 3> Corners{{{-0.8f, -0.8f, 1.0f}, {0.9f, -0.6f, 2.5f}, {-0.5f, 0.85f, 0.5f}}};
+
+alignas(256) constexpr std::array<std::uint32_t, 8> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf800020f, 0x03020100, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 8> ExecCopyPixelCode{
+    0xbefe0a7e, 0xc8080000, 0xbe80047e, 0xbefe0400, 0xc8090001, 0xf800180f, 0x02020202, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> RepeatedWqmPixelCode{
+    0xbefe0a7e, 0xc8080000, 0xbefe0a7e, 0xc8090001, 0xf800180f, 0x02020202, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> ExpandedExecPixelCode{
+    0xc8080000, 0xbefe0a7e, 0xc8090001, 0xf800180f, 0x02020202, 0xbf810000,
+};
+
+std::array<std::array<float, 4>, 3> ClipPositions() {
+    std::array<std::array<float, 4>, 3> positions{};
+    for (std::size_t i = 0; i < Corners.size(); ++i) positions[i] = {Corners[i][0] * Corners[i][2], Corners[i][1] * Corners[i][2], 0.5f * Corners[i][2], Corners[i][2]};
+    return positions;
+}
+
+struct Expected {
+    bool inside;
+    float value;
+};
+
+Expected Interpolated(std::uint32_t x, std::uint32_t y, bool snapped) {
+    std::array<std::array<double, 2>, 3> window{};
+    for (std::size_t i = 0; i < Corners.size(); ++i) window[i] = {(Corners[i][0] + 1.0) * Width / 2.0, (1.0 - Corners[i][1]) * Height / 2.0};
+    if (snapped) {
+        for (auto& corner : window) for (auto& coordinate : corner) coordinate = std::round(coordinate * 256.0) / 256.0;
+    }
+    const double px = x + 0.5;
+    const double py = y + 0.5;
+    const auto edge = [](const std::array<double, 2>& a, const std::array<double, 2>& b, double cx, double cy) { return (b[0] - a[0]) * (cy - a[1]) - (b[1] - a[1]) * (cx - a[0]); };
+    const double area = edge(window[0], window[1], window[2][0], window[2][1]);
+    const std::array<double, 3> weights{edge(window[1], window[2], px, py) / area, edge(window[2], window[0], px, py) / area, edge(window[0], window[1], px, py) / area};
+    double denominator = 0.0;
+    for (std::size_t i = 0; i < Corners.size(); ++i) denominator += weights[i] / Corners[i][2];
+    const auto positions = ClipPositions();
+    double value = 0.0;
+    for (std::size_t i = 0; i < Corners.size(); ++i) value += weights[i] / Corners[i][2] / denominator * positions[i][0];
+    return {std::min({weights[0], weights[1], weights[2]}) > 0.02, static_cast<float>(value)};
+}
+
+std::uint32_t CheckPixels() {
+    std::uint32_t checked = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto expected = Interpolated(x, y, false);
+            if (!expected.inside) continue;
+            const auto snapped = Interpolated(x, y, true);
+            for (std::uint32_t channel = 0; channel < 4; ++channel) {
+                const auto value = Pixels[(y * Width + x) * 4u + channel];
+                Require(std::fabs(value - expected.value) <= 1e-4f || std::fabs(value - snapped.value) <= 1e-4f, "EXEC interpolation: incorrect pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel));
+            }
+            ++checked;
+        }
+    }
+    Require(checked > 200u, "EXEC interpolation: too few interior pixels were checked");
+    return checked;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) {
+    auto target = device.Target();
+    target.fragmentShaderBarycentricEnabled = false;
+    static const auto positions = ClipPositions();
+    const auto address = reinterpret_cast<std::uintptr_t>(positions.data());
+    const std::array<std::uint32_t, 4> vertexUserData{static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u), 3u, 0x01016facu};
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target, {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = waveSize == 32u;
+    pixel.interpolatorCount = 1;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCenter);
+    pixel.hasPerspectiveCenterVgpr = true;
+    pixel.executeOnNoop = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    const auto compile = [&](std::span<const std::uint32_t> code) {
+        const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+        ShaderRecompiler::RecompileRequest fragment{
+            {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+            {waveSize, 0, {}, std::nullopt, pixel, std::nullopt, memory}, target, PixelPushLayout(vertexPush, target)
+        };
+        fragment.useCache = false;
+        return ShaderRecompiler::Recompile(fragment);
+    };
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Pixels.data()), {Width, Height}, VK_FORMAT_R32G32B32A32_SFLOAT, sizeof(Pixels), 0xe4u, AgcDriver::Graphics::ColorTileMode::Linear, 16u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Width, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+    for (const auto code : {std::span<const std::uint32_t>(ExecCopyPixelCode), std::span<const std::uint32_t>(RepeatedWqmPixelCode)}) {
+        const auto result = compile(code);
+        Require(!result.barycentricEmulation.active, "changing EXEC between v_interp_p1/p2 needed a generated stage");
+        Pixels.fill(-7.0f);
+        const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderStage::Vertex, &vertexResult, 0}, {ShaderStage::Fragment, &result, PixelPushOffset(vertexPush, target)}}};
+        device.Draw(state, {0, 3, 0, 1, 0, false}, shaders);
+        device.WaitIdle();
+        std::printf("wave%u %zu words: host interpolation matched %u pixels\n", waveSize, code.size(), CheckPixels());
+    }
+    Require(compile(ExpandedExecPixelCode).barycentricEmulation.active, "a partial interpolation written before WQM incorrectly used host interpolation");
+    std::printf("wave%u: expanded EXEC keeps barycentric emulation\n", waveSize);
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (const auto wave : {32u, 64u}) Run(*device, wave);
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/shader/recompiler/Optimization/src/ConstantFolder.cpp

@@ -99,14 +99,23 @@ bool runsAllStartingLanes(const IrProgram& program, const IrBlock* target) {
     return false;
 }
 
-bool holdsStartingLanes(const IrProgram& program, const IrValue& word, std::uint32_t index, std::uint32_t depth = 0) {
+enum class LaneCoverage { Real, Quads };
+
+bool holdsStartingLanes(const IrProgram& program, const IrValue& word, std::uint32_t index, LaneCoverage coverage, std::uint32_t depth = 0) {
     if (depth > 8u) return false;
     if (word.Opcode() == IrOpcode::BitwiseAnd32) {
-        return holdsStartingLanes(program, resolveArg(word, 0), index, depth + 1u) && holdsStartingLanes(program, resolveArg(word, 1), index, depth + 1u);
+        return holdsStartingLanes(program, resolveArg(word, 0), index, coverage, depth + 1u) && holdsStartingLanes(program, resolveArg(word, 1), index, coverage, depth + 1u);
     }
     if (word.Opcode() == IrOpcode::BitwiseOr32) {
-        return holdsStartingLanes(program, resolveArg(word, 0), index, depth + 1u) || holdsStartingLanes(program, resolveArg(word, 1), index, depth + 1u);
+        return holdsStartingLanes(program, resolveArg(word, 0), index, coverage, depth + 1u) || holdsStartingLanes(program, resolveArg(word, 1), index, coverage, depth + 1u);
+    }
+    if (word.Opcode() == IrOpcode::CompositeExtractU64 && isImmediateU32(resolveArg(word, 1), index)) {
+        const auto& mask = resolveArg(word, 0);
+        if (mask.Opcode() != IrOpcode::WqmU64) return false;
+        const auto& lanes = resolveArg(mask, 0);
+        return lanes.Opcode() == IrOpcode::CompositeConstructU64 && holdsStartingLanes(program, resolveArg(lanes, index), index, LaneCoverage::Real, depth + 1u);
     }
+    if (coverage == LaneCoverage::Quads) return false;
     if (word.Opcode() != IrOpcode::CompositeExtractU32x4 || !isImmediateU32(resolveArg(word, 1), index)) return false;
     const auto& ballot = resolveArg(word, 0);
     if (ballot.Opcode() != IrOpcode::Ballot || !runsAllStartingLanes(program, ballot.Parent())) return false;
@@ -145,13 +154,8 @@ bool foldStartingQuadBitTest(const IrProgram& program, IrBuilder& builder, IrVal
     } else if (program.WaveSize() != 32u) {
         return false;
     }
-    if (low->Opcode() != IrOpcode::CompositeExtractU64 || !isImmediateU32(resolveArg(*low, 1), 0u)) return false;
-    const auto& wqm = resolveArg(*low, 0);
-    if (high != nullptr && (high->Opcode() != IrOpcode::CompositeExtractU64 || &resolveArg(*high, 0) != &wqm || !isImmediateU32(resolveArg(*high, 1), 1u))) return false;
-    if (wqm.Opcode() != IrOpcode::WqmU64) return false;
-    const auto& lanes = resolveArg(wqm, 0);
-    if (lanes.Opcode() != IrOpcode::CompositeConstructU64 || !holdsStartingLanes(program, resolveArg(lanes, 0), 0u)) return false;
-    if (high != nullptr && !holdsStartingLanes(program, resolveArg(lanes, 1), 1u)) return false;
+    if (!holdsStartingLanes(program, *low, 0u, LaneCoverage::Quads)) return false;
+    if (high != nullptr && !holdsStartingLanes(program, *high, 1u, LaneCoverage::Quads)) return false;
     replaceWith(inst, builder.ConstantBool(true));
     return true;
 }