diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 130508267e2..32efe223866 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -358,6 +359,9 @@ void verifyEntryRoundTrip() { auto previousVersion = file; previousVersion[4] = static_cast(ShaderDiskCache::FormatVersion - 1u); require(ShaderDiskCache::DecodeEntry(previousVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the previous format version loads"); + auto upstreamVersion = file; + upstreamVersion[4] = std::byte{30}; + require(ShaderDiskCache::DecodeEntry(upstreamVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the upstream resource plan format loads"); auto incompatible = variant; incompatible.artifact.runtimeAbiVersion = RuntimeAbi::Version + 1u; const auto incompatibleFile = ShaderDiskCache::EncodeEntry(key, incompatible); @@ -1554,6 +1558,9 @@ void verifyPlanEntryDamage(const std::vector& key, const IrResourcePl auto previousFormat = file; previousFormat[4] = static_cast(ShaderDiskCache::FormatVersion - 1u); require(ShaderDiskCache::DecodePlanEntry(previousFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of the previous format loads"); + auto upstreamFormat = file; + upstreamFormat[4] = std::byte{30}; + require(ShaderDiskCache::DecodePlanEntry(upstreamFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry with the upstream resource plan format loads"); auto otherSource = file; otherSource[8] ^= std::byte{1}; require(ShaderDiskCache::DecodePlanEntry(otherSource, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of another source version loads"); @@ -1621,7 +1628,23 @@ void verifyMalformedPlans() { require(!ShaderDiskCache::EncodePlan(emptied, bytes), "a plan with an empty value slot was encoded"); } +void verifyIndirectImagePlanModes() { + std::set> tableEncodings; + for (std::uint32_t flags = 0u; flags < 8u; ++flags) { + auto plan = handBuiltPlan(); + auto& table = *plan.descriptorSources[1].indirectImage; + table.laneKey = (flags & 1u) != 0u; + table.indexed = (flags & 2u) != 0u; + table.implicitDerivatives = (flags & 4u) != 0u; + const auto bytes = encodedPlan(plan); + require(tableEncodings.insert(bytes).second, "indirect image table modes share a disk plan encoding"); + IrResourcePlan decoded; + require(ShaderDiskCache::DecodePlan(bytes, decoded) && samePlan(plan, decoded), "indirect image table modes changed in a disk plan round trip"); + } +} + void verifyPlanRoundTrip() { + verifyIndirectImagePlanModes(); const auto handBuilt = handBuiltPlan(); const auto handBytes = encodedPlan(handBuilt); IrResourcePlan handDecoded; @@ -1881,6 +1904,13 @@ void verifyDefaultDirectory(const char* self) { int main(int argc, char** argv) { try { + if (argc == 2 && std::string_view(argv[1]) == "--resource-plan-codec") { + verifyResultRoundTrip(); + verifyEntryRoundTrip(); + verifyIndirectImagePlanModes(); + std::cout << "resource plan codec tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--load") return runLoadingProcess(); if (argc == 2 && std::string_view(argv[1]) == "--no-failure-memo") return runWithoutFailureMemo(); if (argc == 2 && std::string_view(argv[1]).starts_with("--load-plan")) return runPlanLoadingProcess(argv[1]); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp new file mode 100644 index 00000000000..4de2a85f283 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WaterfallImages.cpp @@ -0,0 +1,310 @@ +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using namespace ShaderRecompiler; + +constexpr std::uint32_t Records = 4; +constexpr std::uint32_t Textures = 3; + +struct alignas(4096) GuestData { + std::array, Records> table{}; + std::array output{}; +}; + +alignas(256) std::array, Textures> textures{}; +std::array guest; + +alignas(256) constexpr std::array WholeWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, + 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PartialWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, + 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, + 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, + 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, + 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PartialWave64{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x36200087u, 0xbe9c047eu, 0x7daa2087u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94047eu, 0xbe961414u, 0xd7600008u, 0x00002d02u, + 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, + 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98047eu, 0x7da40408u, + 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, + 0x00020a0eu, 0xbe9a047eu, 0x8a941a14u, 0xbefe0418u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe041cu, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array GuardedWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e1402c1u, 0xbe9c037eu, 0x7da80202u, 0xbf880026u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, + 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, + 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, + 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, + 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array StaleWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040208u, 0x7e060209u, 0x7e08020au, 0x7e0a020bu, + 0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0x36200087u, 0xbe9c037eu, 0x7daa2087u, 0xe0382000u, + 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbefe031cu, 0xbe94037eu, 0xbe961314u, 0xd7600008u, + 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, + 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, + 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, + 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xbefe031cu, + 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array FirstLaneWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0x7e100502u, 0x7e120503u, 0x7e140504u, 0x7e160505u, 0x7e180506u, 0x7e1a0507u, + 0x7e1c0508u, 0x7e1e0509u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, + 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a7e1a18u, 0xbf85ffeau, + 0xbf8c3f70u, 0xbefe0314u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array Compare64Wave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7de40408u, 0x7de4080au, 0x7de40c0cu, 0x7de4100eu, + 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe4u, 0xbf8c3f70u, 0xe0702000u, + 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array MaskedStaleWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0x7e040280u, 0x7e060280u, 0x7e080280u, 0x7e0a0280u, + 0x7e0c0280u, 0x7e0e0280u, 0x7e100280u, 0x7e120280u, 0x36200087u, 0xbe9c037eu, 0x7da42087u, 0x7e040208u, + 0x7e060209u, 0x7e08020au, 0x7e0a020bu, 0x7e0c020cu, 0x7e0e020du, 0x7e10020eu, 0x7e12020fu, 0xbefe031cu, + 0x7daa2087u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, 0xbf8c3f70u, 0xbefe031cu, 0xbe94037eu, + 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, 0x00002d04u, 0xd760000bu, + 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, 0x00002d08u, 0xd760000fu, + 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, 0x7da40c0cu, 0x7da40e0du, + 0x7da4100eu, 0x7da4120fu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xbf85ffe0u, + 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array SevenDwordsWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0xf0000f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, + 0xbf85ffe1u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array OutsideWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xbe9a037eu, 0x8a141a14u, 0xbefe0318u, 0xf0000f08u, + 0x00020a0eu, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array StoreWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf0200f08u, 0x00020a0eu, 0xbe9a037eu, 0x8a141a14u, + 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +alignas(256) constexpr std::array LodWave32{ + 0x36020083u, 0x2c1c0082u, 0x7e1e0280u, 0x7e140280u, 0xe0382000u, 0x80000201u, 0xe0382010u, 0x80000601u, + 0xbf8c3f70u, 0xbe94037eu, 0xbe961314u, 0xd7600008u, 0x00002d02u, 0xd7600009u, 0x00002d03u, 0xd760000au, + 0x00002d04u, 0xd760000bu, 0x00002d05u, 0xd760000cu, 0x00002d06u, 0xd760000du, 0x00002d07u, 0xd760000eu, + 0x00002d08u, 0xd760000fu, 0x00002d09u, 0xbe98037eu, 0x7da40408u, 0x7da40609u, 0x7da4080au, 0x7da40a0bu, + 0x7da40c0cu, 0x7da40e0du, 0x7da4100eu, 0x7da4120fu, 0xf1800308u, 0x00820a0eu, 0xbe9a037eu, 0x8a141a14u, + 0xbefe0318u, 0xbf85ffe0u, 0xbf8c3f70u, 0xe0702000u, 0x80010a00u, 0xbf810000u, +}; + +std::array Buffer(const void* pointer, std::uint32_t stride, std::uint32_t records) { + const auto address = reinterpret_cast(pointer); + return {static_cast(address), static_cast(address >> 32u) | (stride << 16u), records, 0x01016facu}; +} + +std::array Image(std::uint32_t texture) { + const auto address = reinterpret_cast(textures[texture].data()); + const std::uint32_t format = texture == 1u ? 22u : 20u; + return {static_cast(address >> 8u), static_cast(address >> 40u) | (format << 20u) | (3u << 30u), 7u, 0x90000facu, 0u, 0u, 0u, 0u}; +} + +std::uint32_t TextureOf(std::uint32_t iteration, std::uint32_t key) { + return key == Records - 1u - iteration ? Textures : (key + iteration) % Textures; +} + +bool DecoratesNonUniform(std::span words) { + for (std::size_t index = 5u; index < words.size();) { + const auto count = words[index] >> 16u; + if (count == 0u) return false; + if ((words[index] & 0xffffu) == spv::OpDecorate && count >= 3u && index + 2u < words.size() && words[index + 2u] == spv::DecorationNonUniform) return true; + index += count; + } + return false; +} + +RecompileRequest Request(const AgcDriver::VulkanDevice& device, std::span code, std::uint32_t wave, std::uint32_t lanes, std::span userData, const std::array& regions) { + const ShaderComputeStageInfo compute{{lanes, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + return {{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {wave, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}}; +} + +std::array UserData(GuestData& data, std::uint32_t lanes) { + const auto table = Buffer(data.table.data(), 32u, Records); + const auto output = Buffer(data.output.data(), 4u, lanes); + const auto fixed = Image(0u); + constexpr std::array sampler{0x00000092u, 0x00fff000u, 0u, 0u}; + std::array userData{}; + std::copy(table.begin(), table.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4u); + std::copy(fixed.begin(), fixed.end(), userData.begin() + 8u); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 16u); + return userData; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t wave, bool partial, const std::string& what) { + for (std::uint32_t texture = 0u; texture < Textures; ++texture) { + for (std::uint32_t x = 0u; x < textures[texture].size(); ++x) textures[texture][x] = 0x1000u * (texture + 1u) + x; + } + const auto lanes = wave == 64u ? 64u : std::min(32u, device.Target().subgroupSize); + CompiledShaderArtifact artifact; + for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) { + auto& data = guest[iteration]; + for (std::uint32_t key = 0u; key < Records; ++key) { + const auto texture = TextureOf(iteration, key); + data.table[key] = texture == Textures ? std::array{} : Image(texture); + } + data.output.fill(0xdeadbeefu); + const auto userData = UserData(data, lanes); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + const auto shader = Recompile(Request(device, code, wave, lanes, userData, regions)); + Require(DecoratesNonUniform(shader.spirv.Words()), what + ": the per-lane table slot is not decorated NonUniform"); + if (artifact.variantId == 0u) artifact = shader; + else Require(shader.cacheHit && shader.variantId == artifact.variantId, what + ": a new table changed the compiled artifact"); + device.Dispatch(shader, 1u, 1u, 1u); + device.SubmitRecorded(false); + } + device.WaitIdle(); + for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) { + for (std::uint32_t lane = 0u; lane < lanes; ++lane) { + const auto texture = TextureOf(iteration, lane % Records); + const bool skipped = partial && lane % 8u == 7u; + const auto expected = skipped || texture == Textures ? 0u : textures[texture][lane / 4u]; + const auto actual = guest[iteration].output[lane]; + Require(actual == expected, what + ": iteration " + std::to_string(iteration) + " lane " + std::to_string(lane) + " read " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } +} + +void RequireRefused(AgcDriver::VulkanDevice& device, std::span code, const std::string& what) { + auto& data = guest[0]; + const auto userData = UserData(data, 32u); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + auto request = Request(device, code, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); + request.useCache = false; + bool refused = false; + try { + static_cast(Recompile(request)); + } catch (const std::exception& error) { + refused = std::string_view(error.what()).find("GetImageResource dword 0 is not a valid runtime value") != std::string_view::npos; + } + Require(refused, what + " was accepted"); +} + +void RunWithoutRecords(AgcDriver::VulkanDevice& device, const std::array& table, const std::string& what) { + auto& data = guest[0]; + data.output.fill(0xdeadbeefu); + const auto lanes = std::min(32u, device.Target().subgroupSize); + auto userData = UserData(data, lanes); + std::copy(table.begin(), table.end(), userData.begin()); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + std::string failure; + try { + const auto shader = Recompile(Request(device, GuardedWave32, 32u, lanes, userData, regions)); + device.Dispatch(shader, 1u, 1u, 1u); + device.SubmitRecorded(false); + device.WaitIdle(); + } catch (const std::exception& error) { + failure = error.what(); + } + Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u)); + for (std::uint32_t lane = 0u; lane < lanes; ++lane) { + const auto actual = data.output[lane]; + Require(actual == 0xffffffffu, what + ": lane " + std::to_string(lane) + " read " + std::to_string(actual) + ", expected the value it holds outside the loop"); + } +} + +void RunLod(AgcDriver::VulkanDevice& device, const std::array& records, bool refused, const std::string& what) { + auto& data = guest[0]; + for (std::uint32_t key = 0u; key < Records; ++key) data.table[key] = records[key] == Textures ? std::array{} : Image(records[key]); + const auto userData = UserData(data, 32u); + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + auto request = Request(device, LodWave32, 32u, std::min(32u, device.Target().subgroupSize), userData, regions); + request.useCache = false; + std::string failure; + try { + const auto shader = Recompile(request); + Require(DecoratesNonUniform(shader.spirv.Words()), what + ": the per-lane table slot is not decorated NonUniform"); + } catch (const std::exception& error) { + failure = error.what(); + } + if (refused) Require(failure.find("different runtime modes under an implicit-derivative access") != std::string::npos, what + " was not refused: " + failure.substr(0u, 300u)); + else Require(failure.empty(), what + " failed: " + failure.substr(0u, 300u)); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, WholeWave32, 32u, false, "wave32 waterfall over the whole wave"); + Run(*device, PartialWave32, 32u, true, "wave32 waterfall under a partial EXEC"); + if (device->Target().subgroupSize >= 32u) Run(*device, PartialWave64, 64u, true, "wave64 waterfall under a partial EXEC"); + Run(*device, FirstLaneWave32, 32u, false, "wave32 waterfall picking lanes with v_readfirstlane"); + Run(*device, Compare64Wave32, 32u, false, "wave32 waterfall comparing the T# with v_cmpx_eq_u64"); + Run(*device, GuardedWave32, 32u, false, "wave32 waterfall behind a branch over the lanes inside the table"); + RunWithoutRecords(*device, {0u, 0u, 0u, 0u}, "a waterfall no lane enters over a null table V#"); + RunWithoutRecords(*device, Buffer(guest[0].table.data(), 0u, 0u), "a waterfall no lane enters over a table V# with stride 0 and no records"); + auto unchecked = Buffer(guest[0].table.data(), 32u, 0u); + unchecked[1] |= 1u << 31u; + unchecked[3] |= (1u << 23u) | (2u << 28u); + RunWithoutRecords(*device, unchecked, "a waterfall no lane enters over a swizzled ADD_TID table V# with OOB_SELECT 2 and no records"); + RequireRefused(*device, StaleWave32, "a waterfall over a table T# and a T# left in the VGPRs of lanes the load skipped"); + RequireRefused(*device, MaskedStaleWave32, "a waterfall over a table T# and a T# written under EXEC to the VGPRs of lanes the load skipped"); + RequireRefused(*device, SevenDwordsWave32, "a waterfall that compares seven of the eight T# dwords"); + RequireRefused(*device, OutsideWave32, "an image load after the waterfall restores EXEC"); + RequireRefused(*device, StoreWave32, "an image store in a waterfall"); + RunLod(*device, {1u, 1u, 1u, 1u}, false, "an LOD query in a waterfall over a table of one image mode"); + RunLod(*device, {0u, 1u, 0u, 1u}, true, "an LOD query in a waterfall over a table of float and integer images"); + RunLod(*device, {Textures, 0u, 0u, 0u}, false, "an LOD query in a waterfall over a null record 0 and integer images"); + RunLod(*device, {Textures, 0u, 1u, 0u}, true, "an LOD query in a waterfall over a null record 0 and float and integer images"); + std::cout << "waterfall image tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index f9df378d48d..12d5018dc73 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -20,8 +20,9 @@ struct BlockInfo { }; struct DescriptorSource { - // A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32`, the key a - // wave-uniform runtime value (handle argument `keyArg`). With `hasMaterial` the key is itself + // A T# loaded from a table buffer (`heapSource`) at `entryOffset + key * 32` (the table V#'s + // stride when `indexed`), the key a wave-uniform runtime value (handle argument `keyArg`), or + // each lane's own value when `laneKey`. With `hasMaterial` the key is itself // `M[readfirstlane(i) * selectorStride + selectorOffset]` over `materialSource`, so the key // set can be enumerated from the material records (see ResourceMaterializer). struct IndirectImage { @@ -32,6 +33,9 @@ struct DescriptorSource { std::uint32_t keyArg = 0; std::uint32_t entryOffset = 0; bool hasMaterial = false; + bool laneKey = false; + bool indexed = false; + bool implicitDerivatives = false; bool operator==(const IndirectImage& other) const = default; }; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 37da1a30cfe..be633c5df72 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -333,8 +333,19 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con const ShaderBufferResource heap = decodeBufferDescriptor(heapValue); const std::uint64_t heapSize = heap.GetSize(); if (heap.Type() != 0u || heap.Base48() > std::numeric_limits::max() - heapSize) throw std::runtime_error("bindless heap buffer is invalid"); - if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor"); - const auto entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; + std::uint64_t entryBytes = TableEntryBytes; + std::uint32_t entries = 0u; + if (table.indexed) { + if (heap.NumRecords() != 0u) { + if (heap.SwizzleEnabled() || heap.AddTid() || heap.OutOfBounds() > 1u) throw std::runtime_error("bindless image table indexed through a V# with swizzling, ADD_TID or an unbounded range check is not implemented"); + entryBytes = heap.Stride(); + if (entryBytes < static_cast(table.entryOffset) + TableEntryBytes) throw std::runtime_error("bindless heap record is smaller than its descriptor"); + entries = heap.NumRecords(); + } + } else { + if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor"); + entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; + } const auto readWord = [&](std::uint64_t address, std::uint32_t& word) { if (!runtime.readMemory(runtime.userContext, address, &word)) { throw std::runtime_error("failed to read a bindless image table from memory"); @@ -388,11 +399,12 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con const std::uint64_t heapBase = heap.Base48(); std::vector candidates(keys.size()); std::vector valid(keys.size(), 0u); + std::vector modes(keys.size(), 0u); std::uint32_t paddedNull = 0u; for (std::size_t i = 0u; i < keys.size(); ++i) { auto& candidate = candidates[i]; candidate.dwordCount = 8u; - const std::uint64_t address = heapBase + table.entryOffset + static_cast(keys[i]) * TableEntryBytes; + const std::uint64_t address = heapBase + table.entryOffset + static_cast(keys[i]) * entryBytes; for (std::uint32_t dword = 0u; dword < 8u; ++dword) readWord(address + dword * sizeof(std::uint32_t), candidate.dwords[dword]); if (nullImageDescriptor(candidate)) { ++paddedNull; @@ -401,9 +413,17 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con if (!validImageDescriptor(candidate, image.r128)) throw std::runtime_error("bindless image table contains an invalid descriptor"); const auto decoded = decodeImageDescriptor(candidate, image, plan.srgbDecodeFormats); if (decoded.fmask) throw std::runtime_error("bindless FMASK images are unsupported"); - static_cast(ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex))); + modes[i] = ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex)); valid[i] = 1u; } + if (table.laneKey && table.implicitDerivatives) { + std::optional boundMode; + for (std::size_t i = 0u; i < keys.size(); ++i) { + if (valid[i] == 0u) continue; + if (boundMode.has_value() && modes[i] != *boundMode) throw std::runtime_error("bindless image table keyed per lane holds images of different runtime modes under an implicit-derivative access, which is not implemented"); + boundMode = modes[i]; + } + } resolution.mapping.clear(); for (std::size_t i = 0u; i < keys.size(); ++i) { if (valid[i] != 0u) resolution.mapping.emplace_back(keys[i], static_cast(i)); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index ee4eaee5f81..749c249ad85 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -112,6 +112,7 @@ class Tracker { if (!m_program.Resources().srtPlanComplete) { fail("SRT plan is not ready"); } + FoldWaterfallDescriptors(); SplitDescriptorPhis(); PlanIndirectImages(); std::unordered_map blockIndices; @@ -150,7 +151,9 @@ class Tracker { plan.handle->ReplaceArgument(dword, plan.key); } for (const auto index : plan.memory) { - m_program.Resources().memoryInfo[index].planningOnly = true; + if (plan.planningReads) { + m_program.Resources().memoryInfo[index].planningOnly = true; + } } } std::erase_if(m_program.Metadata().dynamicReads, [&](IrValue* value) { @@ -184,6 +187,14 @@ class Tracker { std::array roots {}; std::array memory {}; std::array reads {}; + bool planningReads = true; + }; + + struct LaneTableRead { + IrValue* written = nullptr; + IrValue* other = nullptr; + IrValue* load = nullptr; + std::uint32_t component = 0; }; void MakeSource(const IrValue& handle, std::uint32_t width, bool sampler, bool sampleAdjust, DescriptorSource& descriptor) { @@ -590,6 +601,359 @@ class Tracker { return true; } + static bool MatchLaneTableRead(IrValue* value, LaneTableRead& read) { + value = value->Resolve(); + read = LaneTableRead {}; + if (value->Opcode() == IrOpcode::SelectU32 && value->ArgumentCount() == 3u) { + read.written = value->Argument(0)->Resolve(); + read.other = value->Argument(2)->Resolve(); + value = value->Argument(1)->Resolve(); + } + const auto extract = value->Opcode(); + const bool extracted = extract == IrOpcode::CompositeExtractU32x2 || extract == IrOpcode::CompositeExtractU32x3 || extract == IrOpcode::CompositeExtractU32x4; + if (extracted) { + if (value->ArgumentCount() != 2u || !immediateU32(value->Argument(1), read.component)) { + return false; + } + value = value->Argument(0)->Resolve(); + } + const auto load = value->Opcode(); + const bool wide = load == IrOpcode::LoadBufferU32x2 || load == IrOpcode::LoadBufferU32x3 || load == IrOpcode::LoadBufferU32x4; + if ((wide != extracted) || (!wide && load != IrOpcode::LoadBufferU32) || value->ArgumentCount() != 5u) { + return false; + } + read.load = value; + return true; + } + + bool SameBufferAddress(const IrValue& left, const IrValue& right) const { + if (&left == &right) { + return true; + } + const IrValue* leftHandle = left.Argument(0)->Resolve(); + const IrValue* rightHandle = right.Argument(0)->Resolve(); + if (leftHandle != rightHandle) { + if (leftHandle->Opcode() != IrOpcode::GetBufferResource || rightHandle->Opcode() != IrOpcode::GetBufferResource || leftHandle->ArgumentCount() != 4u || rightHandle->ArgumentCount() != 4u) { + return false; + } + for (std::size_t dword = 0; dword < 4u; dword++) { + if (!EquivalentValue(m_program.Resources(), leftHandle->Argument(dword), rightHandle->Argument(dword))) { + return false; + } + } + } + for (std::size_t argument = 1; argument < 4u; argument++) { + if (!EquivalentValue(m_program.Resources(), left.Argument(argument), right.Argument(argument))) { + return false; + } + } + return true; + } + + bool MayHoldDescriptor(std::array tuple, std::uint32_t depth, std::vector>& visited) const { + for (IrValue*& value : tuple) { + value = value->Resolve(); + } + if (std::ranges::find(visited, tuple) != visited.end()) { + return false; + } + visited.push_back(tuple); + if (std::ranges::all_of(tuple, [](const IrValue* value) { return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == 0u; })) { + return false; + } + constexpr SrtWalker walker; + LaneTableRead read; + const bool runtime = std::ranges::all_of(tuple, [&](const IrValue* value) { return value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); }); + if (runtime || std::ranges::all_of(tuple, [&](IrValue* value) { return MatchLaneTableRead(value, read); })) { + return true; + } + if (depth == phiSearchDepth) { + return true; + } + const IrValue& first = *tuple[0]; + if (first.Opcode() == IrOpcode::SelectU32 && first.ArgumentCount() == 3u) { + const IrValue* condition = first.Argument(0)->Resolve(); + std::array selected {}; + std::array kept {}; + for (std::size_t dword = 0; dword < tuple.size(); dword++) { + const IrValue& value = *tuple[dword]; + if (value.Opcode() != IrOpcode::SelectU32 || value.ArgumentCount() != 3u || value.Argument(0)->Resolve() != condition) { + return false; + } + selected[dword] = value.Argument(1); + kept[dword] = value.Argument(2); + } + return MayHoldDescriptor(selected, depth + 1u, visited) || MayHoldDescriptor(kept, depth + 1u, visited); + } + if (!first.IsPhi()) { + return false; + } + for (std::size_t edge = 0; edge < first.ArgumentCount(); edge++) { + std::array incoming {}; + for (std::size_t dword = 0; dword < tuple.size(); dword++) { + if (!tuple[dword]->IsPhi() || tuple[dword]->Parent() != first.Parent()) { + return false; + } + incoming[dword] = PhiIncoming(*tuple[dword], first.PhiBlock(edge)); + if (incoming[dword] == nullptr) { + return true; + } + } + if (MayHoldDescriptor(incoming, depth + 1u, visited)) { + return true; + } + } + return false; + } + + bool TryMakeLaneTableImage(IrValue& handle, bool implicitDerivatives, IndirectImagePlan& plan) { + if (handle.Opcode() != IrOpcode::GetImageResource || handle.ArgumentCount() != 8u || handle.Parent() == nullptr) { + return false; + } + const auto& memoryInfo = m_program.Resources().memoryInfo; + std::array reads {}; + const MemoryInfo* first = nullptr; + std::uint64_t firstPosition = 0; + for (std::uint32_t dword = 0; dword < reads.size(); dword++) { + auto& read = reads[dword]; + if (!MatchLaneTableRead(handle.Argument(dword), read) || read.written != reads[0].written) { + return false; + } + const auto index = read.load->Flags().index; + if (index >= memoryInfo.size()) { + return false; + } + const auto& memory = memoryInfo[index]; + if (memory.kind != ResourceKind::Buffer || memory.dataBits != 32u || memory.formatted || memory.typed || !memory.idxen || read.component >= memory.dataDwords) { + return false; + } + const std::uint64_t position = static_cast(memory.offset) + sizeof(std::uint32_t) * read.component; + if (dword == 0u) { + first = &memory; + firstPosition = position; + } else if (memory.offen != first->offen || position != firstPosition + sizeof(std::uint32_t) * dword || !SameBufferAddress(*reads[0].load, *read.load)) { + return false; + } + } + if (reads[0].other != nullptr) { + std::array others {}; + std::ranges::transform(reads, others.begin(), &LaneTableRead::other); + std::vector> visited; + if (MayHoldDescriptor(others, 0u, visited)) { + return false; + } + } + + const IrValue& load = *reads[0].load; + std::uint32_t voffset = 0; + std::uint32_t soffset = 0; + if (!immediateU32(load.Argument(3), soffset) || (first->offen && !immediateU32(load.Argument(2), voffset))) { + return false; + } + const std::uint64_t entryOffset = firstPosition + voffset + soffset; + IrValue* key = load.Argument(1)->Resolve(); + if (entryOffset > std::numeric_limits::max() || key->Type() != IrType::U32) { + return false; + } + DescriptorSource heapSource; + std::uint32_t heapSourceIndex = 0; + if (!MakeRuntimeBufferSource(*load.Argument(0)->Resolve(), heapSourceIndex, heapSource)) { + return false; + } + if (reads[0].written != nullptr) { + const std::array arguments {reads[0].written, key, &m_builder.Constant(std::numeric_limits::max())}; + key = EmitBefore(handle, IrOpcode::SelectU32, IrType::U32, arguments); + } + + DescriptorSource::IndirectImage table; + table.heapSource = heapSourceIndex; + table.materialSource = heapSourceIndex; + table.entryOffset = static_cast(entryOffset); + table.laneKey = true; + table.indexed = true; + table.implicitDerivatives = implicitDerivatives; + DescriptorSource imageSource; + imageSource.dwordCount = 8u; + std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin()); + std::copy(heapSource.dwords.begin(), heapSource.dwords.begin() + 4u, imageSource.dwords.begin() + 4u); + for (auto& known : m_sources) { + if (!known.indirectImage.has_value() || !known.indirectImage->laneKey) { + continue; + } + auto shared = *known.indirectImage; + shared.implicitDerivatives = table.implicitDerivatives; + if (shared == table) { + table.implicitDerivatives = known.indirectImage->implicitDerivatives = known.indirectImage->implicitDerivatives || table.implicitDerivatives; + } + } + imageSource.indirectImage = table; + + plan.handle = &handle; + plan.source = InternSource(imageSource); + plan.key = key; + plan.roots = imageSource.dwords; + plan.planningReads = false; + return true; + } + + bool ImplicitDerivatives(const IrValue& inst) const { + if (inst.Opcode() == IrOpcode::ImageQueryLod) { + return true; + } + const auto& memoryInfo = m_program.Resources().memoryInfo; + const auto index = inst.Flags().index; + return inst.Opcode() == IrOpcode::ImageSampleRaw && (index >= memoryInfo.size() || !ImageSampleExplicitLod(memoryInfo[index].imageSampleFlags, m_program.Resources().stage)); + } + + static bool LaneRead(const IrValue& value) { + return (value.Opcode() == IrOpcode::ReadLane || value.Opcode() == IrOpcode::ReadFirstLane) && value.ArgumentCount() == 2u; + } + + static bool FoldableImageRead(IrOpcode op) { + switch (op) { + case IrOpcode::ImageSampleRaw: + case IrOpcode::ImageGatherRaw: + case IrOpcode::ImageQueryLod: + case IrOpcode::ImageQueryDimensions: + case IrOpcode::ImageRead: return true; + default: return false; + } + } + + static IrValue* ObservingCondition(const IrValue& inst) { + IrValue* condition = nullptr; + std::vector pending {&inst}; + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + for (const IrUse& use : value->OperandUses()) { + const IrValue* user = use.user; + if (user->IsIdentity() || (user->Opcode() == IrOpcode::CompositeExtractU32x4 && use.operand == 0u)) { + pending.push_back(user); + continue; + } + if (user->Opcode() != IrOpcode::SelectU32 || user->ArgumentCount() != 3u || use.operand != 1u) { + return nullptr; + } + IrValue* selector = user->Argument(0)->Resolve(); + if (condition != nullptr && selector != condition) { + return nullptr; + } + condition = selector; + } + } + return condition; + } + + using Equalities = std::vector>; + + static Equalities ImpliedEqualities(IrValue* condition) { + constexpr std::size_t limit = 1024u; + Equalities equal; + std::vector> pending {{condition, true}}; + for (std::size_t visited = 0; !pending.empty() && visited < limit; visited++) { + auto [value, holds] = pending.back(); + pending.pop_back(); + value = value->Resolve(); + const auto op = value->Opcode(); + if (op == IrOpcode::LogicalNot && value->ArgumentCount() == 1u) { + pending.emplace_back(value->Argument(0), !holds); + } else if (((op == IrOpcode::LogicalAnd && holds) || (op == IrOpcode::LogicalOr && !holds)) && value->ArgumentCount() == 2u) { + pending.emplace_back(value->Argument(0), holds); + pending.emplace_back(value->Argument(1), holds); + } else if (((op == IrOpcode::IEqual32 && holds) || (op == IrOpcode::INotEqual32 && !holds)) && value->ArgumentCount() == 2u) { + equal.emplace_back(value->Argument(0)->Resolve(), value->Argument(1)->Resolve()); + } else if (((op == IrOpcode::IEqual64 && holds) || (op == IrOpcode::INotEqual64 && !holds)) && value->ArgumentCount() == 2u) { + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + if (left->Opcode() != IrOpcode::CompositeConstructU64 || right->Opcode() != IrOpcode::CompositeConstructU64 || left->ArgumentCount() != 2u || right->ArgumentCount() != 2u) { + continue; + } + for (std::size_t half = 0; half < 2u; half++) { + equal.emplace_back(left->Argument(half)->Resolve(), right->Argument(half)->Resolve()); + } + } + } + return equal; + } + + void FoldLaneReads(IrValue& inst, std::uint32_t slot, IrOpcode expected, std::size_t width, const Equalities& equal) { + IrValue* handle = inst.Argument(slot)->Resolve(); + if (handle->Opcode() != expected || handle->ArgumentCount() != width) { + return; + } + constexpr SrtWalker walker; + std::vector dwords; + bool valid = true; + bool folded = false; + for (std::size_t dword = 0; dword < width; dword++) { + IrValue* value = handle->Argument(dword)->Resolve(); + valid = valid && value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); + if (!LaneRead(*value)) { + dwords.push_back(value); + continue; + } + IrValue* source = value->Argument(0)->Resolve(); + const bool proven = std::ranges::any_of(equal, [&](const auto& pair) { + return (pair.first == value && pair.second == source) || (pair.first == source && pair.second == value); + }); + if (!proven) { + return; + } + dwords.push_back(source); + folded = true; + } + if (valid || !folded) { + return; + } + IrValue* copy = EmitCopy(*handle, dwords, inst); + bool accepted = std::ranges::all_of(dwords, [&](IrValue* value) { + return value->Type() == IrType::U32 && walker.ValidateRuntimeValue(m_program.Resources(), value); + }); + static const bool tables = std::getenv("APS5_NO_BINDLESS_IMAGES") == nullptr; + IndirectImagePlan plan; + if (!accepted && tables && expected == IrOpcode::GetImageResource && TryMakeLaneTableImage(*copy, ImplicitDerivatives(inst), plan)) { + m_indirectImages.push_back(std::move(plan)); + accepted = true; + } + if (!accepted) { + copy->Invalidate(); + copy->Parent()->RemoveInstruction(copy); + return; + } + inst.ReplaceArgument(slot, copy); + m_foldedHandles.push_back(copy); + } + + void FoldWaterfallDescriptors() { + std::vector candidates; + for (auto& block : m_program.Blocks()) { + for (IrValue* inst : block->Instructions()) { + if (FoldableImageRead(inst->Opcode())) { + candidates.push_back(inst); + } + } + } + for (IrValue* inst : candidates) { + const bool sampler = ImageOpcodeInfoOf(inst->Opcode()).needsSampler; + if (inst->Parent() == nullptr || inst->ArgumentCount() < (sampler ? 2u : 1u)) { + continue; + } + IrValue* condition = ObservingCondition(*inst); + if (condition == nullptr) { + continue; + } + const Equalities equal = ImpliedEqualities(condition); + if (equal.empty()) { + continue; + } + FoldLaneReads(*inst, 0u, IrOpcode::GetImageResource, 8u, equal); + if (sampler) { + FoldLaneReads(*inst, 1u, IrOpcode::GetSamplerResource, 4u, equal); + } + } + } + const IndirectImagePlan* FindIndirectImage(const IrValue& handle) const { const auto found = std::ranges::find_if(m_indirectImages, [&](const IndirectImagePlan& plan) { return plan.handle == &handle; @@ -599,7 +963,7 @@ class Tracker { bool IsIndirectPlanningMemory(std::uint32_t index) const { return std::ranges::any_of(m_indirectImages, [&](const IndirectImagePlan& plan) { - return std::ranges::find(plan.memory, index) != plan.memory.end(); + return plan.planningReads && std::ranges::find(plan.memory, index) != plan.memory.end(); }); } @@ -1117,6 +1481,9 @@ class Tracker { } } for (IrValue* inst : candidates) { + if (std::ranges::find(m_foldedHandles, inst->Argument(0)->Resolve()) != m_foldedHandles.end()) { + continue; + } SplitDescriptorPhi(*inst); } } @@ -1494,6 +1861,7 @@ class Tracker { std::vector m_indirectImages; std::unordered_map> m_memoryUsers; bool m_memoryUsersIndexed = false; + std::vector m_foldedHandles; struct EdgeSelectorEntry { IrBlock* block = nullptr; std::vector edges; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 355e720e767..d8880fc6727 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -147,7 +147,7 @@ static_assert(fieldCount() == 21, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 30, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 3, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 7, "IndirectImage changed: update the plan encoder"); +static_assert(fieldCount() == 10, "IndirectImage changed: update the plan encoder"); static_assert(fieldCount() == 4, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); static_assert(fieldCount() == 5, "UniformFill changed: update the plan encoder"); @@ -973,6 +973,9 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { out.Value(table.keyArg); out.Value(table.entryOffset); out.Value(table.hasMaterial); + out.Value(table.laneKey); + out.Value(table.indexed); + out.Value(table.implicitDerivatives); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { out.Value(valueIndex(block.condition)); @@ -1115,6 +1118,9 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { in.Value(table.keyArg); in.Value(table.entryOffset); in.Value(table.hasMaterial); + in.Value(table.laneKey); + in.Value(table.indexed); + in.Value(table.implicitDerivatives); source.indirectImage = indirect ? std::optional(table) : std::nullopt; }); reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) { diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 327eb2f30a6..c3a26c6d84f 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 31; +inline constexpr std::uint32_t FormatVersion = 32; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index 4989ed437a5..ee2e467613b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -72,6 +72,18 @@ std::uint32_t RuntimeMipCount(SpirvEmitterState& state, std::uint32_t resource) return Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpISub, TypeU32(state), last, first), ConstantU32(state, 1u)); } +namespace { + +bool TableKeyPerLane(const SpirvEmitterState& state, std::uint32_t resource) { + const auto& images = state.program.Info().images; + const auto& sources = state.program.Resources().descriptorSources; + if (resource >= images.size() || images[resource].source >= sources.size()) return false; + const auto& table = sources[images[resource].source].indirectImage; + return table.has_value() && table->laneKey; +} + +} + std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource) { const bool storage = ImageBindingResourceClass(kind) == ImageResourceClass::Storage; const auto supported = [&](std::uint32_t capability) { @@ -88,7 +100,7 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), metadata, state.shaderDataStorageVariable, ConstantU32(state, 0u), offset); const auto index = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, TypeU32(state), index, metadata); - if (!state.tableIndexNonUniform) { + if (!state.tableIndexNonUniform && !TableKeyPerLane(state, resource)) { return index; } const auto nonUniform = storage ? spv::CapabilityStorageImageArrayNonUniformIndexing : spv::CapabilitySampledImageArrayNonUniformIndexing; @@ -104,8 +116,8 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki return index; } -void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t slotId, std::uint32_t id) { - if (slotId != 0 && state.tableIndexNonUniform) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); +void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId, std::uint32_t id) { + if (slotId != 0 && (state.tableIndexNonUniform || TableKeyPerLane(state, resource))) state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); } std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass numericClass) { @@ -184,11 +196,11 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t } else { pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, TableImageIndex(state, kind, resource)); - DecorateTableNonUniform(state, slotId, pointer); + DecorateTableNonUniform(state, resource, slotId, pointer); } const auto image = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, image, pointer); - DecorateTableNonUniform(state, slotId, image); + DecorateTableNonUniform(state, resource, slotId, image); return image; } @@ -210,7 +222,7 @@ std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, const auto sampledImage = state.module.AllocateId(); const auto sampledType = state.module.Type(spv::OpTypeSampledImage, ImageType(state, imageResource)); state.module.AddFunction(spv::OpSampledImage, sampledType, sampledImage, image, samplerId); - DecorateTableNonUniform(state, slotId, sampledImage); + DecorateTableNonUniform(state, resource, slotId, sampledImage); return sampledImage; } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index f8f267fd2c8..13a4ff0dd79 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -43,6 +43,7 @@ - [8_SRGB and 8_8_SRGB loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) on a device that cannot sample R8_SRGB or R8G8_SRGB return the sRGB EOTF rounded to nearest even at bf16 precision, as adriadam10 measured on RDNA2 (gfx1035) in #866; the PS5's gfx1013 is not measured. - [Samples and gathers](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. +- A [waterfall loop](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) over T#s is compiled as one access per lane only when each lane loaded its T# with `buffer_load_dword*` (IDXEN, immediate VOFFSET and SOFFSET) from one V# that the SRT walker evaluates: the access becomes a [bindless image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) keyed by the lane's index, with the NonUniform decoration. The loop is recognised when every T# dword of an image read, sample, gather or query comes from `v_readlane_b32` or `v_readfirstlane_b32` and the EXEC its results are written under implies that each of them equals the lane's own VGPR. Waterfalls over other per-lane T#s, over S#s, around image stores or atomics, and over T# VGPRs that hold another descriptor in lanes the load skipped (in all eight dwords alike, directly or through the same masked writes and phis) still throw. A lane the load skipped whose T# VGPRs hold any other value reads zeros instead of using that value as a T#. Every lane runs the access with its own record, so a sample with implicit derivatives or an LOD query throws at dispatch when the bound records need different runtime image modes. Lanes whose record is null or outside the table take the zero path of the per-lane mode switch under record 0's mode, the first mode when record 0 is null, so around such a sample or query the derivatives of quad neighbours whose records need another mode are still undefined. A table V# with records must be structured (stride at least the record offset plus 32 bytes, `OOB_SELECT` 0 or 1) without swizzling or `ADD_TID`, else the dispatch throws. One with `num_records` 0, a null V# included, binds no record, and every lane that runs the access reads zeros: with no records every `OOB_SELECT` of the GFX10 table in Mesa [ac_descriptors.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/ac_descriptors.c) puts each load out of range. The table is bound whole, so a table V# with more than `BindlessTableSlots` (16) records throws at dispatch. None of this was measured on hardware. - [Shader MODE register access](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. - A [flat SRT read](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. - Repeated `s_load`s of one [flat SRT](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own.