diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 10da8fa1284..b373847f056 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -279,6 +279,11 @@ target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver l set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_dpp_write_tests) +add_test_executable(agc_masked_select_tests EXCLUDE_FROM_ALL tests/MaskedSelect.cpp) +target_link_libraries(agc_masked_select_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_masked_select_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_masked_select_tests) + add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp) target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -498,7 +503,7 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources) endforeach() if(BUILD_TESTING) - foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_dpp_write_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests}) string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}") set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF) add_test(NAME ${agcTestName} COMMAND ${agcTest}) diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp new file mode 100644 index 00000000000..119f8792511 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -0,0 +1,378 @@ +#include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/MaskedSelectEliminator.hpp" +#include "RdnaDecoder/RdnaInstruction.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +struct Builder { + IrProgram program; + IrBlock* block = nullptr; + + Builder() { + block = &newBlock(); + program.SetEntryBlock(*block); + } + + IrBlock& newBlock() { + auto& created = program.CreateBlock(); + program.BlockOrder().push_back(&created); + return created; + } + + IrValue& constant(std::uint32_t value) { + auto& created = program.CreateValue(IrOpcode::Void, IrType::U32); + created.SetImmediateU32(value); + return created; + } + + IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list arguments, std::uint64_t flags = 0) { + auto& created = program.CreateValue(opcode, type, flags); + for (auto* argument : arguments) created.AddArgument(argument); + block->AppendInstruction(&created); + return created; + } + + IrValue& lane() { + return emit(IrOpcode::LaneId, IrType::U32, {}); + } + + IrValue& mask(std::uint32_t lanes) { + return emit(IrOpcode::ULessThan32, IrType::Bool, {&lane(), &constant(lanes)}); + } + + IrValue& select(IrValue& condition, IrValue& active, IrValue& inactive) { + return emit(IrOpcode::SelectU32, IrType::U32, {&condition, &active, &inactive}); + } + + IrValue& add(IrValue& value, std::uint32_t immediate) { + return emit(IrOpcode::IAdd32, IrType::U32, {&value, &constant(immediate)}); + } + + IrValue& logicalAnd(IrValue& left, IrValue& right) { + return emit(IrOpcode::LogicalAnd, IrType::Bool, {&left, &right}); + } + + IrValue& threadBit(IrValue& low, IrValue& high) { + auto& laneId = lane(); + auto& word = select(emit(IrOpcode::ULessThan32, IrType::Bool, {&laneId, &constant(32u)}), low, high); + auto& shift = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&laneId, &constant(31u)}); + auto& shifted = emit(IrOpcode::ShiftRightLogical32, IrType::U32, {&word, &shift}); + auto& bit = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&shifted, &constant(1u)}); + return emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &constant(0u)}); + } + + IrValue& ballotBit(IrValue& predicate) { + auto& ballot = emit(IrOpcode::Ballot, IrType::U32x4, {&predicate}); + auto& low = emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(0u)}); + auto& high = emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(1u)}); + return threadBit(low, high); + } + + void keep(IrValue& value) { + (void)emit(IrOpcode::ReferenceU32, IrType::Void, {&value}); + } + + std::uint32_t eliminate() { + return MaskedSelectEliminator{}.Eliminate(program).removedSelects; + } +}; + +bool removed(const IrValue& value) { + return value.Parent() == nullptr; +} + +bool expect(const char* name, bool condition) { + if (!condition) std::fprintf(stderr, "%s\n", name); + return condition; +} + +bool run(const char* name, const std::function& test) { + try { + return expect(name, test()); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s: %s\n", name, error.what()); + return false; + } +} + +} + +int main() { + bool passed = true; + + passed &= run("a write read only under its own exec must lose its select", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& next = b.select(exec, b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 1u && removed(written) && !removed(next); + }); + + passed &= run("a write read under a wider exec must keep its select", [] { + Builder b; + auto& wide = b.mask(32u); + auto& exec = b.logicalAnd(wide, b.mask(16u)); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& next = b.select(wide, b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a write read under a narrower exec must lose its select", [] { + Builder b; + auto& exec = b.mask(32u); + auto& narrow = b.logicalAnd(exec, b.mask(16u)); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& next = b.select(narrow, b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 1u && removed(written); + }); + + passed &= run("a write whose old lanes flow into the next write must keep its select", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& next = b.select(exec, b.add(old, 2u), written); + b.keep(next); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a chain of writes under one exec must collapse once the last one goes", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& first = b.select(exec, b.add(old, 1u), old); + auto& second = b.select(exec, b.add(first, 2u), first); + auto& sink = b.select(exec, b.add(second, 3u), old); + b.keep(sink); + return b.eliminate() == 2u && removed(first) && removed(second) && !removed(sink); + }); + + passed &= run("cross-lane reads must keep the select", [] { + bool ok = true; + for (const IrOpcode opcode : {IrOpcode::ReadFirstLane, IrOpcode::DppMoveU32, IrOpcode::BpermuteU32, IrOpcode::Permlane16U32, IrOpcode::SwizzleU32}) { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& read = b.emit(opcode, IrType::U32, {&written, &exec}); + b.keep(b.select(exec, read, old)); + ok &= b.eliminate() == 0u && !removed(written); + } + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)})}); + b.keep(b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)})); + return ok && b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a branch condition derived from the write keeps the select", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& taken = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)}); + (void)b.emit(IrOpcode::Reference, IrType::Void, {&taken}); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a compare masked by the same exec must not keep the select", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& compare = b.logicalAnd(exec, b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)})); + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&compare}); + b.keep(b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)})); + return b.eliminate() == 1u && removed(written); + }); + + passed &= run("an exec rebuilt from a ballot of a narrower mask implies the write's exec", [] { + Builder b; + auto& exec = b.mask(48u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& later = b.ballotBit(b.logicalAnd(exec, b.mask(40u))); + auto& next = b.select(later, b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 1u && removed(written); + }); + + passed &= run("an exec rebuilt from a ballot of a wider mask keeps the select", [] { + Builder b; + auto& wide = b.mask(48u); + auto& exec = b.logicalAnd(wide, b.mask(40u)); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& later = b.ballotBit(wide); + b.keep(b.select(later, b.add(written, 2u), old)); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a select under the full wave's exec must go whatever reads it", [] { + Builder b; + auto& full = b.threadBit(b.constant(0xffffffffu), b.constant(0xffffffffu)); + auto& old = b.lane(); + auto& written = b.select(full, b.add(old, 1u), old); + b.keep(written); + return b.eliminate() != 0u && removed(written); + }); + + passed &= run("a partial wave's exec is not the full wave", [] { + Builder b; + auto& partial = b.threadBit(b.constant(0xffffffffu), b.constant(0x0000ffffu)); + auto& old = b.lane(); + auto& written = b.select(partial, b.add(old, 1u), old); + b.keep(written); + return b.eliminate() == 0u && !removed(written); + }); + + const auto diamond = [](bool wideRead) { + Builder b; + auto& wide = b.mask(48u); + auto& exec = b.logicalAnd(wide, b.mask(16u)); + auto& old = b.lane(); + IrBlock& head = *b.block; + IrBlock& body = b.newBlock(); + IrBlock& merge = b.newBlock(); + head.AddBranch(&body); + head.AddBranch(&merge); + body.AddBranch(&merge); + b.block = &body; + auto& written = b.select(exec, b.add(old, 1u), old); + b.block = &merge; + auto& phi = b.program.CreateValue(IrOpcode::Phi, IrType::U32); + merge.AppendInstruction(&phi); + phi.AddPhiOperand(&head, &old); + phi.AddPhiOperand(&body, &written); + b.keep(b.select(wideRead ? wide : exec, b.add(phi, 2u), old)); + return std::pair{b.eliminate(), removed(written)}; + }; + + passed &= run("a write merged by a phi and read under its own exec must lose its select", [&] { + const auto [count, gone] = diamond(false); + return count == 1u && gone; + }); + + passed &= run("a write merged by a phi and read under the wider exec must keep its select", [&] { + const auto [count, gone] = diamond(true); + return count == 0u && !gone; + }); + + passed &= run("a write in a loop must keep its select", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + IrBlock& head = *b.block; + IrBlock& loop = b.newBlock(); + IrBlock& exit = b.newBlock(); + head.AddBranch(&loop); + loop.AddBranch(&loop); + loop.AddBranch(&exit); + b.block = &loop; + auto& written = b.select(exec, b.add(old, 1u), old); + auto& next = b.select(exec, b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a write read inside a later loop must keep its select", [] { + Builder b; + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + IrBlock& head = *b.block; + IrBlock& loop = b.newBlock(); + head.AddBranch(&loop); + loop.AddBranch(&loop); + b.block = &loop; + b.keep(b.select(exec, b.add(written, 2u), old)); + return b.eliminate() == 0u && !removed(written); + }); + + const auto sample = [](IrShaderStage stage, std::uint32_t sampleFlags) { + Builder b; + b.program.Resources().stage = stage; + MemoryInfo memory; + memory.kind = ResourceKind::Image; + memory.imageSampleFlags = sampleFlags; + b.program.Resources().memoryInfo.push_back(memory); + auto& exec = b.mask(16u); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&written, &old}); + auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {}); + auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {}); + MemoryFlags flags{0u, 0u}; + std::uint64_t bits = 0; + std::memcpy(&bits, &flags, sizeof(flags)); + auto& sampled = b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address}, bits); + b.keep(b.select(exec, b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&sampled, &b.constant(0u)}), old)); + b.eliminate(); + return removed(written); + }; + + passed &= run("an implicit-LOD pixel sample reads its quad's coordinates and keeps the select", [&] { + return !sample(IrShaderStage::Pixel, 0u); + }); + + passed &= run("an explicit-LOD sample reads only its lane and drops the select", [&] { + return sample(IrShaderStage::Pixel, RdnaImageSampleFlagLevelZero) && sample(IrShaderStage::Mesh, 0u); + }); + + const auto guarded = [](IrOpcode opcode, bool sameExec, bool asActive) { + Builder b; + auto& wide = b.mask(48u); + auto& exec = b.logicalAnd(wide, b.mask(16u)); + auto& old = b.lane(); + auto& written = b.select(exec, b.add(old, 1u), old); + IrValue& active = asActive ? b.emit(IrOpcode::INotEqual32, IrType::Bool, {&written, &b.constant(0u)}) : sameExec ? exec : wide; + switch (opcode) { + case IrOpcode::WriteSharedU32: + b.emit(opcode, IrType::Void, {&old, &written, &active}); + break; + case IrOpcode::LoadBufferU32: + b.keep(b.select(exec, b.emit(opcode, IrType::U32, {&b.emit(IrOpcode::GetBufferResource, IrType::BufferResource, {}), &written, &b.constant(0u), &b.constant(0u), &active}), old)); + break; + default: + b.emit(opcode, IrType::Void, {&b.emit(IrOpcode::CompositeConstructU32x4, IrType::U32x4, {&written, &old, &old, &old}), &active}); + break; + } + b.eliminate(); + return removed(written); + }; + + passed &= run("a store, load or export whose exec implies the write's exec drops the select", [&] { + return guarded(IrOpcode::WriteSharedU32, true, false) && guarded(IrOpcode::LoadBufferU32, true, false) && guarded(IrOpcode::SetAttribute, true, false); + }); + + passed &= run("a store, load or export under a wider exec keeps the select", [&] { + return !guarded(IrOpcode::WriteSharedU32, false, false) && !guarded(IrOpcode::LoadBufferU32, false, false) && !guarded(IrOpcode::SetAttribute, false, false); + }); + + passed &= run("a value used as a store's exec keeps the select", [&] { + return !guarded(IrOpcode::WriteSharedU32, true, true); + }); + + passed &= run("an atomic keeps the select", [&] { + return !guarded(IrOpcode::SharedAtomicIAdd32, true, false); + }); + + return passed ? 0 : 1; +} diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 29d4bf17680..50ff3356de6 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -96,6 +96,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ConstantFolder.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/DeadCodeEliminator.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ReadLaneEliminator.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/MaskedSelectEliminator.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ResourceTracker.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ResourceMaterializer.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/BindingAllocator.cpp diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index 4e2574a14e4..59f3c075d05 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -7,6 +7,7 @@ #include "Optimization/BindingAllocator.hpp" #include "Optimization/ConstantFolder.hpp" #include "Optimization/DeadCodeEliminator.hpp" +#include "Optimization/MaskedSelectEliminator.hpp" #include "Optimization/ReadLaneEliminator.hpp" #include "Optimization/ResourceTracker.hpp" #include "Optimization/ShaderInfoCollector.hpp" @@ -208,6 +209,13 @@ int main(int argc, char** argv) { deadCodeEliminator.Eliminate(program); } + MaskedSelectEliminator maskedSelectEliminator; + const auto maskedSelectStats = maskedSelectEliminator.Eliminate(program); + std::cout << "MaskedSelectEliminator removed " << maskedSelectStats.removedSelects << "\n"; + if (maskedSelectStats.removedSelects != 0u) { + deadCodeEliminator.Eliminate(program); + } + SrtWalker srtWalker; srtWalker.BuildPlan(program); std::cout << "SrtWalker::BuildPlan OK\n"; diff --git a/core/shader/recompiler/Optimization/include/Optimization/MaskedSelectEliminator.hpp b/core/shader/recompiler/Optimization/include/Optimization/MaskedSelectEliminator.hpp new file mode 100644 index 00000000000..fad7eed9c1b --- /dev/null +++ b/core/shader/recompiler/Optimization/include/Optimization/MaskedSelectEliminator.hpp @@ -0,0 +1,20 @@ +#ifndef CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_MASKEDSELECTELIMINATOR_HPP +#define CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_MASKEDSELECTELIMINATOR_HPP + +#include "IntermediateRepresentation/IrProgram.hpp" +#include + +namespace ShaderRecompiler { + +struct MaskedSelectEliminationStats { + std::uint32_t removedSelects = 0; +}; + +class MaskedSelectEliminator { +public: + [[nodiscard]] MaskedSelectEliminationStats Eliminate(IrProgram& program) const; +}; + +} + +#endif diff --git a/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp b/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp new file mode 100644 index 00000000000..74f4cda6f71 --- /dev/null +++ b/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp @@ -0,0 +1,394 @@ +#include "Optimization/MaskedSelectEliminator.hpp" +#include "Optimization/DeadCodeEliminator.hpp" +#include "RdnaDecoder/RdnaInstruction.hpp" +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +constexpr std::size_t VisitLimit = 4096; +constexpr std::uint32_t DepthLimit = 8; + +bool isSelect(IrOpcode opcode) { + return opcode == IrOpcode::SelectU1 || opcode == IrOpcode::SelectU32 || opcode == IrOpcode::SelectF32; +} + +bool isLaneLocal(IrOpcode opcode) { + if (IrOpcodeHasSideEffects(opcode)) return false; + switch (opcode) { + case IrOpcode::Identity: + case IrOpcode::IAdd32: + case IrOpcode::ISub32: + case IrOpcode::IMul32: + case IrOpcode::SelectU1: + case IrOpcode::SelectU32: + case IrOpcode::SelectF32: + case IrOpcode::BitCastU16F16: + case IrOpcode::BitCastF16U16: + case IrOpcode::BitCastU32F32: + case IrOpcode::BitCastF32U32: + case IrOpcode::ConvertU16U32: + case IrOpcode::ConvertU32U16: + case IrOpcode::ConvertU8U32: + case IrOpcode::ConvertU32U8: + case IrOpcode::ConvertF32F16: + case IrOpcode::ConvertF16F32: + case IrOpcode::ConvertS32F32: + case IrOpcode::ConvertU32F32: + case IrOpcode::ConvertF32S32: + case IrOpcode::ConvertF32U32: + case IrOpcode::ConvertF32F64: + case IrOpcode::ConvertF64F32: + case IrOpcode::ConvertF64S32: + case IrOpcode::ConvertF64U32: + case IrOpcode::ConvertS32F64: + case IrOpcode::ConvertU32F64: + case IrOpcode::CompositeConstructU64: + case IrOpcode::CompositeConstructU32x2: + case IrOpcode::CompositeConstructU32x3: + case IrOpcode::CompositeConstructF32x2: + case IrOpcode::CompositeConstructU32x4: + case IrOpcode::CompositeExtractU64: + case IrOpcode::CompositeExtractU32x2: + case IrOpcode::CompositeExtractU32x3: + case IrOpcode::CompositeExtractU32x4: + case IrOpcode::PackHalf2x16: + case IrOpcode::PackFloat2x16Rtz: + case IrOpcode::BitFieldInsert: + case IrOpcode::BitFieldUExtract: + case IrOpcode::BitFieldSExtract: + case IrOpcode::IAdd64: + case IrOpcode::IAddCarry32: + case IrOpcode::ISub64: + case IrOpcode::IMul64: + case IrOpcode::SMulHi: + case IrOpcode::UMulHi: + case IrOpcode::IAbs32: + case IrOpcode::ShiftLeftLogical32: + case IrOpcode::ShiftLeftLogical64: + case IrOpcode::ShiftRightLogical32: + case IrOpcode::ShiftRightLogical64: + case IrOpcode::ShiftRightArithmetic32: + case IrOpcode::ShiftRightArithmetic64: + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseAnd64: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + case IrOpcode::BitwiseNot32: + case IrOpcode::BitReverse32: + case IrOpcode::BitCount32: + case IrOpcode::BitCount64: + case IrOpcode::FindUMsb32: + case IrOpcode::FindUMsb64: + case IrOpcode::FindILsb32: + case IrOpcode::SMin32: + case IrOpcode::UMin32: + case IrOpcode::SMax32: + case IrOpcode::UMax32: + case IrOpcode::SMinTri32: + case IrOpcode::UMinTri32: + case IrOpcode::SMaxTri32: + case IrOpcode::UMaxTri32: + case IrOpcode::SMedTri32: + case IrOpcode::UMedTri32: + case IrOpcode::SLessThan32: + case IrOpcode::SLessThan64: + case IrOpcode::ULessThan32: + case IrOpcode::ULessThan64: + case IrOpcode::IEqual32: + case IrOpcode::IEqual64: + case IrOpcode::SLessThanEqual32: + case IrOpcode::ULessThanEqual32: + case IrOpcode::SGreaterThan32: + case IrOpcode::UGreaterThan32: + case IrOpcode::UGreaterThan64: + case IrOpcode::INotEqual32: + case IrOpcode::INotEqual64: + case IrOpcode::SGreaterThanEqual32: + case IrOpcode::UGreaterThanEqual32: + case IrOpcode::LogicalOr: + case IrOpcode::LogicalAnd: + case IrOpcode::LogicalXor: + case IrOpcode::LogicalNot: + case IrOpcode::FPOrdEqual32: + case IrOpcode::FPUnordEqual32: + case IrOpcode::FPOrdNotEqual32: + case IrOpcode::FPUnordNotEqual32: + case IrOpcode::FPOrdLessThan32: + case IrOpcode::FPUnordLessThan32: + case IrOpcode::FPOrdGreaterThan32: + case IrOpcode::FPUnordGreaterThan32: + case IrOpcode::FPOrdLessThanEqual32: + case IrOpcode::FPUnordLessThanEqual32: + case IrOpcode::FPOrdGreaterThanEqual32: + case IrOpcode::FPUnordGreaterThanEqual32: + case IrOpcode::FPIsNan32: + case IrOpcode::FPCmpClass32: + case IrOpcode::FPAbs32: + case IrOpcode::FPNeg32: + case IrOpcode::FPSaturate32: + case IrOpcode::FPAdd32: + case IrOpcode::FPSub32: + case IrOpcode::FPFma32: + case IrOpcode::FPMad32: + case IrOpcode::FPMul32: + case IrOpcode::FPMin32: + case IrOpcode::FPMax32: + case IrOpcode::FPMinTri32: + case IrOpcode::FPMaxTri32: + case IrOpcode::FPMedTri32: + case IrOpcode::FPRecip32: + case IrOpcode::FPRecipIFlag32: + case IrOpcode::FPRecipSqrt32: + case IrOpcode::FPSqrt: + case IrOpcode::FPSin: + case IrOpcode::FPCos: + case IrOpcode::FPExp2: + case IrOpcode::FPLog2: + case IrOpcode::FPRoundEven32: + case IrOpcode::FPFloor32: + case IrOpcode::FPCeil32: + case IrOpcode::FPTrunc32: + case IrOpcode::FPFract32: + case IrOpcode::FPAdd64: + case IrOpcode::FPMul64: + case IrOpcode::FPFma64: + case IrOpcode::FPMin64: + case IrOpcode::FPMax64: + case IrOpcode::FPSaturate64: + case IrOpcode::FPLdexp64: + case IrOpcode::FPRoundEven64: + case IrOpcode::FPFloor64: + case IrOpcode::FPCeil64: + case IrOpcode::FPTrunc64: + case IrOpcode::FPFract64: + case IrOpcode::FPFrexpMant64: + case IrOpcode::FPFrexpExp64: + case IrOpcode::MakeImageAddress: + return true; + default: + return false; + } +} + +bool readsOnlyWhereActive(IrOpcode opcode) { + switch (opcode) { + case IrOpcode::LoadBufferU8: + case IrOpcode::LoadBufferU16: + case IrOpcode::LoadBufferU32: + case IrOpcode::LoadBufferU32x2: + case IrOpcode::LoadBufferU32x3: + case IrOpcode::LoadBufferU32x4: + case IrOpcode::StoreBufferU8: + case IrOpcode::StoreBufferU16: + case IrOpcode::StoreBufferU32: + case IrOpcode::StoreBufferU32x2: + case IrOpcode::StoreBufferU32x3: + case IrOpcode::StoreBufferU32x4: + case IrOpcode::LoadSharedU8: + case IrOpcode::LoadSharedU16: + case IrOpcode::LoadSharedU32: + case IrOpcode::LoadSharedU32x2: + case IrOpcode::LoadSharedU32x3: + case IrOpcode::LoadSharedU32x4: + case IrOpcode::WriteSharedU8: + case IrOpcode::WriteSharedU16: + case IrOpcode::WriteSharedU32: + case IrOpcode::WriteSharedU32x2: + case IrOpcode::WriteSharedU32x3: + case IrOpcode::WriteSharedU32x4: + case IrOpcode::SetAttribute: + return true; + default: + return false; + } +} + +bool isExplicitLodSample(const IrProgram& program, const IrValue& value) { + if (value.Opcode() != IrOpcode::ImageSampleRaw) return false; + if (program.Resources().stage != IrShaderStage::Pixel) return true; + const auto index = value.Flags().index; + if (index >= program.Resources().memoryInfo.size()) return false; + return (program.Resources().memoryInfo[index].imageSampleFlags & (RdnaImageSampleFlagDerivative | RdnaImageSampleFlagLod | RdnaImageSampleFlagLevelZero)) != 0u; +} + +bool isImmediate(const IrValue* value, std::uint32_t immediate) { + return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == immediate; +} + +const IrValue* otherArgument(const IrValue* value, std::uint32_t immediate) { + if (value->ArgumentCount() != 2u) return nullptr; + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + if (isImmediate(right, immediate)) return left; + if (isImmediate(left, immediate)) return right; + return nullptr; +} + +struct ThreadBit { + const IrValue* low = nullptr; + const IrValue* high = nullptr; +}; + +bool isLane(const IrValue* value) { + return value->Resolve()->Opcode() == IrOpcode::LaneId; +} + +std::optional threadBit(const IrValue* value, std::uint32_t waveSize) { + value = value->Resolve(); + if (value->Opcode() != IrOpcode::INotEqual32) return std::nullopt; + const IrValue* bit = otherArgument(value, 0u); + if (bit == nullptr || bit->Opcode() != IrOpcode::BitwiseAnd32) return std::nullopt; + const IrValue* shifted = otherArgument(bit, 1u); + if (shifted == nullptr || shifted->Opcode() != IrOpcode::ShiftRightLogical32) return std::nullopt; + const IrValue* amount = shifted->Argument(1)->Resolve(); + if (amount->Opcode() != IrOpcode::BitwiseAnd32) return std::nullopt; + const IrValue* lane = otherArgument(amount, 31u); + if (lane == nullptr || !isLane(lane)) return std::nullopt; + const IrValue* word = shifted->Argument(0)->Resolve(); + if (waveSize == 32u || word->HasImmediate()) return ThreadBit{word, word}; + if (word->Opcode() != IrOpcode::SelectU32) return std::nullopt; + const IrValue* half = word->Argument(0)->Resolve(); + if (half->Opcode() != IrOpcode::ULessThan32 || !isLane(half->Argument(0)) || !isImmediate(half->Argument(1)->Resolve(), 32u)) return std::nullopt; + return ThreadBit{word->Argument(1)->Resolve(), word->Argument(2)->Resolve()}; +} + +bool alwaysTrue(const IrValue* value, std::uint32_t waveSize, std::uint32_t depth = 0) { + value = value->Resolve(); + if (value->HasImmediate()) return value->Type() == IrType::Bool && value->ImmediateBool(); + if (depth > DepthLimit) return false; + if (value->Opcode() == IrOpcode::LogicalAnd) return alwaysTrue(value->Argument(0), waveSize, depth + 1u) && alwaysTrue(value->Argument(1), waveSize, depth + 1u); + const auto bit = threadBit(value, waveSize); + return bit && isImmediate(bit->low, 0xffffffffu) && isImmediate(bit->high, 0xffffffffu); +} + +bool implies(const IrValue* condition, const IrValue* mask, std::uint32_t waveSize, std::uint32_t depth = 0); + +bool wordsImply(const IrValue* low, const IrValue* high, const IrValue* mask, std::uint32_t waveSize, std::uint32_t depth) { + if (depth > DepthLimit) return false; + if (isImmediate(low, 0u) && (waveSize == 32u || isImmediate(high, 0u))) return true; + if (low->Opcode() == IrOpcode::CompositeExtractU32x4 && isImmediate(low->Argument(1)->Resolve(), 0u)) { + const IrValue* ballot = low->Argument(0)->Resolve(); + const bool sameBallot = waveSize == 32u || (high->Opcode() == IrOpcode::CompositeExtractU32x4 && high->Argument(0)->Resolve() == ballot && isImmediate(high->Argument(1)->Resolve(), 1u)); + if (ballot->Opcode() == IrOpcode::Ballot && sameBallot && implies(ballot->Argument(0), mask, waveSize, depth + 1u)) return true; + } + if (low->Opcode() != IrOpcode::BitwiseAnd32) return false; + if (waveSize == 32u) return wordsImply(low->Argument(0)->Resolve(), high, mask, waveSize, depth + 1u) || wordsImply(low->Argument(1)->Resolve(), high, mask, waveSize, depth + 1u); + if (high->Opcode() != IrOpcode::BitwiseAnd32) return false; + for (std::size_t lowIndex = 0; lowIndex < 2u; ++lowIndex) { + for (std::size_t highIndex = 0; highIndex < 2u; ++highIndex) { + if (wordsImply(low->Argument(lowIndex)->Resolve(), high->Argument(highIndex)->Resolve(), mask, waveSize, depth + 1u)) return true; + } + } + return false; +} + +bool implies(const IrValue* condition, const IrValue* mask, std::uint32_t waveSize, std::uint32_t depth) { + condition = condition->Resolve(); + if (condition == mask || alwaysTrue(mask, waveSize)) return true; + if (condition->HasImmediate()) return condition->Type() == IrType::Bool && !condition->ImmediateBool(); + if (depth > DepthLimit) return false; + if (condition->Opcode() == IrOpcode::LogicalAnd) return implies(condition->Argument(0), mask, waveSize, depth + 1u) || implies(condition->Argument(1), mask, waveSize, depth + 1u); + const auto bit = threadBit(condition, waveSize); + return bit && wordsImply(bit->low, bit->high, mask, waveSize, depth + 1u); +} + +std::unordered_set blocksOutsideLoops(const IrProgram& program) { + const auto& order = program.BlockOrder(); + if (order.empty()) return {}; + std::unordered_map state; + std::unordered_set looping; + std::vector> stack; + state[order.front()] = 1; + stack.emplace_back(order.front(), 0u); + while (!stack.empty()) { + auto& [block, next] = stack.back(); + if (next == block->Successors().size()) { + state[block] = 2; + stack.pop_back(); + continue; + } + const IrBlock* source = block; + const IrBlock* successor = block->Successors()[next++]; + int& successorState = state[successor]; + if (successorState == 0) { + successorState = 1; + stack.emplace_back(successor, 0u); + } else if (successorState == 1) { + std::unordered_set body{successor}; + std::vector pending{source}; + while (!pending.empty()) { + const IrBlock* member = pending.back(); + pending.pop_back(); + if (!body.insert(member).second) continue; + for (const IrBlock* predecessor : member->Predecessors()) pending.push_back(predecessor); + } + looping.insert(body.begin(), body.end()); + } + } + std::unordered_set result; + for (const auto& [block, visited] : state) { + if (visited == 2 && !looping.contains(block)) result.insert(block); + } + return result; +} + +bool unobservedWhereMasked(const IrProgram& program, const std::unordered_set& once, const IrValue& select, const IrValue* mask) { + const auto waveSize = program.WaveSize(); + std::vector pending{&select}; + std::unordered_set visited{&select}; + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + for (const IrUse& use : value->OperandUses()) { + const IrValue* user = use.user; + if (user->Parent() == nullptr || !once.contains(user->Parent())) return false; + if (isSelect(user->Opcode()) && use.operand == 1u && implies(user->Argument(0), mask, waveSize)) continue; + if (user->Opcode() == IrOpcode::LogicalAnd && use.operand < 2u && implies(user->Argument(1u - use.operand), mask, waveSize)) continue; + if (readsOnlyWhereActive(user->Opcode()) && use.operand + 1u < user->ArgumentCount() && implies(user->Argument(user->ArgumentCount() - 1u), mask, waveSize)) continue; + if (!user->IsPhi() && !isLaneLocal(user->Opcode()) && !isExplicitLodSample(program, *user)) return false; + if (visited.insert(user).second) { + if (visited.size() > VisitLimit) return false; + pending.push_back(user); + } + } + } + return true; +} + +} + +MaskedSelectEliminationStats MaskedSelectEliminator::Eliminate(IrProgram& program) const { + MaskedSelectEliminationStats stats; + const auto once = blocksOutsideLoops(program); + bool changed = true; + while (changed) { + changed = false; + for (auto blockIt = program.BlockOrder().rbegin(); blockIt != program.BlockOrder().rend(); ++blockIt) { + const bool single = once.contains(*blockIt); + auto& instructions = (*blockIt)->Instructions(); + auto it = instructions.end(); + while (it != instructions.begin()) { + --it; + IrValue* inst = *it; + if (!isSelect(inst->Opcode()) || inst->ArgumentCount() != 3u || !inst->HasUses()) continue; + const IrValue* mask = inst->Argument(0)->Resolve(); + if (!alwaysTrue(mask, program.WaveSize()) && (!single || !unobservedWhereMasked(program, once, *inst, mask))) continue; + inst->ReplaceAllUsesWith(inst->Argument(1)); + inst->Invalidate(); + inst->SetParent(nullptr); + it = instructions.erase(it); + ++stats.removedSelects; + changed = true; + } + } + if (changed) DeadCodeEliminator{}.Eliminate(program); + } + return stats; +} + +} diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 89dd3909a15..817f4fc7ffb 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -19,6 +19,7 @@ #include "Optimization/include/Optimization/ConstantFolder.hpp" #include "Optimization/include/Optimization/DeadCodeEliminator.hpp" #include "Optimization/include/Optimization/DescriptorBindingBuilder.hpp" +#include "Optimization/include/Optimization/MaskedSelectEliminator.hpp" #include "Optimization/include/Optimization/ReadLaneEliminator.hpp" #include "Optimization/include/Optimization/RequestMemoryView.hpp" #include "Optimization/include/Optimization/ResourceMaterializer.hpp" @@ -155,6 +156,11 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { deadCodeEliminator.Eliminate(program); } + constexpr MaskedSelectEliminator maskedSelectEliminator; + if (maskedSelectEliminator.Eliminate(program).removedSelects != 0u) { + deadCodeEliminator.Eliminate(program); + } + constexpr SrtWalker srtWalker; srtWalker.BuildPlan(program); deadCodeEliminator.Eliminate(program);