diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp index 17c06cd64a4..02a24473836 100644 --- a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -34,6 +34,12 @@ struct Builder { return created; } + IrValue& truth() { + auto& created = program.CreateValue(IrOpcode::Void, IrType::Bool); + created.SetImmediateBool(true); + return created; + } + IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list arguments, std::uint64_t flags = 0) { auto& created = program.CreateValue(opcode, type, flags); for (auto* argument : arguments) created.AddArgument(argument); @@ -592,14 +598,24 @@ int main() { return b.eliminate() == 0u && !removed(written); }); - const auto sample = [](IrShaderStage stage, std::uint32_t sampleFlags, IrOpcode opcode = IrOpcode::ImageSampleRaw) { + const auto wholeQuads = [](Builder& b, IrValue& live) -> IrValue& { + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&live}); + auto& allLow = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); + auto& allHigh = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); + auto& low = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allLow, &b.constant(0u)})}); + auto& high = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allHigh, &b.constant(1u)})}); + auto& wqm = b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&low, &high})}); + return b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(1u)})); + }; + + const auto sample = [&](IrShaderStage stage, std::uint32_t sampleFlags, IrOpcode opcode = IrOpcode::ImageSampleRaw, bool wholeQuadExec = false) { Builder b; b.program.Resources().stage = stage; MemoryInfo memory; memory.kind = ResourceKind::Image; memory.imageSampleFlags = sampleFlags; b.program.Resources().memoryInfo.push_back(memory); - auto& exec = b.mask(16u); + auto& exec = wholeQuadExec ? wholeQuads(b, b.mask(16u)) : b.mask(16u); auto& old = b.lane(); auto& written = b.select(exec, b.add(old, 1u), old); auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&written, &old}); @@ -618,6 +634,9 @@ int main() { return !sample(IrShaderStage::Pixel, 0u); }); + passed &= run("an implicit-LOD pixel sample under whole quad mode reads no masked quad and drops the select", [&] { + return sample(IrShaderStage::Pixel, 0u, IrOpcode::ImageSampleRaw, true); + }); passed &= run("an explicit-LOD sample reads only its lane and drops the select", [&] { return sample(IrShaderStage::Pixel, RdnaImageSampleFlagLevelZero) && sample(IrShaderStage::Mesh, 0u); }); @@ -697,5 +716,54 @@ int main() { return !guarded(IrOpcode::SharedAtomicIAdd32, true, false); }); + passed &= run("a write under whole quad mode read only by the lanes it was widened from drops the select in every stage", [&] { + const auto widened = [&](IrShaderStage stage) { + Builder b; + b.program.Resources().stage = stage; + auto& live = b.mask(16u); + auto& quads = wholeQuads(b, live); + auto& old = b.lane(); + auto& written = b.select(quads, b.add(old, 1u), old); + auto& next = b.select(b.logicalAnd(live, b.mask(8u)), b.add(written, 2u), old); + b.keep(next); + return b.eliminate() != 0u && removed(written) && !removed(next); + }; + return widened(IrShaderStage::Pixel) && widened(IrShaderStage::Compute); + }); + + passed &= run("a write under whole quad mode read by a lane outside the widened mask keeps the select in compute", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Compute; + auto& live = b.mask(16u); + auto& quads = wholeQuads(b, live); + auto& old = b.lane(); + auto& written = b.select(quads, b.add(old, 1u), old); + auto& next = b.select(b.mask(32u), b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 0u && !removed(written); + }); + + passed &= run("a write under whole quad mode read only by the lanes it was widened from drops the select", [&] { + Builder b; + auto& live = b.mask(16u); + auto& quads = wholeQuads(b, live); + auto& old = b.lane(); + auto& written = b.select(quads, b.add(old, 1u), old); + auto& next = b.select(b.logicalAnd(live, b.mask(8u)), b.add(written, 2u), old); + b.keep(next); + return b.eliminate() != 0u && removed(written) && !removed(next); + }); + + passed &= run("a write under whole quad mode read by other lanes keeps the select", [&] { + Builder b; + auto& live = b.mask(16u); + auto& quads = wholeQuads(b, live); + auto& old = b.lane(); + auto& written = b.select(quads, b.add(old, 1u), old); + auto& next = b.select(b.mask(32u), b.add(written, 2u), old); + b.keep(next); + return b.eliminate() == 0u && !removed(written); + }); + return passed ? 0 : 1; } diff --git a/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp b/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp index e500ead2ae5..601dedf0566 100644 --- a/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp +++ b/core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp @@ -271,6 +271,64 @@ bool alwaysTrue(const IrValue* value, std::uint32_t waveSize, std::uint32_t dept return bit && isImmediate(bit->low, 0xffffffffu) && isImmediate(bit->high, 0xffffffffu); } +using BlockSet = std::unordered_set; + +BlockSet uniformEntryBlocks(const IrProgram& program) { + BlockSet result; + if (program.BlockOrder().empty()) return result; + const IrBlock* block = program.BlockOrder().front(); + while (block->Predecessors().empty() || (block->Predecessors().size() == 1u && result.contains(block->Predecessors().front()))) { + if (!result.insert(block).second || block->Successors().size() != 1u) break; + block = block->Successors().front(); + } + return result; +} + +bool sameCondition(const IrValue* condition, const IrValue* predicate, std::uint32_t depth) { + condition = condition->Resolve(); + predicate = predicate->Resolve(); + if (condition == predicate || (predicate->HasImmediate() && predicate->Type() == IrType::Bool && predicate->ImmediateBool())) return true; + if (depth > DepthLimit) return false; + return condition->Opcode() == IrOpcode::LogicalAnd && (sameCondition(condition->Argument(0), predicate, depth + 1u) || sameCondition(condition->Argument(1), predicate, depth + 1u)); +} + +bool withinBallotWord(const IrValue* condition, const IrValue* word, std::uint32_t half, const BlockSet& uniform, std::uint32_t depth) { + if (word->Opcode() != IrOpcode::CompositeExtractU32x4 || !isImmediate(word->Argument(1)->Resolve(), half)) return false; + const IrValue* ballot = word->Argument(0)->Resolve(); + return ballot->Opcode() == IrOpcode::Ballot && uniform.contains(ballot->Parent()) && sameCondition(condition, ballot->Argument(0), depth + 1u); +} + +bool withinWords(const IrValue* condition, const IrValue* low, const IrValue* high, std::uint32_t waveSize, const BlockSet& uniform, std::uint32_t depth) { + if (depth > DepthLimit) return false; + if (low->Opcode() == IrOpcode::CompositeExtractU32x4) { + return withinBallotWord(condition, low, 0u, uniform, depth) && (waveSize == 32u || withinBallotWord(condition, high, 1u, uniform, depth)); + } + if (low->Opcode() != IrOpcode::BitwiseAnd32) return false; + const IrValue* lowLeft = low->Argument(0)->Resolve(); + const IrValue* lowRight = low->Argument(1)->Resolve(); + if (waveSize == 32u) return withinWords(condition, lowLeft, high, waveSize, uniform, depth + 1u) && withinWords(condition, lowRight, high, waveSize, uniform, depth + 1u); + if (high->Opcode() != IrOpcode::BitwiseAnd32) return false; + const IrValue* highLeft = high->Argument(0)->Resolve(); + const IrValue* highRight = high->Argument(1)->Resolve(); + return (withinWords(condition, lowLeft, highLeft, waveSize, uniform, depth + 1u) && withinWords(condition, lowRight, highRight, waveSize, uniform, depth + 1u)) || + (withinWords(condition, lowLeft, highRight, waveSize, uniform, depth + 1u) && withinWords(condition, lowRight, highLeft, waveSize, uniform, depth + 1u)); +} + +const IrValue* wholeQuadSource(const IrValue* mask, std::uint32_t waveSize) { + const auto bit = threadBit(mask, waveSize); + if (!bit || bit->low->Opcode() != IrOpcode::CompositeExtractU64 || !isImmediate(bit->low->Argument(1)->Resolve(), 0u)) return nullptr; + const IrValue* wqm = bit->low->Argument(0)->Resolve(); + if (waveSize != 32u && (bit->high->Opcode() != IrOpcode::CompositeExtractU64 || bit->high->Argument(0)->Resolve() != wqm || !isImmediate(bit->high->Argument(1)->Resolve(), 1u))) return nullptr; + if (wqm->Opcode() != IrOpcode::WqmU64) return nullptr; + const IrValue* source = wqm->Argument(0)->Resolve(); + return source->Opcode() == IrOpcode::CompositeConstructU64 ? source : nullptr; +} + +bool withinQuadsOf(const IrValue* condition, const IrValue* mask, std::uint32_t waveSize, const BlockSet& uniform) { + const IrValue* source = wholeQuadSource(mask, waveSize); + return source != nullptr && withinWords(condition, source->Argument(0)->Resolve(), source->Argument(1)->Resolve(), waveSize, uniform, 0u); +} + enum class Region : std::uint8_t { Unreached, Once, Loop }; Region regionOf(const std::vector& regions, const IrBlock* block) { @@ -297,7 +355,7 @@ struct ImplicationKeyHash { class Implications { public: - Implications(const std::vector& regions, std::uint32_t waveSize, std::size_t& work) : regions(regions), waveSize(waveSize), work(work) {} + Implications(const std::vector& regions, const BlockSet& uniform, std::uint32_t waveSize, std::size_t& work) : regions(regions), uniform(uniform), waveSize(waveSize), work(work) {} void Reset(const IrValue* writeMask, bool acrossLoops) { mask = writeMask; @@ -318,7 +376,9 @@ class Implications { --work; const Attempt attempt = assume(key); bool result = false; - if (condition->Opcode() == IrOpcode::LogicalAnd) { + if (withinQuadsOf(condition, mask, waveSize, uniform)) { + result = true; + } else if (condition->Opcode() == IrOpcode::LogicalAnd) { result = Implies(condition->Argument(0), depth + 1u) || Implies(condition->Argument(1), depth + 1u); } else if (condition->IsPhi()) { result = merges(*condition); @@ -414,6 +474,7 @@ class Implications { } const std::vector& regions; + const BlockSet& uniform; std::uint32_t waveSize; std::size_t& work; const IrValue* mask = nullptr; @@ -474,12 +535,13 @@ std::size_t walkLimit(const IrProgram& program) { class MaskedLaneWalk { public: - MaskedLaneWalk(const IrProgram& program, const std::vector& regions) : program(program), regions(regions), limit(walkLimit(program)), implications(regions, program.WaveSize(), work) {} + MaskedLaneWalk(const IrProgram& program, const std::vector& regions) : program(program), regions(regions), uniform(uniformEntryBlocks(program)), limit(walkLimit(program)), implications(regions, uniform, program.WaveSize(), work) {} bool UnobservedWhereMasked(const IrValue& select, const IrValue* mask) { const bool acrossLoops = mask->HasImmediate() || regionOf(regions, mask->Parent()) == Region::Once; work = limit; implications.Reset(mask, acrossLoops); + const bool quadMasked = program.Resources().stage == IrShaderStage::Pixel && wholeQuadSource(mask, program.WaveSize()) != nullptr; startVisits(); pending.clear(); pending.push_back(&select); @@ -493,7 +555,7 @@ class MaskedLaneWalk { if (isSelect(user->Opcode()) && use.operand == 1u && implications.Implies(user->Argument(0))) continue; if (user->Opcode() == IrOpcode::LogicalAnd && use.operand < 2u && implications.Implies(user->Argument(1u - use.operand))) continue; if (readsOnlyWhereActive(user->Opcode()) && use.operand + 1u < user->ArgumentCount() && implications.Implies(user->Argument(user->ArgumentCount() - 1u))) continue; - if (!user->IsPhi() && !isLaneLocal(user->Opcode()) && !isExplicitLodSample(program, *user)) return false; + if (!user->IsPhi() && !isLaneLocal(user->Opcode()) && !isExplicitLodSample(program, *user) && !(quadMasked && user->Opcode() == IrOpcode::ImageSampleRaw)) return false; if (firstVisit(*user)) { if (work == 0) return false; --work; @@ -521,6 +583,7 @@ class MaskedLaneWalk { const IrProgram& program; const std::vector& regions; + BlockSet uniform; std::size_t limit; std::size_t work = 0; Implications implications;