Skip to content

fix(shader): drop masked selects proven unobserved through whole quad mode #3391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adriadam10 merged 1 commit into from
Oct 11, 2026
Merged

fix(shader): drop masked selects proven unobserved through whole quad mode #3391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 70 additions & 2 deletions core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ struct Builder {
return created;
}

IrValue& truth() {
auto& created = program.CreateValue(IrOpcode::Void, IrType::Bool);
created.SetImmediateBool(true);
return created;
}

IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list<IrValue*> arguments, std::uint64_t flags = 0) {
auto& created = program.CreateValue(opcode, type, flags);
for (auto* argument : arguments) created.AddArgument(argument);
Expand Down Expand Up @@ -592,14 +598,24 @@ int main() {
return b.eliminate() == 0u && !removed(written);
});

const auto sample = [](IrShaderStage stage, std::uint32_t sampleFlags, IrOpcode opcode = IrOpcode::ImageSampleRaw) {
const auto wholeQuads = [](Builder& b, IrValue& live) -> IrValue& {
auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&live});
auto& allLow = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
auto& allHigh = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
auto& low = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allLow, &b.constant(0u)})});
auto& high = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allHigh, &b.constant(1u)})});
auto& wqm = b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&low, &high})});
return b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(1u)}));
};

const auto sample = [&](IrShaderStage stage, std::uint32_t sampleFlags, IrOpcode opcode = IrOpcode::ImageSampleRaw, bool wholeQuadExec = false) {
Builder b;
b.program.Resources().stage = stage;
MemoryInfo memory;
memory.kind = ResourceKind::Image;
memory.imageSampleFlags = sampleFlags;
b.program.Resources().memoryInfo.push_back(memory);
auto& exec = b.mask(16u);
auto& exec = wholeQuadExec ? wholeQuads(b, b.mask(16u)) : b.mask(16u);
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&written, &old});
Expand All @@ -618,6 +634,9 @@ int main() {
return !sample(IrShaderStage::Pixel, 0u);
});

passed &= run("an implicit-LOD pixel sample under whole quad mode reads no masked quad and drops the select", [&] {
return sample(IrShaderStage::Pixel, 0u, IrOpcode::ImageSampleRaw, true);
});
passed &= run("an explicit-LOD sample reads only its lane and drops the select", [&] {
return sample(IrShaderStage::Pixel, RdnaImageSampleFlagLevelZero) && sample(IrShaderStage::Mesh, 0u);
});
Expand Down Expand Up @@ -697,5 +716,54 @@ int main() {
return !guarded(IrOpcode::SharedAtomicIAdd32, true, false);
});

passed &= run("a write under whole quad mode read only by the lanes it was widened from drops the select in every stage", [&] {
const auto widened = [&](IrShaderStage stage) {
Builder b;
b.program.Resources().stage = stage;
auto& live = b.mask(16u);
auto& quads = wholeQuads(b, live);
auto& old = b.lane();
auto& written = b.select(quads, b.add(old, 1u), old);
auto& next = b.select(b.logicalAnd(live, b.mask(8u)), b.add(written, 2u), old);
b.keep(next);
return b.eliminate() != 0u && removed(written) && !removed(next);
};
return widened(IrShaderStage::Pixel) && widened(IrShaderStage::Compute);
});

passed &= run("a write under whole quad mode read by a lane outside the widened mask keeps the select in compute", [&] {
Builder b;
b.program.Resources().stage = IrShaderStage::Compute;
auto& live = b.mask(16u);
auto& quads = wholeQuads(b, live);
auto& old = b.lane();
auto& written = b.select(quads, b.add(old, 1u), old);
auto& next = b.select(b.mask(32u), b.add(written, 2u), old);
b.keep(next);
return b.eliminate() == 0u && !removed(written);
});

passed &= run("a write under whole quad mode read only by the lanes it was widened from drops the select", [&] {
Builder b;
auto& live = b.mask(16u);
auto& quads = wholeQuads(b, live);
auto& old = b.lane();
auto& written = b.select(quads, b.add(old, 1u), old);
auto& next = b.select(b.logicalAnd(live, b.mask(8u)), b.add(written, 2u), old);
b.keep(next);
return b.eliminate() != 0u && removed(written) && !removed(next);
});

passed &= run("a write under whole quad mode read by other lanes keeps the select", [&] {
Builder b;
auto& live = b.mask(16u);
auto& quads = wholeQuads(b, live);
auto& old = b.lane();
auto& written = b.select(quads, b.add(old, 1u), old);
auto& next = b.select(b.mask(32u), b.add(written, 2u), old);
b.keep(next);
return b.eliminate() == 0u && !removed(written);
});

return passed ? 0 : 1;
}
71 changes: 67 additions & 4 deletions core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,64 @@ bool alwaysTrue(const IrValue* value, std::uint32_t waveSize, std::uint32_t dept
return bit && isImmediate(bit->low, 0xffffffffu) && isImmediate(bit->high, 0xffffffffu);
}

using BlockSet = std::unordered_set<const IrBlock*>;

BlockSet uniformEntryBlocks(const IrProgram& program) {
BlockSet result;
if (program.BlockOrder().empty()) return result;
const IrBlock* block = program.BlockOrder().front();
while (block->Predecessors().empty() || (block->Predecessors().size() == 1u && result.contains(block->Predecessors().front()))) {
if (!result.insert(block).second || block->Successors().size() != 1u) break;
block = block->Successors().front();
}
return result;
}

bool sameCondition(const IrValue* condition, const IrValue* predicate, std::uint32_t depth) {
condition = condition->Resolve();
predicate = predicate->Resolve();
if (condition == predicate || (predicate->HasImmediate() && predicate->Type() == IrType::Bool && predicate->ImmediateBool())) return true;
if (depth > DepthLimit) return false;
return condition->Opcode() == IrOpcode::LogicalAnd && (sameCondition(condition->Argument(0), predicate, depth + 1u) || sameCondition(condition->Argument(1), predicate, depth + 1u));
}

bool withinBallotWord(const IrValue* condition, const IrValue* word, std::uint32_t half, const BlockSet& uniform, std::uint32_t depth) {
if (word->Opcode() != IrOpcode::CompositeExtractU32x4 || !isImmediate(word->Argument(1)->Resolve(), half)) return false;
const IrValue* ballot = word->Argument(0)->Resolve();
return ballot->Opcode() == IrOpcode::Ballot && uniform.contains(ballot->Parent()) && sameCondition(condition, ballot->Argument(0), depth + 1u);
}

bool withinWords(const IrValue* condition, const IrValue* low, const IrValue* high, std::uint32_t waveSize, const BlockSet& uniform, std::uint32_t depth) {
if (depth > DepthLimit) return false;
if (low->Opcode() == IrOpcode::CompositeExtractU32x4) {
return withinBallotWord(condition, low, 0u, uniform, depth) && (waveSize == 32u || withinBallotWord(condition, high, 1u, uniform, depth));
}
if (low->Opcode() != IrOpcode::BitwiseAnd32) return false;
const IrValue* lowLeft = low->Argument(0)->Resolve();
const IrValue* lowRight = low->Argument(1)->Resolve();
if (waveSize == 32u) return withinWords(condition, lowLeft, high, waveSize, uniform, depth + 1u) && withinWords(condition, lowRight, high, waveSize, uniform, depth + 1u);
if (high->Opcode() != IrOpcode::BitwiseAnd32) return false;
const IrValue* highLeft = high->Argument(0)->Resolve();
const IrValue* highRight = high->Argument(1)->Resolve();
return (withinWords(condition, lowLeft, highLeft, waveSize, uniform, depth + 1u) && withinWords(condition, lowRight, highRight, waveSize, uniform, depth + 1u)) ||
(withinWords(condition, lowLeft, highRight, waveSize, uniform, depth + 1u) && withinWords(condition, lowRight, highLeft, waveSize, uniform, depth + 1u));
}

const IrValue* wholeQuadSource(const IrValue* mask, std::uint32_t waveSize) {
const auto bit = threadBit(mask, waveSize);
if (!bit || bit->low->Opcode() != IrOpcode::CompositeExtractU64 || !isImmediate(bit->low->Argument(1)->Resolve(), 0u)) return nullptr;
const IrValue* wqm = bit->low->Argument(0)->Resolve();
if (waveSize != 32u && (bit->high->Opcode() != IrOpcode::CompositeExtractU64 || bit->high->Argument(0)->Resolve() != wqm || !isImmediate(bit->high->Argument(1)->Resolve(), 1u))) return nullptr;
if (wqm->Opcode() != IrOpcode::WqmU64) return nullptr;
const IrValue* source = wqm->Argument(0)->Resolve();
return source->Opcode() == IrOpcode::CompositeConstructU64 ? source : nullptr;
}

bool withinQuadsOf(const IrValue* condition, const IrValue* mask, std::uint32_t waveSize, const BlockSet& uniform) {
const IrValue* source = wholeQuadSource(mask, waveSize);
return source != nullptr && withinWords(condition, source->Argument(0)->Resolve(), source->Argument(1)->Resolve(), waveSize, uniform, 0u);
}

enum class Region : std::uint8_t { Unreached, Once, Loop };

Region regionOf(const std::vector<Region>& regions, const IrBlock* block) {
Expand All @@ -297,7 +355,7 @@ struct ImplicationKeyHash {

class Implications {
public:
Implications(const std::vector<Region>& regions, std::uint32_t waveSize, std::size_t& work) : regions(regions), waveSize(waveSize), work(work) {}
Implications(const std::vector<Region>& regions, const BlockSet& uniform, std::uint32_t waveSize, std::size_t& work) : regions(regions), uniform(uniform), waveSize(waveSize), work(work) {}

void Reset(const IrValue* writeMask, bool acrossLoops) {
mask = writeMask;
Expand All @@ -318,7 +376,9 @@ class Implications {
--work;
const Attempt attempt = assume(key);
bool result = false;
if (condition->Opcode() == IrOpcode::LogicalAnd) {
if (withinQuadsOf(condition, mask, waveSize, uniform)) {
result = true;
} else if (condition->Opcode() == IrOpcode::LogicalAnd) {
result = Implies(condition->Argument(0), depth + 1u) || Implies(condition->Argument(1), depth + 1u);
} else if (condition->IsPhi()) {
result = merges(*condition);
Expand Down Expand Up @@ -414,6 +474,7 @@ class Implications {
}

const std::vector<Region>& regions;
const BlockSet& uniform;
std::uint32_t waveSize;
std::size_t& work;
const IrValue* mask = nullptr;
Expand Down Expand Up @@ -474,12 +535,13 @@ std::size_t walkLimit(const IrProgram& program) {

class MaskedLaneWalk {
public:
MaskedLaneWalk(const IrProgram& program, const std::vector<Region>& regions) : program(program), regions(regions), limit(walkLimit(program)), implications(regions, program.WaveSize(), work) {}
MaskedLaneWalk(const IrProgram& program, const std::vector<Region>& regions) : program(program), regions(regions), uniform(uniformEntryBlocks(program)), limit(walkLimit(program)), implications(regions, uniform, program.WaveSize(), work) {}

bool UnobservedWhereMasked(const IrValue& select, const IrValue* mask) {
const bool acrossLoops = mask->HasImmediate() || regionOf(regions, mask->Parent()) == Region::Once;
work = limit;
implications.Reset(mask, acrossLoops);
const bool quadMasked = program.Resources().stage == IrShaderStage::Pixel && wholeQuadSource(mask, program.WaveSize()) != nullptr;
startVisits();
pending.clear();
pending.push_back(&select);
Expand All @@ -493,7 +555,7 @@ class MaskedLaneWalk {
if (isSelect(user->Opcode()) && use.operand == 1u && implications.Implies(user->Argument(0))) continue;
if (user->Opcode() == IrOpcode::LogicalAnd && use.operand < 2u && implications.Implies(user->Argument(1u - use.operand))) continue;
if (readsOnlyWhereActive(user->Opcode()) && use.operand + 1u < user->ArgumentCount() && implications.Implies(user->Argument(user->ArgumentCount() - 1u))) continue;
if (!user->IsPhi() && !isLaneLocal(user->Opcode()) && !isExplicitLodSample(program, *user)) return false;
if (!user->IsPhi() && !isLaneLocal(user->Opcode()) && !isExplicitLodSample(program, *user) && !(quadMasked && user->Opcode() == IrOpcode::ImageSampleRaw)) return false;
if (firstVisit(*user)) {
if (work == 0) return false;
--work;
Expand Down Expand Up @@ -521,6 +583,7 @@ class MaskedLaneWalk {

const IrProgram& program;
const std::vector<Region>& regions;
BlockSet uniform;
std::size_t limit;
std::size_t work = 0;
Implications implications;
Expand Down
Loading