perf(shader): take the f32 mul, fma and add result fixups only when needed

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp

@@ -524,6 +524,10 @@ struct ExportFlags {
     std::uint32_t pc = 0;
 };
 
+struct SelectFlags {
+    bool rarelyTrue = false;
+};
+
 [[nodiscard]] std::size_t IrOpcodeOperandCount(IrOpcode opcode);
 
 [[nodiscard]] IrType IrOpcodeType(IrOpcode opcode);

core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp

@@ -2,8 +2,10 @@
 #include "SpirvBackend/SpirvBda.hpp"
 #include "SpirvBackend/SpirvEmitterInstructions.hpp"
 #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp"
+#include <algorithm>
 #include <cstdio>
 #include <cstdlib>
+#include <queue>
 #include <string>
 #include <spirv/unified1/spirv.hpp>
 #include <cstdint>
@@ -718,16 +720,74 @@ void EmitStructuredInstruction(SpirvValueEmitContext& ctx, StructuredFunctionSta
     EmitDirectInstruction(ctx, inst);
 }
 
+bool IsRareSelect(const IrValue& inst) {
+    return inst.Opcode() == IrOpcode::SelectF32 && inst.ArgumentCount() == 3u && inst.Flags<SelectFlags>().rarelyTrue;
+}
+
+std::unordered_map<const IrValue*, std::vector<const IrValue*>> RareSelectArms(const SpirvEmitterState& state, const IrBlock* block) {
+    std::unordered_map<const IrValue*, std::vector<const IrValue*>> arms;
+    if (std::none_of(block->Instructions().begin(), block->Instructions().end(), [](const IrValue* inst) { return IsRareSelect(*inst); })) return arms;
+    const BlockInfo* info = BlockInfoFor(state.program, block);
+    if (info == nullptr || info->terminator.loopHeader || state.continueTarget) return arms;
+    std::unordered_set<const IrValue*> terminators;
+    for (const auto& other : state.program.Metadata().blockInfo) {
+        terminators.insert(other.condition);
+        terminators.insert(other.indirectTarget);
+    }
+    std::unordered_map<const IrValue*, std::size_t> position;
+    for (const IrValue* inst : block->Instructions()) position.emplace(inst, position.size());
+    for (const IrValue* select : block->Instructions()) {
+        if (!IsRareSelect(*select)) continue;
+        std::unordered_set<const IrValue*> arm;
+        std::unordered_set<const IrValue*> seen;
+        const auto later = [&](const IrValue* lhs, const IrValue* rhs) { return position.at(lhs) < position.at(rhs); };
+        std::priority_queue<const IrValue*, std::vector<const IrValue*>, decltype(later)> pending(later);
+        const auto consider = [&](const IrValue* value) {
+            if (value == nullptr || value->HasImmediate() || value->Parent() != block || !seen.insert(value).second) return;
+            if (value == select->Argument(0) || value == select->Argument(2) || terminators.contains(value)) return;
+            if (value->Opcode() == IrOpcode::Phi || value->Type() == IrType::Void || IrOpcodeHasSideEffects(value->Opcode())) return;
+            if (state.laneCount == 2u && state.sharedLaneValues.contains(value)) return;
+            pending.push(value);
+        };
+        consider(select->Argument(1));
+        while (!pending.empty()) {
+            const IrValue* value = pending.top();
+            pending.pop();
+            const auto& uses = value->Uses();
+            if (uses.empty() || std::any_of(uses.begin(), uses.end(), [&](const IrValue* use) { return use != select && !arm.contains(use); })) continue;
+            arm.insert(value);
+            for (std::size_t index = 0; index < value->ArgumentCount(); ++index) consider(value->Argument(index));
+        }
+        std::vector<const IrValue*> ordered(arm.begin(), arm.end());
+        std::sort(ordered.begin(), ordered.end(), later);
+        if (!ordered.empty()) arms.emplace(select, std::move(ordered));
+    }
+    return arms;
+}
+
+void EmitRareSelect(SpirvValueEmitContext& ctx, StructuredFunctionState& functionState, const IrValue& select, const std::vector<const IrValue*>& arm) {
+    auto& state = ctx.state;
+    const auto value = EmitValueOrDefaultIfCondition(state, ctx.Arg(select, 0), TypeId(state, select.Type()), ctx.Arg(select, 2), [&] {
+        for (const IrValue* inst : arm) EmitStructuredInstruction(ctx, functionState, *inst);
+        return ctx.Arg(select, 1);
+    });
+    ctx.Define(select, value);
+}
+
 void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& functionState, const IrBlock* block) {
     auto& state = ctx.state;
     state.currentBlock = block;
     EmitLabel(state, ctx.Label(block));
+    const auto rareArms = RareSelectArms(state, block);
+    std::unordered_set<const IrValue*> deferred;
+    for (const auto& [select, arm] : rareArms) deferred.insert(arm.begin(), arm.end());
     bool emittedNonPhi = false;
     // Wave LDS ordering (see WaveLdsScope): a barrier separates an LDS write from the next LDS access
     // and a read from the next write. The block may be entered right after a write.
     bool ldsWritten = true;
     bool ldsRead = false;
     for (const IrValue* inst : block->Instructions()) {
+        if (deferred.contains(inst)) continue;
         if (inst->Opcode() == IrOpcode::Phi) {
             if (emittedNonPhi) {
                 ctx.Fail(*inst, "appears after a non-Phi instruction");
@@ -764,7 +824,9 @@ void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& fu
             }
             SpirvValueEmitContext& lane = half == 0 ? ctx : *ctx.otherHalf;
             state.laneHalf = half;
-            if (half == 0 || (inst->Opcode() != IrOpcode::Barrier && inst->Opcode() != IrOpcode::MeshAllocate)) {
+            if (const auto rare = rareArms.find(inst); rare != rareArms.end()) {
+                EmitRareSelect(lane, functionState, *inst, rare->second);
+            } else if (half == 0 || (inst->Opcode() != IrOpcode::Barrier && inst->Opcode() != IrOpcode::MeshAllocate)) {
                 EmitStructuredInstruction(lane, functionState, *inst);
             }
         }

core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp

@@ -73,6 +73,7 @@ class TranslationContext {
     IrU32 quietNan32(IrU32 bits);
     IrU32 quietNan16(IrU32 bits);
     IrValue* nanResultF32(std::initializer_list<IrValue*> sources, IrValue* result, IrValue* invalidProduct = nullptr);
+    IrValue* rareFixupF32(IrValue* plain, IrValue* fixed, bool tinyFlushed);
     IrValue& invalidProductF32(IrValue* lhs, IrValue* rhs);
     std::array<IrU32, 2> quietNan64(const std::array<IrU32, 2>& bits);
     std::array<IrU32, 2> readU32Pair(const RdnaOperand& operand);

core/shader/recompiler/Translation/src/FloatInstructions.cpp

@@ -572,14 +572,16 @@ bool TranslationContext::floatBinary(const RdnaInstruction& inst, IrOpcode opcod
         const RdnaOperand& operand = sourceAt(inst, reverse ? 1u - index : index);
         args[index] = readOperand(operand, IrOpcodeArgumentType(opcode, index));
     }
-    IrValue* result = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1]});
+    IrValue* plain = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1]});
+    IrValue* result = plain;
     if (opcode == IrOpcode::FPMul32) result = &flushTinyProduct(args[0], args[1], result).Value();
+    const bool tinyFlushed = result != plain;
     if (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32) result = nanResultF32({args[0], args[1]}, result);
     if (opcode == IrOpcode::FPSub32) {
         IrValue* negated = &ir.BitCastF32(ir.BitwiseXor(ir.BitCastU32(*args[1]), ir.Constant(0x80000000u)));
         result = reverse ? nanResultF32({negated, args[0]}, result) : nanResultF32({args[0], negated}, result);
     }
-    writeOperand(inst.destination, result);
+    writeOperand(inst.destination, rareFixupF32(plain, result, tinyFlushed));
     return true;
 }
 
@@ -590,11 +592,13 @@ bool TranslationContext::floatTernary(const RdnaInstruction& inst, IrOpcode opco
         const IrType type = IrOpcodeArgumentType(opcode, index);
         args[index] = type == IrType::F32 && mix ? &readMixF32(operand).Value() : readOperand(operand, type);
     }
-    IrValue* result = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1], args[2]});
+    IrValue* plain = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1], args[2]});
+    IrValue* result = plain;
     if (opcode == IrOpcode::FPFma32) result = &flushTinyProduct(args[0], args[1], result, args[2]).Value();
+    const bool tinyFlushed = result != plain;
     const bool f32Sources = !mix || !(sourceAt(inst, 0u).opSelHi || sourceAt(inst, 1u).opSelHi || (!accumulator && sourceAt(inst, 2u).opSelHi));
     if (opcode == IrOpcode::FPFma32 && f32Sources) result = nanResultF32({args[0], args[1], args[2]}, result, &invalidProductF32(args[0], args[1]));
-    writeOperand(inst.destination, result);
+    writeOperand(inst.destination, rareFixupF32(plain, result, tinyFlushed));
     return true;
 }
 

core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp

@@ -355,6 +355,19 @@ IrValue* TranslationContext::nanResultF32(std::initializer_list<IrValue*> source
     return &ir.BitCastF32(*bits);
 }
 
+IrValue* TranslationContext::rareFixupF32(IrValue* plain, IrValue* fixed, bool tinyFlushed) {
+    if (fixed == plain) return fixed;
+    const std::uint32_t roundMode = floatMode.has_value() ? floatMode->floatMode & 3u : 0u;
+    if (tinyFlushed && roundMode != 0u) return fixed;
+    IrValue* special = nullptr;
+    if (tinyFlushed) {
+        special = &ir.ULessThan(ir.BitwiseAnd(ir.IAdd(ir.BitCastU32(*plain), ir.Constant(0x007fffffu)), ir.Constant(0x7fffffffu)), ir.Constant(0x01000000u));
+    } else {
+        special = &ir.Emit(IrOpcode::FPIsNan32, IrType::U1, {plain});
+    }
+    return &ir.Emit(IrOpcode::SelectF32, IrType::F32, {special, fixed, plain}, SelectFlags{true});
+}
+
 IrValue& TranslationContext::invalidProductF32(IrValue* lhs, IrValue* rhs) {
     IrValue& lhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*lhs), ir.Constant(0x7fffffffu));
     IrValue& rhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*rhs), ir.Constant(0x7fffffffu));