From 776e26b79685994ae58517199ff8e98d18c54dfe Mon Sep 17 00:00:00 2001 From: Karan Vora <52519664+karanvora2599@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:03:29 -0400 Subject: [PATCH] perf(shader): take the f32 mul, fma and add result fixups only when needed v_mul_f32, v_fma_f32, v_add_f32 and v_sub_f32 compute two fixups on every result: the AMD NaN result (the first NaN operand, quieted, or the default NaN) and, for products, the flush of results that are tiny after rounding to 24 bits. Both change the result only when it is a NaN or its magnitude is at most 2^-126. They were straight-line selects, so a v_mul_f32 took 16.5 SASS instructions on an RTX 4060 where the multiply is one. The translator now wraps the fixed result in a select that takes it only when the host result is NaN (add and sub) or NaN or at most 2^-126 in magnitude (mul and fma, round to nearest even). That select is flagged as rarely true, and the SPIR-V backend moves the instructions used only by its rare arm into a branch. The rare arm is the same IR as before, so results are unchanged. Loop headers, continue targets and values shared between the two halves of a wave64 program keep plain selects. A dependent chain of 2048 v_mul_f32 runs 2.2 times as fast, v_fma_f32 1.85 times and v_add_f32 1.13 times; on all-NaN inputs v_mul_f32 is 6% slower. Static SASS grows, since both paths are compiled. --- .../IntermediateRepresentation/IrOpcode.hpp | 4 ++ .../SpirvBackend/src/SpirvFlowEmitter.cpp | 64 ++++++++++++++++++- .../Translation/TranslationContext.hpp | 1 + .../Translation/src/FloatInstructions.cpp | 12 ++-- .../src/TranslationContext/OperandAccess.cpp | 13 ++++ 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 6ed27287abc..ba7415e3ff3 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -524,6 +524,10 @@ struct ExportFlags { std::uint32_t pc = 0; }; +struct SelectFlags { + bool rarelyTrue = false; +}; + [[nodiscard]] std::size_t IrOpcodeOperandCount(IrOpcode opcode); [[nodiscard]] IrType IrOpcodeType(IrOpcode opcode); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 769ec4c3291..94c371b3500 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -2,8 +2,10 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" +#include #include #include +#include #include #include #include @@ -718,16 +720,74 @@ void EmitStructuredInstruction(SpirvValueEmitContext& ctx, StructuredFunctionSta EmitDirectInstruction(ctx, inst); } +bool IsRareSelect(const IrValue& inst) { + return inst.Opcode() == IrOpcode::SelectF32 && inst.ArgumentCount() == 3u && inst.Flags().rarelyTrue; +} + +std::unordered_map> RareSelectArms(const SpirvEmitterState& state, const IrBlock* block) { + std::unordered_map> arms; + if (std::none_of(block->Instructions().begin(), block->Instructions().end(), [](const IrValue* inst) { return IsRareSelect(*inst); })) return arms; + const BlockInfo* info = BlockInfoFor(state.program, block); + if (info == nullptr || info->terminator.loopHeader || state.continueTarget) return arms; + std::unordered_set terminators; + for (const auto& other : state.program.Metadata().blockInfo) { + terminators.insert(other.condition); + terminators.insert(other.indirectTarget); + } + std::unordered_map position; + for (const IrValue* inst : block->Instructions()) position.emplace(inst, position.size()); + for (const IrValue* select : block->Instructions()) { + if (!IsRareSelect(*select)) continue; + std::unordered_set arm; + std::unordered_set seen; + const auto later = [&](const IrValue* lhs, const IrValue* rhs) { return position.at(lhs) < position.at(rhs); }; + std::priority_queue, decltype(later)> pending(later); + const auto consider = [&](const IrValue* value) { + if (value == nullptr || value->HasImmediate() || value->Parent() != block || !seen.insert(value).second) return; + if (value == select->Argument(0) || value == select->Argument(2) || terminators.contains(value)) return; + if (value->Opcode() == IrOpcode::Phi || value->Type() == IrType::Void || IrOpcodeHasSideEffects(value->Opcode())) return; + if (state.laneCount == 2u && state.sharedLaneValues.contains(value)) return; + pending.push(value); + }; + consider(select->Argument(1)); + while (!pending.empty()) { + const IrValue* value = pending.top(); + pending.pop(); + const auto& uses = value->Uses(); + if (uses.empty() || std::any_of(uses.begin(), uses.end(), [&](const IrValue* use) { return use != select && !arm.contains(use); })) continue; + arm.insert(value); + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) consider(value->Argument(index)); + } + std::vector ordered(arm.begin(), arm.end()); + std::sort(ordered.begin(), ordered.end(), later); + if (!ordered.empty()) arms.emplace(select, std::move(ordered)); + } + return arms; +} + +void EmitRareSelect(SpirvValueEmitContext& ctx, StructuredFunctionState& functionState, const IrValue& select, const std::vector& arm) { + auto& state = ctx.state; + const auto value = EmitValueOrDefaultIfCondition(state, ctx.Arg(select, 0), TypeId(state, select.Type()), ctx.Arg(select, 2), [&] { + for (const IrValue* inst : arm) EmitStructuredInstruction(ctx, functionState, *inst); + return ctx.Arg(select, 1); + }); + ctx.Define(select, value); +} + void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& functionState, const IrBlock* block) { auto& state = ctx.state; state.currentBlock = block; EmitLabel(state, ctx.Label(block)); + const auto rareArms = RareSelectArms(state, block); + std::unordered_set deferred; + for (const auto& [select, arm] : rareArms) deferred.insert(arm.begin(), arm.end()); bool emittedNonPhi = false; // Wave LDS ordering (see WaveLdsScope): a barrier separates an LDS write from the next LDS access // and a read from the next write. The block may be entered right after a write. bool ldsWritten = true; bool ldsRead = false; for (const IrValue* inst : block->Instructions()) { + if (deferred.contains(inst)) continue; if (inst->Opcode() == IrOpcode::Phi) { if (emittedNonPhi) { ctx.Fail(*inst, "appears after a non-Phi instruction"); @@ -764,7 +824,9 @@ void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& fu } SpirvValueEmitContext& lane = half == 0 ? ctx : *ctx.otherHalf; state.laneHalf = half; - if (half == 0 || (inst->Opcode() != IrOpcode::Barrier && inst->Opcode() != IrOpcode::MeshAllocate)) { + if (const auto rare = rareArms.find(inst); rare != rareArms.end()) { + EmitRareSelect(lane, functionState, *inst, rare->second); + } else if (half == 0 || (inst->Opcode() != IrOpcode::Barrier && inst->Opcode() != IrOpcode::MeshAllocate)) { EmitStructuredInstruction(lane, functionState, *inst); } } diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index bc79d3fd180..b370d893f28 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -73,6 +73,7 @@ class TranslationContext { IrU32 quietNan32(IrU32 bits); IrU32 quietNan16(IrU32 bits); IrValue* nanResultF32(std::initializer_list sources, IrValue* result, IrValue* invalidProduct = nullptr); + IrValue* rareFixupF32(IrValue* plain, IrValue* fixed, bool tinyFlushed); IrValue& invalidProductF32(IrValue* lhs, IrValue* rhs); std::array quietNan64(const std::array& bits); std::array readU32Pair(const RdnaOperand& operand); diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index e09885cb3f3..30dd7be2482 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -572,14 +572,16 @@ bool TranslationContext::floatBinary(const RdnaInstruction& inst, IrOpcode opcod const RdnaOperand& operand = sourceAt(inst, reverse ? 1u - index : index); args[index] = readOperand(operand, IrOpcodeArgumentType(opcode, index)); } - IrValue* result = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1]}); + IrValue* plain = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1]}); + IrValue* result = plain; if (opcode == IrOpcode::FPMul32) result = &flushTinyProduct(args[0], args[1], result).Value(); + const bool tinyFlushed = result != plain; if (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32) result = nanResultF32({args[0], args[1]}, result); if (opcode == IrOpcode::FPSub32) { IrValue* negated = &ir.BitCastF32(ir.BitwiseXor(ir.BitCastU32(*args[1]), ir.Constant(0x80000000u))); result = reverse ? nanResultF32({negated, args[0]}, result) : nanResultF32({args[0], negated}, result); } - writeOperand(inst.destination, result); + writeOperand(inst.destination, rareFixupF32(plain, result, tinyFlushed)); return true; } @@ -590,11 +592,13 @@ bool TranslationContext::floatTernary(const RdnaInstruction& inst, IrOpcode opco const IrType type = IrOpcodeArgumentType(opcode, index); args[index] = type == IrType::F32 && mix ? &readMixF32(operand).Value() : readOperand(operand, type); } - IrValue* result = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1], args[2]}); + IrValue* plain = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1], args[2]}); + IrValue* result = plain; if (opcode == IrOpcode::FPFma32) result = &flushTinyProduct(args[0], args[1], result, args[2]).Value(); + const bool tinyFlushed = result != plain; const bool f32Sources = !mix || !(sourceAt(inst, 0u).opSelHi || sourceAt(inst, 1u).opSelHi || (!accumulator && sourceAt(inst, 2u).opSelHi)); if (opcode == IrOpcode::FPFma32 && f32Sources) result = nanResultF32({args[0], args[1], args[2]}, result, &invalidProductF32(args[0], args[1])); - writeOperand(inst.destination, result); + writeOperand(inst.destination, rareFixupF32(plain, result, tinyFlushed)); return true; } diff --git a/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp b/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp index 27b6e20f809..e862001e960 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp @@ -355,6 +355,19 @@ IrValue* TranslationContext::nanResultF32(std::initializer_list source return &ir.BitCastF32(*bits); } +IrValue* TranslationContext::rareFixupF32(IrValue* plain, IrValue* fixed, bool tinyFlushed) { + if (fixed == plain) return fixed; + const std::uint32_t roundMode = floatMode.has_value() ? floatMode->floatMode & 3u : 0u; + if (tinyFlushed && roundMode != 0u) return fixed; + IrValue* special = nullptr; + if (tinyFlushed) { + special = &ir.ULessThan(ir.BitwiseAnd(ir.IAdd(ir.BitCastU32(*plain), ir.Constant(0x007fffffu)), ir.Constant(0x7fffffffu)), ir.Constant(0x01000000u)); + } else { + special = &ir.Emit(IrOpcode::FPIsNan32, IrType::U1, {plain}); + } + return &ir.Emit(IrOpcode::SelectF32, IrType::F32, {special, fixed, plain}, SelectFlags{true}); +} + IrValue& TranslationContext::invalidProductF32(IrValue* lhs, IrValue* rhs) { IrValue& lhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*lhs), ir.Constant(0x7fffffffu)); IrValue& rhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*rhs), ir.Constant(0x7fffffffu));