perf(shader): guard packed f16 special-value repairs

core/libs/prx/libSceAgcDriver/tests/execution/PackedFloat16Native.cpp

@@ -147,14 +147,11 @@ int main() {
         const auto supports = [&](spv::Capability capability) {
             return std::ranges::find(target.supportedCapabilities, static_cast<std::uint32_t>(capability)) != target.supportedCapabilities.end();
         };
-        if (!supports(spv::CapabilityFloat16) || !supports(spv::CapabilityRoundingModeRTE) || !supports(spv::CapabilityDenormPreserve)) {
-            std::puts("packed f16 native: skipped, the device has no native f16 with RTE rounding and kept denormals");
-            return VulkanTestSkipped;
-        }
+        const bool hasNativeF16 = supports(spv::CapabilityFloat16) && supports(spv::CapabilityRoundingModeRTE) && supports(spv::CapabilityDenormPreserve);
         std::vector<std::uint32_t> withoutFloat16;
         std::ranges::copy_if(target.supportedCapabilities, std::back_inserter(withoutFloat16), [](std::uint32_t capability) { return capability != spv::CapabilityFloat16; });
         auto emulatedTarget = target;
-        emulatedTarget.supportedCapabilities = withoutFloat16;
+        if (hasNativeF16) emulatedTarget.supportedCapabilities = withoutFloat16;
 
         const std::span<const std::uint32_t> code(Code);
         std::vector<std::uint32_t> userData(8, 0u);
@@ -188,9 +185,50 @@ int main() {
             const auto emulated = compile(emulatedTarget, mode);
             const F16x2Math nativeMath = FindF16x2Math(native.spirv.Words());
             const F16x2Math emulatedMath = FindF16x2Math(emulated.spirv.Words());
-            Require(nativeMath.add && nativeMath.mul, std::string("packed f16 native: ") + mode.name + " does not use f16vec2 add and mul");
-            Require(!nativeMath.fma, std::string("packed f16 native: ") + mode.name + " uses an f16vec2 Fma, which Vulkan does not require to be fused");
+            if (!mode.floatMode.has_value()) {
+                if (hasNativeF16) {
+                    std::printf("packed f16 test shader SPIR-V words: guarded native=%zu, existing f32 rules=%zu\n",
+                        native.spirv.Words().size(), emulated.spirv.Words().size());
+                } else {
+                    std::printf("packed f16 test shader SPIR-V words on a device without native f16: %zu\n", native.spirv.Words().size());
+                }
+            }
+            if (hasNativeF16) {
+                Require(nativeMath.add && nativeMath.mul, std::string("packed f16 native: ") + mode.name + " does not use f16vec2 add and mul");
+                Require(!nativeMath.fma, std::string("packed f16 native: ") + mode.name + " uses an f16vec2 Fma, which Vulkan does not require to be fused");
+            } else {
+                Require(!nativeMath.add && !nativeMath.mul && !nativeMath.fma,
+                    std::string("packed f16 native: ") + mode.name + " uses native f16 without required Vulkan capabilities");
+            }
             Require(!emulatedMath.add && !emulatedMath.mul && !emulatedMath.fma, std::string("packed f16 native: ") + mode.name + " uses f16vec2 math without Float16");
+
+            for (std::uint32_t tid = 0u; tid < Threads; ++tid) {
+                std::uint32_t* row = &Input[tid * Inputs];
+                row[0] = 0x7f117d01u;
+                row[1] = 0x3c003c00u;
+                row[2] = 0x40004000u;
+                row[3] = 0x38003800u;
+            }
+            const auto nativeSpecialOutput = run(native);
+            const auto emulatedSpecialOutput = run(emulated);
+            for (std::uint32_t tid = 0u; tid < Threads; ++tid) {
+                for (std::uint32_t index = 0u; index < Results; ++index) {
+                    const std::uint32_t actual = nativeSpecialOutput[tid * Results + index];
+                    const std::uint32_t expected = emulatedSpecialOutput[tid * Results + index];
+                    const std::uint32_t* row = &Input[tid * Inputs];
+                    Require(actual == expected, std::string("packed f16 native: all-lanes slow path, ") + mode.name + ", " + Names[index]
+                        + " with v4=" + Hex(row[0]) + " v5=" + Hex(row[1]) + " v6=" + Hex(row[2]) + " v7=" + Hex(row[3])
+                        + " is " + Hex(actual) + ", current rules give " + Hex(expected));
+                    if (!hasNativeF16 && index < 3u) {
+                        const std::uint32_t wanted = mode.floatMode.has_value() && mode.floatMode->ieeeMode ? 0x7f117f01u : 0x7f117d01u;
+                        Require(actual == wanted, std::string("packed f16 native: all-lanes slow path, ") + mode.name + ", " + Names[index]
+                            + " is " + Hex(actual) + ", existing special-value rules give " + Hex(wanted));
+                    }
+                }
+            }
+            compared += Threads * Results * 2u;
+
+            if (!hasNativeF16) continue;
             for (std::uint32_t batch = 0; batch < Batches; ++batch) {
                 for (auto& word : Input) word = Half() | (Half() << 16u);
                 const auto nativeOutput = run(native);

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp

@@ -145,6 +145,7 @@ enum class IrOpcode : std::uint16_t {
     PackFloat2x16Rtz,
     FPAdd16x2,
     FPMul16x2,
+    FPPackedF16Rules,
     FPAbs32,
     FPNeg32,
     FPSaturate32,

core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp

@@ -186,6 +186,7 @@ constexpr std::array<OpcodeMeta, static_cast<std::size_t>(IrOpcode::Count)> Meta
     makeMeta("PackFloat2x16Rtz", U32, F32, F32),
     makeMeta("FPAdd16x2", U32, U32, U32),
     makeMeta("FPMul16x2", U32, U32, U32),
+    makeMeta("FPPackedF16Rules", U32, U32, U32, U32, U32, U32),
     makeMeta("FPAbs32", F32, F32),
     makeMeta("FPNeg32", F32, F32),
     makeMeta("FPSaturate32", F32, F32),

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp

@@ -15,6 +15,7 @@ std::uint32_t EmitCompositeExtractU64(SpirvEmitterState& state, std::uint32_t ar
 std::uint32_t EmitPackFloat2x16Rtz(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1);
 std::uint32_t EmitFPAdd16x2(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1);
 std::uint32_t EmitFPMul16x2(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1);
+std::uint32_t EmitFPPackedF16Rules(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, std::uint32_t fast, const IrValue* flags);
 std::uint32_t EmitFPSaturate32(SpirvEmitterState& state, std::uint32_t arg0);
 std::uint32_t EmitIAdd64(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1);
 std::uint32_t EmitISub64(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1);

core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp

@@ -1453,6 +1453,117 @@ std::uint32_t EmitFPMul16x2(SpirvEmitterState& state, std::uint32_t arg0, std::u
     return Unary(state, spv::OpBitcast, TypeU32(state), Exact(state, product));
 }
 
+std::uint32_t EmitFPPackedF16Rules(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, std::uint32_t fast, const IrValue* flags) {
+    const std::uint32_t mode = flags->ImmediateU32();
+    const bool hasThird = (mode & 1u) != 0u;
+    const bool ieee = (mode & 2u) != 0u;
+    const bool fp16Overflow = (mode & 4u) != 0u;
+    const bool native = (mode & 8u) != 0u;
+    const bool clamp = (mode & 16u) != 0u;
+    const bool dx10Clamp = (mode & 32u) != 0u;
+    const auto u32 = TypeU32(state);
+    const auto boolean = TypeBool(state);
+    const auto inputCount = hasThird ? 3u : 2u;
+    const std::array<std::uint32_t, 3> inputs{arg0, arg1, arg2};
+    const auto op = [&](std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, opcode, u32, lhs, rhs); };
+    const auto test = [&](std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, opcode, boolean, lhs, rhs); };
+
+    if (native && clamp && dx10Clamp) return fast;
+
+    std::uint32_t nanWords = op(spv::OpBitwiseAnd, arg0, ConstantU32(state, 0x7fff7fffu));
+    nanWords = op(spv::OpIAdd, nanWords, ConstantU32(state, 0x03ff03ffu));
+    for (std::uint32_t index = 1u; index < inputCount; ++index) {
+        const auto candidate = op(spv::OpIAdd,
+            op(spv::OpBitwiseAnd, inputs[index], ConstantU32(state, 0x7fff7fffu)),
+            ConstantU32(state, 0x03ff03ffu));
+        nanWords = op(spv::OpBitwiseOr, nanWords, candidate);
+    }
+    const auto resultCandidate = op(spv::OpIAdd,
+        op(spv::OpBitwiseAnd, fast, ConstantU32(state, 0x7fff7fffu)),
+        ConstantU32(state, 0x03ff03ffu));
+    nanWords = op(spv::OpBitwiseOr, nanWords, resultCandidate);
+    const auto anyNan = test(spv::OpINotEqual,
+        op(spv::OpBitwiseAnd, nanWords, ConstantU32(state, 0x80008000u)), ConstantU32(state, 0u));
+
+    std::uint32_t guard = anyNan;
+    if (native && fp16Overflow) {
+        std::uint32_t overflow = ConstantBool(state, false);
+        for (std::uint32_t half = 0u; half < 2u; ++half) {
+            const auto lane = half == 0u
+                ? op(spv::OpBitwiseAnd, fast, ConstantU32(state, 0xffffu))
+                : op(spv::OpShiftRightLogical, fast, ConstantU32(state, 16u));
+            std::uint32_t laneOverflow = test(spv::OpIEqual,
+                op(spv::OpBitwiseAnd, lane, ConstantU32(state, 0x7fffu)), ConstantU32(state, 0x7c00u));
+            for (std::uint32_t index = 0u; index < inputCount; ++index) {
+                const auto source = half == 0u
+                    ? op(spv::OpBitwiseAnd, inputs[index], ConstantU32(state, 0xffffu))
+                    : op(spv::OpShiftRightLogical, inputs[index], ConstantU32(state, 16u));
+                const auto sourceIsInf = test(spv::OpIEqual,
+                    op(spv::OpBitwiseAnd, source, ConstantU32(state, 0x7fffu)), ConstantU32(state, 0x7c00u));
+                laneOverflow = Binary(state, spv::OpLogicalAnd, boolean, laneOverflow,
+                    Unary(state, spv::OpLogicalNot, boolean, sourceIsInf));
+            }
+            overflow = Binary(state, spv::OpLogicalOr, boolean, overflow, laneOverflow);
+        }
+        guard = Binary(state, spv::OpLogicalOr, boolean, guard, overflow);
+    }
+
+    return EmitValueOrDefaultIfCondition(state, guard, u32, fast, [&] {
+        const auto laneRules = [&](std::uint32_t half) {
+            std::array<std::uint32_t, 3> bits{};
+            for (std::uint32_t index = 0u; index < inputCount; ++index) {
+                bits[index] = half == 0u
+                    ? op(spv::OpBitwiseAnd, inputs[index], ConstantU32(state, 0xffffu))
+                    : op(spv::OpShiftRightLogical, inputs[index], ConstantU32(state, 16u));
+            }
+            const auto lane = half == 0u
+                ? op(spv::OpBitwiseAnd, fast, ConstantU32(state, 0xffffu))
+                : op(spv::OpShiftRightLogical, fast, ConstantU32(state, 16u));
+            const auto magnitude = [&](std::uint32_t value) { return op(spv::OpBitwiseAnd, value, ConstantU32(state, 0x7fffu)); };
+            const auto isNan = [&](std::uint32_t value) { return test(spv::OpUGreaterThan, magnitude(value), ConstantU32(state, 0x7c00u)); };
+            const auto isSnan = [&](std::uint32_t value) {
+                return Binary(state, spv::OpLogicalAnd, boolean, isNan(value),
+                    test(spv::OpIEqual, op(spv::OpBitwiseAnd, value, ConstantU32(state, 0x0200u)), ConstantU32(state, 0u)));
+            };
+            const auto quiet = [&](std::uint32_t value) {
+                return ieee ? op(spv::OpBitwiseOr, value, ConstantU32(state, 0x0200u)) : value;
+            };
+            const auto infZero = [&](std::uint32_t inf, std::uint32_t zero) {
+                const auto left = test(spv::OpIEqual, magnitude(inf), ConstantU32(state, 0x7c00u));
+                const auto right = test(spv::OpIEqual, magnitude(zero), ConstantU32(state, 0u));
+                return Binary(state, spv::OpLogicalAnd, boolean, left, right);
+            };
+            const auto invalidProduct = Binary(state, spv::OpLogicalOr, boolean,
+                infZero(bits[0], bits[1]), infZero(bits[1], bits[0]));
+
+            std::uint32_t value = lane;
+            if (native && fp16Overflow) {
+                std::uint32_t overflowLane = test(spv::OpIEqual, magnitude(value), ConstantU32(state, 0x7c00u));
+                for (std::uint32_t index = 0u; index < inputCount; ++index) {
+                    overflowLane = Binary(state, spv::OpLogicalAnd, boolean, overflowLane,
+                        test(spv::OpINotEqual, magnitude(bits[index]), ConstantU32(state, 0x7c00u)));
+                }
+                const auto limited = op(spv::OpBitwiseOr,
+                    op(spv::OpBitwiseAnd, value, ConstantU32(state, 0x8000u)), ConstantU32(state, 0x7bffu));
+                value = Select(state, u32, overflowLane, limited, value);
+            }
+            value = Select(state, u32, isNan(value), ConstantU32(state, 0xfe00u), value);
+            for (std::uint32_t index = inputCount; index-- > 0u;) {
+                std::uint32_t selected = isNan(bits[index]);
+                if (index == 2u) {
+                    selected = Binary(state, spv::OpLogicalAnd, boolean, selected,
+                        Unary(state, spv::OpLogicalNot, boolean, invalidProduct));
+                }
+                value = Select(state, u32, selected, quiet(bits[index]), value);
+            }
+            return value;
+        };
+        const auto low = laneRules(0u);
+        const auto high = laneRules(1u);
+        return op(spv::OpBitwiseOr, low, op(spv::OpShiftLeftLogical, high, ConstantU32(state, 16u)));
+    });
+}
+
 std::uint32_t EmitF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value) {
     if (NativeF16Rte(state)) return EmitNativeF32ToF16BitsRte(state, value);
     const auto u32 = TypeU32(state);

core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp

@@ -365,6 +365,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) {
         case IrOpcode::PackFloat2x16Rtz: return Invoke(EmitPackFloat2x16Rtz, ctx, inst);
         case IrOpcode::FPAdd16x2: return Invoke(EmitFPAdd16x2, ctx, inst);
         case IrOpcode::FPMul16x2: return Invoke(EmitFPMul16x2, ctx, inst);
+        case IrOpcode::FPPackedF16Rules: return Invoke(EmitFPPackedF16Rules, ctx, inst);
         case IrOpcode::FPAbs32: return Invoke(EmitFPAbs32, ctx, inst);
         case IrOpcode::FPNeg32: return Invoke(EmitFPNeg32, ctx, inst);
         case IrOpcode::FPSaturate32: return Invoke(EmitFPSaturate32, ctx, inst);

core/shader/recompiler/Translation/src/FloatInstructions.cpp

@@ -48,7 +48,36 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc
         return IrU32(ir.Emit(opcode == IrOpcode::FPAdd32 ? IrOpcode::FPAdd16x2 : IrOpcode::FPMul16x2, IrType::U32, {&lhs.Value(), &rhs.Value()}));
     };
     IrU32 result = quietSnan ? packU16Lanes(minMaxLane(false), minMaxLane(true)) : native ? nativeResult() : packHalf2x16(translateLane(false), translateLane(true));
-    if (native || !inst.destination.clamp || !dx10Clamp() || quietSnan) {
+    const bool hasPackedF16Rules = !quietSnan && (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32 || opcode == IrOpcode::FPFma32);
+    if (hasPackedF16Rules && (native || !inst.destination.clamp || !dx10Clamp())) {
+        const auto packedBits = [&](const RdnaOperand& operand) {
+            return packU16Lanes(laneBits(operand, false), laneBits(operand, true));
+        };
+        const std::uint32_t count = accumulator ? 3u : inst.sourceCount;
+        IrU32 thirdBits(ir.Constant(0u));
+        if (count == 3u) {
+            const RdnaOperand& third = accumulator ? accumulatorOperand(inst) : sourceAt(inst, 2u);
+            thirdBits = packedBits(third);
+        }
+        const IrU32 firstBits = packedBits(sourceAt(inst, 0u));
+        const IrU32 secondBits = packedBits(sourceAt(inst, 1u));
+        const std::uint32_t flags = (count == 3u ? 1u : 0u)
+            | (ieeeMode ? 2u : 0u)
+            | (fp16Overflow() ? 4u : 0u)
+            | (native ? 8u : 0u)
+            | (inst.destination.clamp ? 16u : 0u)
+            | (dx10Clamp() ? 32u : 0u);
+        result = IrU32(ir.Emit(IrOpcode::FPPackedF16Rules, IrType::U32, {
+            &firstBits.Value(),
+            &secondBits.Value(),
+            &thirdBits.Value(),
+            &result.Value(),
+            &ir.Constant(flags),
+        }));
+        result = packU16Lanes(
+            inst.destination.clamp ? clampF16Bits(inst.destination, IrU32(ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu)))) : IrU32(ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu))),
+            inst.destination.clamp ? clampF16Bits(inst.destination, IrU32(ir.ShiftRightLogical(result.Value(), ir.Constant(16u)))) : IrU32(ir.ShiftRightLogical(result.Value(), ir.Constant(16u))));
+    } else if (native || !inst.destination.clamp || !dx10Clamp() || quietSnan) {
         const auto isNan = [&](const IrU32& bits) {
             return IrU1(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u)));
         };