diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PackedFloat16Native.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PackedFloat16Native.cpp index e1455fa12df..bc05f7b9917 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PackedFloat16Native.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PackedFloat16Native.cpp @@ -147,14 +147,11 @@ int main() { const auto supports = [&](spv::Capability capability) { return std::ranges::find(target.supportedCapabilities, static_cast(capability)) != target.supportedCapabilities.end(); }; - if (!supports(spv::CapabilityFloat16) || !supports(spv::CapabilityRoundingModeRTE) || !supports(spv::CapabilityDenormPreserve)) { - std::puts("packed f16 native: skipped, the device has no native f16 with RTE rounding and kept denormals"); - return VulkanTestSkipped; - } + const bool hasNativeF16 = supports(spv::CapabilityFloat16) && supports(spv::CapabilityRoundingModeRTE) && supports(spv::CapabilityDenormPreserve); std::vector withoutFloat16; std::ranges::copy_if(target.supportedCapabilities, std::back_inserter(withoutFloat16), [](std::uint32_t capability) { return capability != spv::CapabilityFloat16; }); auto emulatedTarget = target; - emulatedTarget.supportedCapabilities = withoutFloat16; + if (hasNativeF16) emulatedTarget.supportedCapabilities = withoutFloat16; const std::span code(Code); std::vector userData(8, 0u); @@ -188,9 +185,50 @@ int main() { const auto emulated = compile(emulatedTarget, mode); const F16x2Math nativeMath = FindF16x2Math(native.spirv.Words()); const F16x2Math emulatedMath = FindF16x2Math(emulated.spirv.Words()); - Require(nativeMath.add && nativeMath.mul, std::string("packed f16 native: ") + mode.name + " does not use f16vec2 add and mul"); - Require(!nativeMath.fma, std::string("packed f16 native: ") + mode.name + " uses an f16vec2 Fma, which Vulkan does not require to be fused"); + if (!mode.floatMode.has_value()) { + if (hasNativeF16) { + std::printf("packed f16 test shader SPIR-V words: guarded native=%zu, existing f32 rules=%zu\n", + native.spirv.Words().size(), emulated.spirv.Words().size()); + } else { + std::printf("packed f16 test shader SPIR-V words on a device without native f16: %zu\n", native.spirv.Words().size()); + } + } + if (hasNativeF16) { + Require(nativeMath.add && nativeMath.mul, std::string("packed f16 native: ") + mode.name + " does not use f16vec2 add and mul"); + Require(!nativeMath.fma, std::string("packed f16 native: ") + mode.name + " uses an f16vec2 Fma, which Vulkan does not require to be fused"); + } else { + Require(!nativeMath.add && !nativeMath.mul && !nativeMath.fma, + std::string("packed f16 native: ") + mode.name + " uses native f16 without required Vulkan capabilities"); + } Require(!emulatedMath.add && !emulatedMath.mul && !emulatedMath.fma, std::string("packed f16 native: ") + mode.name + " uses f16vec2 math without Float16"); + + for (std::uint32_t tid = 0u; tid < Threads; ++tid) { + std::uint32_t* row = &Input[tid * Inputs]; + row[0] = 0x7f117d01u; + row[1] = 0x3c003c00u; + row[2] = 0x40004000u; + row[3] = 0x38003800u; + } + const auto nativeSpecialOutput = run(native); + const auto emulatedSpecialOutput = run(emulated); + for (std::uint32_t tid = 0u; tid < Threads; ++tid) { + for (std::uint32_t index = 0u; index < Results; ++index) { + const std::uint32_t actual = nativeSpecialOutput[tid * Results + index]; + const std::uint32_t expected = emulatedSpecialOutput[tid * Results + index]; + const std::uint32_t* row = &Input[tid * Inputs]; + Require(actual == expected, std::string("packed f16 native: all-lanes slow path, ") + mode.name + ", " + Names[index] + + " with v4=" + Hex(row[0]) + " v5=" + Hex(row[1]) + " v6=" + Hex(row[2]) + " v7=" + Hex(row[3]) + + " is " + Hex(actual) + ", current rules give " + Hex(expected)); + if (!hasNativeF16 && index < 3u) { + const std::uint32_t wanted = mode.floatMode.has_value() && mode.floatMode->ieeeMode ? 0x7f117f01u : 0x7f117d01u; + Require(actual == wanted, std::string("packed f16 native: all-lanes slow path, ") + mode.name + ", " + Names[index] + + " is " + Hex(actual) + ", existing special-value rules give " + Hex(wanted)); + } + } + } + compared += Threads * Results * 2u; + + if (!hasNativeF16) continue; for (std::uint32_t batch = 0; batch < Batches; ++batch) { for (auto& word : Input) word = Half() | (Half() << 16u); const auto nativeOutput = run(native); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 6ed27287abc..d6f1a9a8bbb 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -145,6 +145,7 @@ enum class IrOpcode : std::uint16_t { PackFloat2x16Rtz, FPAdd16x2, FPMul16x2, + FPPackedF16Rules, FPAbs32, FPNeg32, FPSaturate32, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index cdb2a2d881d..f8352386003 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -186,6 +186,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("PackFloat2x16Rtz", U32, F32, F32), makeMeta("FPAdd16x2", U32, U32, U32), makeMeta("FPMul16x2", U32, U32, U32), + makeMeta("FPPackedF16Rules", U32, U32, U32, U32, U32, U32), makeMeta("FPAbs32", F32, F32), makeMeta("FPNeg32", F32, F32), makeMeta("FPSaturate32", F32, F32), diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index 3c5550ac8d3..b1b75d66840 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -15,6 +15,7 @@ std::uint32_t EmitCompositeExtractU64(SpirvEmitterState& state, std::uint32_t ar std::uint32_t EmitPackFloat2x16Rtz(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1); std::uint32_t EmitFPAdd16x2(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1); std::uint32_t EmitFPMul16x2(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1); +std::uint32_t EmitFPPackedF16Rules(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, std::uint32_t fast, const IrValue* flags); std::uint32_t EmitFPSaturate32(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitIAdd64(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1); std::uint32_t EmitISub64(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 9936d3199ba..439fac48936 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -1453,6 +1453,117 @@ std::uint32_t EmitFPMul16x2(SpirvEmitterState& state, std::uint32_t arg0, std::u return Unary(state, spv::OpBitcast, TypeU32(state), Exact(state, product)); } +std::uint32_t EmitFPPackedF16Rules(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, std::uint32_t fast, const IrValue* flags) { + const std::uint32_t mode = flags->ImmediateU32(); + const bool hasThird = (mode & 1u) != 0u; + const bool ieee = (mode & 2u) != 0u; + const bool fp16Overflow = (mode & 4u) != 0u; + const bool native = (mode & 8u) != 0u; + const bool clamp = (mode & 16u) != 0u; + const bool dx10Clamp = (mode & 32u) != 0u; + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + const auto inputCount = hasThird ? 3u : 2u; + const std::array inputs{arg0, arg1, arg2}; + const auto op = [&](std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, opcode, u32, lhs, rhs); }; + const auto test = [&](std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, opcode, boolean, lhs, rhs); }; + + if (native && clamp && dx10Clamp) return fast; + + std::uint32_t nanWords = op(spv::OpBitwiseAnd, arg0, ConstantU32(state, 0x7fff7fffu)); + nanWords = op(spv::OpIAdd, nanWords, ConstantU32(state, 0x03ff03ffu)); + for (std::uint32_t index = 1u; index < inputCount; ++index) { + const auto candidate = op(spv::OpIAdd, + op(spv::OpBitwiseAnd, inputs[index], ConstantU32(state, 0x7fff7fffu)), + ConstantU32(state, 0x03ff03ffu)); + nanWords = op(spv::OpBitwiseOr, nanWords, candidate); + } + const auto resultCandidate = op(spv::OpIAdd, + op(spv::OpBitwiseAnd, fast, ConstantU32(state, 0x7fff7fffu)), + ConstantU32(state, 0x03ff03ffu)); + nanWords = op(spv::OpBitwiseOr, nanWords, resultCandidate); + const auto anyNan = test(spv::OpINotEqual, + op(spv::OpBitwiseAnd, nanWords, ConstantU32(state, 0x80008000u)), ConstantU32(state, 0u)); + + std::uint32_t guard = anyNan; + if (native && fp16Overflow) { + std::uint32_t overflow = ConstantBool(state, false); + for (std::uint32_t half = 0u; half < 2u; ++half) { + const auto lane = half == 0u + ? op(spv::OpBitwiseAnd, fast, ConstantU32(state, 0xffffu)) + : op(spv::OpShiftRightLogical, fast, ConstantU32(state, 16u)); + std::uint32_t laneOverflow = test(spv::OpIEqual, + op(spv::OpBitwiseAnd, lane, ConstantU32(state, 0x7fffu)), ConstantU32(state, 0x7c00u)); + for (std::uint32_t index = 0u; index < inputCount; ++index) { + const auto source = half == 0u + ? op(spv::OpBitwiseAnd, inputs[index], ConstantU32(state, 0xffffu)) + : op(spv::OpShiftRightLogical, inputs[index], ConstantU32(state, 16u)); + const auto sourceIsInf = test(spv::OpIEqual, + op(spv::OpBitwiseAnd, source, ConstantU32(state, 0x7fffu)), ConstantU32(state, 0x7c00u)); + laneOverflow = Binary(state, spv::OpLogicalAnd, boolean, laneOverflow, + Unary(state, spv::OpLogicalNot, boolean, sourceIsInf)); + } + overflow = Binary(state, spv::OpLogicalOr, boolean, overflow, laneOverflow); + } + guard = Binary(state, spv::OpLogicalOr, boolean, guard, overflow); + } + + return EmitValueOrDefaultIfCondition(state, guard, u32, fast, [&] { + const auto laneRules = [&](std::uint32_t half) { + std::array bits{}; + for (std::uint32_t index = 0u; index < inputCount; ++index) { + bits[index] = half == 0u + ? op(spv::OpBitwiseAnd, inputs[index], ConstantU32(state, 0xffffu)) + : op(spv::OpShiftRightLogical, inputs[index], ConstantU32(state, 16u)); + } + const auto lane = half == 0u + ? op(spv::OpBitwiseAnd, fast, ConstantU32(state, 0xffffu)) + : op(spv::OpShiftRightLogical, fast, ConstantU32(state, 16u)); + const auto magnitude = [&](std::uint32_t value) { return op(spv::OpBitwiseAnd, value, ConstantU32(state, 0x7fffu)); }; + const auto isNan = [&](std::uint32_t value) { return test(spv::OpUGreaterThan, magnitude(value), ConstantU32(state, 0x7c00u)); }; + const auto isSnan = [&](std::uint32_t value) { + return Binary(state, spv::OpLogicalAnd, boolean, isNan(value), + test(spv::OpIEqual, op(spv::OpBitwiseAnd, value, ConstantU32(state, 0x0200u)), ConstantU32(state, 0u))); + }; + const auto quiet = [&](std::uint32_t value) { + return ieee ? op(spv::OpBitwiseOr, value, ConstantU32(state, 0x0200u)) : value; + }; + const auto infZero = [&](std::uint32_t inf, std::uint32_t zero) { + const auto left = test(spv::OpIEqual, magnitude(inf), ConstantU32(state, 0x7c00u)); + const auto right = test(spv::OpIEqual, magnitude(zero), ConstantU32(state, 0u)); + return Binary(state, spv::OpLogicalAnd, boolean, left, right); + }; + const auto invalidProduct = Binary(state, spv::OpLogicalOr, boolean, + infZero(bits[0], bits[1]), infZero(bits[1], bits[0])); + + std::uint32_t value = lane; + if (native && fp16Overflow) { + std::uint32_t overflowLane = test(spv::OpIEqual, magnitude(value), ConstantU32(state, 0x7c00u)); + for (std::uint32_t index = 0u; index < inputCount; ++index) { + overflowLane = Binary(state, spv::OpLogicalAnd, boolean, overflowLane, + test(spv::OpINotEqual, magnitude(bits[index]), ConstantU32(state, 0x7c00u))); + } + const auto limited = op(spv::OpBitwiseOr, + op(spv::OpBitwiseAnd, value, ConstantU32(state, 0x8000u)), ConstantU32(state, 0x7bffu)); + value = Select(state, u32, overflowLane, limited, value); + } + value = Select(state, u32, isNan(value), ConstantU32(state, 0xfe00u), value); + for (std::uint32_t index = inputCount; index-- > 0u;) { + std::uint32_t selected = isNan(bits[index]); + if (index == 2u) { + selected = Binary(state, spv::OpLogicalAnd, boolean, selected, + Unary(state, spv::OpLogicalNot, boolean, invalidProduct)); + } + value = Select(state, u32, selected, quiet(bits[index]), value); + } + return value; + }; + const auto low = laneRules(0u); + const auto high = laneRules(1u); + return op(spv::OpBitwiseOr, low, op(spv::OpShiftLeftLogical, high, ConstantU32(state, 16u))); + }); +} + std::uint32_t EmitF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value) { if (NativeF16Rte(state)) return EmitNativeF32ToF16BitsRte(state, value); const auto u32 = TypeU32(state); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 769ec4c3291..9a90bd6cab0 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -365,6 +365,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::PackFloat2x16Rtz: return Invoke(EmitPackFloat2x16Rtz, ctx, inst); case IrOpcode::FPAdd16x2: return Invoke(EmitFPAdd16x2, ctx, inst); case IrOpcode::FPMul16x2: return Invoke(EmitFPMul16x2, ctx, inst); + case IrOpcode::FPPackedF16Rules: return Invoke(EmitFPPackedF16Rules, ctx, inst); case IrOpcode::FPAbs32: return Invoke(EmitFPAbs32, ctx, inst); case IrOpcode::FPNeg32: return Invoke(EmitFPNeg32, ctx, inst); case IrOpcode::FPSaturate32: return Invoke(EmitFPSaturate32, ctx, inst); diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index e09885cb3f3..a2ae642c9f8 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -48,7 +48,36 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc return IrU32(ir.Emit(opcode == IrOpcode::FPAdd32 ? IrOpcode::FPAdd16x2 : IrOpcode::FPMul16x2, IrType::U32, {&lhs.Value(), &rhs.Value()})); }; IrU32 result = quietSnan ? packU16Lanes(minMaxLane(false), minMaxLane(true)) : native ? nativeResult() : packHalf2x16(translateLane(false), translateLane(true)); - if (native || !inst.destination.clamp || !dx10Clamp() || quietSnan) { + const bool hasPackedF16Rules = !quietSnan && (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32 || opcode == IrOpcode::FPFma32); + if (hasPackedF16Rules && (native || !inst.destination.clamp || !dx10Clamp())) { + const auto packedBits = [&](const RdnaOperand& operand) { + return packU16Lanes(laneBits(operand, false), laneBits(operand, true)); + }; + const std::uint32_t count = accumulator ? 3u : inst.sourceCount; + IrU32 thirdBits(ir.Constant(0u)); + if (count == 3u) { + const RdnaOperand& third = accumulator ? accumulatorOperand(inst) : sourceAt(inst, 2u); + thirdBits = packedBits(third); + } + const IrU32 firstBits = packedBits(sourceAt(inst, 0u)); + const IrU32 secondBits = packedBits(sourceAt(inst, 1u)); + const std::uint32_t flags = (count == 3u ? 1u : 0u) + | (ieeeMode ? 2u : 0u) + | (fp16Overflow() ? 4u : 0u) + | (native ? 8u : 0u) + | (inst.destination.clamp ? 16u : 0u) + | (dx10Clamp() ? 32u : 0u); + result = IrU32(ir.Emit(IrOpcode::FPPackedF16Rules, IrType::U32, { + &firstBits.Value(), + &secondBits.Value(), + &thirdBits.Value(), + &result.Value(), + &ir.Constant(flags), + })); + result = packU16Lanes( + inst.destination.clamp ? clampF16Bits(inst.destination, IrU32(ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu)))) : IrU32(ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu))), + inst.destination.clamp ? clampF16Bits(inst.destination, IrU32(ir.ShiftRightLogical(result.Value(), ir.Constant(16u)))) : IrU32(ir.ShiftRightLogical(result.Value(), ir.Constant(16u)))); + } else if (native || !inst.destination.clamp || !dx10Clamp() || quietSnan) { const auto isNan = [&](const IrU32& bits) { return IrU1(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); };