fix(shader): shuffle both wave64 halves of a scalar with one vector shuffle

core/libs/tests/Wave32WideSubgroup.cpp

@@ -183,6 +183,52 @@ void Check(std::uint32_t size, const Value& masks) {
     }
 }
 
+std::size_t CountOpcode(std::span<const std::uint32_t> code, spv::Op opcode) {
+    std::size_t found = 0;
+    for (std::size_t offset = 5; offset < code.size(); offset += code[offset] >> 16u) {
+        Require((code[offset] >> 16u) != 0, "malformed SPIR-V instruction");
+        found += static_cast<spv::Op>(code[offset] & 0xffffu) == opcode;
+    }
+    return found;
+}
+
+void CheckTwoLaneShuffle() {
+    IrProgram program;
+    program.SetWaveSize(64);
+    SpirvEmitterState state(program, {});
+    SpirvValueEmitContext low(state);
+    SpirvValueEmitContext high(state);
+    low.otherHalf = &high;
+    high.otherHalf = &low;
+    high.half = 1u;
+    IrValue source(IrOpcode::GetAttribute, IrType::U32, 1);
+    IrValue requestedLane(IrOpcode::GetAttribute, IrType::U32, 2);
+    IrValue read(IrOpcode::ReadLane, IrType::U32, 3);
+    read.AddArgument(&source);
+    read.AddArgument(&requestedLane);
+    const auto lowSource = low.Def(&source);
+    const auto highSource = high.Def(&source);
+    const auto requested = high.Def(&requestedLane);
+    const auto shuffled = high.Shuffle(read, 0, requested);
+    Subgroup subgroup(32);
+    Lanes lowLanes(32), highLanes(32), lanes(32);
+    for (std::uint32_t lane = 0; lane < 32; ++lane) {
+        lowLanes[lane][0] = 0x1000u + lane;
+        highLanes[lane][0] = 0x2000u + lane;
+        lanes[lane][0] = (lane * 37u + 5u) % 64u;
+    }
+    subgroup.Set(lowSource, lowLanes);
+    subgroup.Set(highSource, highLanes);
+    subgroup.Set(requested, lanes);
+    const auto code = state.module.Finalize();
+    subgroup.Execute(code);
+    Require(CountOpcode(code, spv::OpGroupNonUniformShuffle) == 1u, "a wave64 shuffle of a scalar was not issued once for both halves");
+    for (std::uint32_t lane = 0; lane < 32; ++lane) {
+        const auto from = lanes[lane][0];
+        Require(subgroup.Get(shuffled)[lane][0] == (from < 32u ? 0x1000u : 0x2000u) + from % 32u, "a wave64 shuffle read the wrong half or lane");
+    }
+}
+
 }
 
 int main() {
@@ -192,6 +238,7 @@ int main() {
                 Check(size, masks);
             }
         }
+        CheckTwoLaneShuffle();
         std::puts("wave32 wide subgroup tests passed");
         return 0;
     } catch (const std::exception& error) {

core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp

@@ -166,8 +166,18 @@ std::uint32_t SpirvValueEmitContext::Shuffle(const IrValue& inst, std::size_t in
     const auto high = state.module.AllocateId();
     const auto inHigh = state.module.AllocateId();
     const auto value = state.module.AllocateId();
-    state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0u), physicalLane);
-    state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1u), physicalLane);
+    if (IsVectorType(inst.Argument(index)->Type())) {
+        state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0u), physicalLane);
+        state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1u), physicalLane);
+    } else {
+        const auto pairType = state.module.Type(spv::OpTypeVector, type, 2u);
+        const auto halves = state.module.AllocateId();
+        const auto shuffled = state.module.AllocateId();
+        state.module.AddFunction(spv::OpCompositeConstruct, pairType, halves, HalfArg(inst, index, 0u), HalfArg(inst, index, 1u));
+        state.module.AddFunction(spv::OpGroupNonUniformShuffle, pairType, shuffled, scope, halves, physicalLane);
+        state.module.AddFunction(spv::OpCompositeExtract, type, low, shuffled, 0u);
+        state.module.AddFunction(spv::OpCompositeExtract, type, high, shuffled, 1u);
+    }
     state.module.AddFunction(spv::OpINotEqual, TypeBool(state), inHigh, EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 32u)), ConstantU32(state, 0u));
     state.module.AddFunction(spv::OpSelect, type, value, inHigh, high, low);
     return value;

docs/dev/debt/shader.md

@@ -99,6 +99,7 @@
 - [f32 denormals](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/mullit/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign (`v_mullit_f32` measured on RDNA2 (gfx1036) over 6907 rows per mode at IEEE=0 and IEEE=1, so a positive denormal src2 counts as zero); the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. `v_div_scale_f32` flushes all three inputs to signed zeros, measured on RDNA2 (gfx1036) over 2744 operand triples per mode at IEEE=0 and IEEE=1 with f32 denormals flushed and kept; its scale-down behaviour and host denormal preservation remain incomplete. `v_div_fixup_f32` flushes its src1 and src2 and keeps a denormal src0 and result, measured on RDNA2 (gfx1036) over 6859 rows per mode at IEEE=0 and IEEE=1. Tininess is detected after rounding, measured on RDNA2 (gfx1036) for `v_mul_f32` and `v_mullit_f32` at IEEE=0 and IEEE=1: a product is flushed when its value rounded to 24 bits with an unbounded exponent is below 2^-126, so `0x3f7ffffe * 0x00800001` (2^-126 * (1 - 2^-46)) gives `0x00800000` and `0x3f7fffff * 0x00800000` (2^-126 * (1 - 2^-24)) gives zero. The check rounds in the f32 round mode of `FLOAT_MODE` (bits 1:0), measured in all four modes, while the products themselves are still rounded to nearest even. This applies to `v_mul_f32`, `v_mul_legacy_f32`, `v_mullit_f32`, and to `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. In the three directed modes the check is computed exactly from the significands. To nearest even (0, the mode of shaders without a known float mode) one factor is multiplied by 2^64, which is exact below 2^64, and the product with the other factor is compared with 2^-62. That multiply is a plain `OpFMul` like the product, and no f32 rounding mode is requested in the SPIR-V, so the check relies on the driver rounding f32 to nearest even by default, as the products do, and a product that rounds up to 2^-126 is the emitted product. `v_fma_mix_f32` follows this rule with mixed f16/f32 sources too, measured on RDNA2 (gfx1036) with f16 in either multiplicand slot, low and high half selection, and an f32 or f16 zero addend: f32 `0x00801002` times f16 `0x3bff` gives `0x00800000` to nearest at IEEE=0 and IEEE=1, while the neighbours `0x00801001` and `0x00801003` give zero and `0x00800001`. The tiny-product boundary was also measured for both signs in all four f32 round modes at IEEE=0; f16 inputs retain their previous conversion and denormal handling. `v_fma_mixlo_f16` and `v_fma_mixhi_f16` flush their f32 sources too and keep a product below 2^-126, measured on RDNA2 (gfx1036). `v_mad_legacy_f32` and `v_mac_legacy_f32` flush their inputs, their product and their result in every mode, see its entry; their product is still checked for tininess on its exact value, so one below 2^-126 gives zero even when it rounds to 2^-126, as the console's separately rounded encoding is not measured at this boundary. Denormal modes 1 and 2 throw for the other instructions. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_dot2_f32_f16` and `v_div_fmas_f32` (on hardware `v_div_fmas_f32` flushes only its result), f32 interpolation and `ds_add_f32` keep the previous lowering; `ds_min/max/cmpst_f32` flush both operands by the field, see the LDS float atomics entry. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested; `v_ldexp_f32` and the f32 compares are the exceptions. `v_cmp_X_f32` and `v_cmpx_X_f32` compare the operands as sign-magnitude integers with the NaN cases handled apart, so a denormal is not read as zero, measured on RDNA2 (gfx1036) over 256 operand pairs per opcode at IEEE=0 and IEEE=1 with f32 denormals kept and flushed. `v_ldexp_f32` preserves subnormal inputs and rounds a subnormal result in the f32 round mode of `FLOAT_MODE`, measured on RDNA2 (gfx1036) over 2163 rows in each of the four round modes at IEEE=0 and IEEE=1. An overflow rounds like any other result (infinity, or the largest finite value when the mode rounds that sign toward zero). With f32 denormals flushed it flushes a subnormal input and result, measured over the same rows in each round mode at IEEE=0 only. A device whose driver splits `Fma` into a multiply and an add flushes a product below 2^-126 before adding a non-zero addend in `v_fma_f32` and `v_fmac_f32`: `0x1fffffff * 0x1fffffff + 0x00800000` gives `0x00800000` instead of the fused `0x00ffffff` on RADV RENOIR (Mesa 26.2.4), whose compiled shader has `v_mad_f32` and no `v_fma_f32`. [F32DenormalFlush](../../../core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp) accepts that sum on such a device.
 - [v_rcp, v_rsq, v_sqrt, v_log, v_exp, v_sin and v_cos](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (f32 and f16) use the driver's approximation for finite results, not the console's: the low bits can differ. Only NaN, invalid, zero and infinite results and the exact sin/cos zeros are chosen on the bits. Measured on RDNA2 (gfx1036) over every f16 input and 32,832 f32 inputs: RADV returns the hardware's bits for every input of rcp, rsq, sqrt and log and all but one of exp (f16 0x11c5, 1 ULP); NVIDIA (615.71.09) stays within 2 ULP except `v_log_f32` near 1.0 (next entry). [v_sin and v_cos](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take the driver's sine or cosine of the radians of the exact cycle, except within 2^-12 of a turn from a zero of the function, where the result is the linear term of the exact distance to the zero: the hardware's constant 6.2831826 (0x40c90fd5) at the integers, which the hardware applies exactly to inputs below 2^-12 of a turn, and the f32 2π at the half and quarter turns, where the hardware is within 7 ULP of it. Outside those bands, a driver result more than 4 ULP from a Taylor series in turns of the exact cycle is replaced by the series, which is within 2 ULP of the true sine and cosine; the hardware is up to 9 ULP from them (gfx1035, 243,092 f32 inputs), so results stay within 13 ULP of the hardware on any driver. RADV returns the hardware's bits for every f16 input and for 32,422 sine and 32,486 cosine results of the 32,832 f32 inputs; the rest are 1 ULP off, except up to 14 ULP on either side of non-zero integers and up to 15 ULP beyond ±256 cycles, where the hardware's own reduction differs from the exact fraction, and up to 6 ULP inside the half- and quarter-turn bands. NVIDIA's own sine is within 1 ULP on 93% of the f32 inputs and up to 1,166 ULP off just outside the bands, its cosine within 163 ULP; the series replaces those results. A denormal f32 input with f32 denormals kept gives the hardware the input times 6.2831826; both hosts flush it.
 - [v_log_f32 and v_log_f16](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) keep the driver's log2 unless, for inputs within 1/8 of 1.0, it is more than 4 ULP from an eight-term series in x - 1, which then replaces it. `v_log_f16` goes through the same f32 lowering and is rounded to half. The series is evaluated for every `v_log` and chosen with `OpSelect` rather than a branch, so its seven fused multiply-adds and two multiplies are an unconditional cost. Measured on RDNA2 over all 3,145,728 f32 inputs within 1/8 of 1.0 (gfx1035, IEEE 0, f32 denormals flushed), the hardware is within 1 ULP of the correctly rounded log2 and the series within 2 ULP of it and within 3 ULP of the hardware (2 except at 0x3f752e47, 0x3f7baab0 and 0x3f7bc4fb), so a kept driver value is within 7 ULP of the hardware and a driver that returns the hardware's bits never triggers the gate; RADV (Mesa 26.2.4) returns the hardware's bits on the 9,280 such inputs measured on gfx1036; NVIDIA (615.71.09) was up to 23,947,836 ULP off at 1 - 2^-24 and is within 6 ULP inside the band after the change; outside it its own log2 stays up to 9 ULP off (measured up to 1.375). Over every f16 input, the change moves one `v_log_f16` result on NVIDIA (0x3bed, 1 ULP, onto the hardware's bits) and none on RADV.
+- [Wave64 shuffles of a scalar on 32-lane hosts](../../../core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp) (shader recompiler) - a wave64 shuffle of a scalar is emitted as one `OpGroupNonUniformShuffle` of a two-component vector holding both halves, not as one scalar shuffle per half. NVIDIA 580.159.03 (RTX 3090) segfaults in `libnvidia-gpucomp` (+0xe3facc) compiling GTA V's compute-69ec8524477a669e when the halves are shuffled separately; what triggers it is not known, and a larger shader may still crash with the vector form
 - [Ordered not-equal, less-than and greater-than compares](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) (shader recompiler) - `FPOrdNotEqual32` is emitted as `OpFUnordNotEqual`, and `FPOrdLessThan32` and `FPOrdGreaterThan32` as their ordered forms, each with both operands checked for NaN on their bits; the results are the same for every input. The AMD Windows driver 32.0.13058.8001 (Radeon 860M, RDNA 3.5) returns 1 for a NaN operand when any float form of not-equal feeds a ballot on a 32-wide subgroup (#825), and so does a less-than or greater-than against an infinity the driver sees as a constant (#3536), presumably rewritten into a not-equal by its compiler
 - [DB_SHADER_CONTROL.PRIMITIVE_ORDERED_PIXEL_SHADER](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the host pixel interlock spans the whole shader, not only up to `s_sendmsg` ORDERED_PS_DONE.
 - [POPS collision SGPRs](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the collision wave ID and intrawave collision mask read 0, so a title skips its POPS wait and runs its per-collision loop once.