fix(shader): add the ordered count of a first active lane in the upper wave64 half

core/libs/prx/libSceAgcDriver/tests/Pm4.cpp

@@ -210,7 +210,9 @@ void testContextAndBases() {
     expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x2063}, 2), 0); }, "dispatch modifiers");
     expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x41}, 4), 0); }, "header flags");
     AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x49}), 0);
+    AgcDriver::Pm4::Validate(makePacket(0x16, {0, 0x49}), 0);
     expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x51}), 0); }, "dispatch modifiers");
+    expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x16, {0, 0x51}), 0); }, "indirect dispatch modifiers");
     execute(state, makePacket(0x13, {32}));
     execute(state, makePacket(0x26, {0x1000, 1}));
     execute(state, makePacket(0x2a, {1}));

core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp

@@ -62,7 +62,7 @@ void verifyDecode() {
     require(decoded[11].op == RdnaOpcode::SCodeEnd, "s_code_end");
 }
 
-std::string recompile(std::span<const std::uint32_t> code) {
+std::string recompile(std::span<const std::uint32_t> code, std::uint32_t subgroupSize = 64u) {
     const std::array<std::uint32_t, 4> userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu};
     const std::array<std::uint32_t, 2> capabilities{1u, 61u};
     const std::array<std::string_view, 1> extensions{"SPV_KHR_storage_buffer_storage_class"};
@@ -74,7 +74,7 @@ std::string recompile(std::span<const std::uint32_t> code) {
     request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
     request.target.vulkanVersion = 0x00403000u;
     request.target.spirvVersion = 0x00010600u;
-    request.target.subgroupSize = 64u;
+    request.target.subgroupSize = subgroupSize;
     request.target.bdaAbiVersion = 1;
     request.target.supportedCapabilities = capabilities;
     request.target.supportedExtensions = extensions;
@@ -104,6 +104,7 @@ void verifyTranslation() {
     require(recompile(trailingLoop).empty(), "a program padded with s_code_end after its last branch is refused: " + recompile(trailingLoop));
     const auto orderedAdd = withStore({0xd8fe0000u, 0x05000001u});
     require(recompile(orderedAdd).empty(), "ds_ordered_count adding to a GDS counter is refused: " + recompile(orderedAdd));
+    require(recompile(orderedAdd, 32u).empty(), "ds_ordered_count of a wave64 on 32-lane subgroups is refused: " + recompile(orderedAdd, 32u));
 
     const std::vector<std::tuple<std::string_view, std::vector<std::uint32_t>, std::string_view>> refused{
         {"s_sethalt 1", withStore({0xbf8d0001u}), "s_sethalt 1 at pc 0 halts the wave"},

core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp

@@ -25,9 +25,11 @@ constexpr std::uint32_t CounterOffset = 0x10;
 constexpr std::array<std::uint32_t, Counters> Starts{1000, 2000, 3000, 4000};
 constexpr std::uint32_t UntouchedAppend = 0x5eed0001u;
 constexpr std::uint32_t UntouchedConsume = 0x5eed0002u;
+constexpr std::uint32_t UntouchedOrdered = 0x5eed0003u;
 constexpr std::size_t ActiveLanesLiteral = 12;
-constexpr std::size_t OrderedM0Literal = 3;
-constexpr std::array<std::uint32_t, 2> OrderedM0{0u, 7u};
+constexpr std::size_t OrderedWaveMaskLiteral = 5;
+constexpr std::size_t OrderedM0Literal = 7;
+constexpr std::size_t OrderedFirstLaneLiteral = 9;
 alignas(256) std::array<std::uint32_t, MaxThreads * Results> Output{};
 alignas(16) std::array<std::uint32_t, Counters> CounterReadback{};
 
@@ -49,13 +51,24 @@ alignas(256) auto Code = std::to_array<std::uint32_t>({
 alignas(256) auto OrderedCode = std::to_array<std::uint32_t>({
     0x34060082,
     0x4a020083,
+    0x7e0a02ff, UntouchedOrdered,
+    0x360400ff, 0x0000003f,
     0xbefc03ff, 0x0000ffff,
+    0x7da604ff, 0x00000000,
     0xd8fe0310, 0x05000001,
+    0xbefe04c1,
     0xbf8cc07f,
     0xe0702000, 0x80000503,
     0xbf810000,
 });
 
+struct OrderedRun {
+    std::uint32_t m0;
+    std::uint32_t firstLaneEighths;
+};
+
+constexpr std::array<OrderedRun, 4> OrderedRuns{{{0, 0}, {7, 0}, {7, 2}, {7, 5}}};
+
 struct Workgroup {
     std::uint32_t threads;
     std::uint32_t waveSize;
@@ -144,24 +157,29 @@ void CheckWaves(const Workgroup& workgroup, std::uint32_t result, std::uint32_t
     }
 }
 
-void CheckOrdered(const Workgroup& workgroup, std::uint32_t m0, const std::array<std::uint32_t, Counters>& counters) {
+void CheckOrdered(const Workgroup& workgroup, std::uint32_t m0, std::uint32_t firstLane, const std::array<std::uint32_t, Counters>& counters) {
     const auto waves = workgroup.threads / workgroup.waveSize;
-    const auto what = workgroup.Name() + " ds_ordered_count with M0 " + std::to_string(m0);
+    const auto what = workgroup.Name() + " ds_ordered_count with M0 " + std::to_string(m0) + " from lane " + std::to_string(firstLane) + " of each wave";
+    std::vector<std::uint32_t> values(waves);
     std::vector<std::uint32_t> order(waves);
     for (std::uint32_t wave = 0; wave < waves; ++wave) {
         order[wave] = wave;
-        const auto first = wave * workgroup.waveSize;
-        for (std::uint32_t lane = first; lane < first + workgroup.waveSize; ++lane) {
+        const auto first = wave * workgroup.waveSize + firstLane;
+        values[wave] = Output[first * Results];
+        for (std::uint32_t lane = wave * workgroup.waveSize; lane < (wave + 1u) * workgroup.waveSize; ++lane) {
             const auto actual = Output[lane * Results];
-            Require(actual == Output[first * Results], what + ": lane " + std::to_string(lane) + " returned " + std::to_string(actual) + ", its wave's first lane " + std::to_string(Output[first * Results]));
+            if (lane < first) {
+                Require(actual == UntouchedOrdered, what + ": inactive lane " + std::to_string(lane) + " was written " + std::to_string(actual));
+                continue;
+            }
+            Require(actual == values[wave], what + ": lane " + std::to_string(lane) + " returned " + std::to_string(actual) + ", its wave's first active lane " + std::to_string(values[wave]));
         }
     }
-    std::sort(order.begin(), order.end(), [&](std::uint32_t left, std::uint32_t right) { return Output[left * workgroup.waveSize * Results] < Output[right * workgroup.waveSize * Results]; });
+    std::sort(order.begin(), order.end(), [&](std::uint32_t left, std::uint32_t right) { return values[left] < values[right]; });
     auto expected = Starts[0];
     for (const auto wave : order) {
-        const auto first = wave * workgroup.waveSize;
-        Require(Output[first * Results] == expected, what + ": wave " + std::to_string(wave) + " returned " + std::to_string(Output[first * Results]) + ", expected " + std::to_string(expected));
-        expected += first + 3u;
+        Require(values[wave] == expected, what + ": wave " + std::to_string(wave) + " returned " + std::to_string(values[wave]) + ", expected " + std::to_string(expected));
+        expected += wave * workgroup.waveSize + firstLane + 3u;
     }
     Require(counters[0] == expected, what + ": GDS counter 0 is " + std::to_string(counters[0]) + ", expected " + std::to_string(expected));
 }
@@ -201,11 +219,14 @@ int main() {
                 Run(*device, workgroup, Code);
                 Check(workgroup, pass, ReadCounters(queue));
             }
-            for (const auto m0 : OrderedM0) {
+            for (const auto& run : OrderedRuns) {
+                const auto firstLane = workgroup.waveSize * run.firstLaneEighths / 8u;
                 SetCounters(queue);
-                OrderedCode[OrderedM0Literal] = m0;
+                OrderedCode[OrderedWaveMaskLiteral] = workgroup.waveSize - 1u;
+                OrderedCode[OrderedM0Literal] = run.m0;
+                OrderedCode[OrderedFirstLaneLiteral] = firstLane;
                 Run(*device, workgroup, OrderedCode);
-                CheckOrdered(workgroup, m0, ReadCounters(queue));
+                CheckOrdered(workgroup, run.m0, firstLane, ReadCounters(queue));
             }
         }
         std::puts("gds append consume tests passed");

core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp

@@ -1283,11 +1283,7 @@ std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, boo
     const auto count = Binary(state, spv::OpIAdd, TypeU32(state), Unary(state, spv::OpBitCount, TypeU32(state), low), Unary(state, spv::OpBitCount, TypeU32(state), high));
     const auto first = ctx.FirstLane(ballot);
     const auto sourceLane = wave64 ? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31u)) : first;
-    auto amount = count;
-    if (ordered) {
-        amount = state.module.AllocateId();
-        state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), amount, ConstantU32(state, spv::ScopeSubgroup), ctx.Arg(inst, 4), EmitHostSubgroupLane(state, sourceLane));
-    }
+    const auto amount = ordered ? ctx.Shuffle(inst, 4, first) : count;
     const auto isFirst = Binary(state, spv::OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), sourceLane);
     const auto storageBounds = EmitMemoryElementInBounds(state, access, index);
     const auto bounds = m0Address ? AndCondition(state, storageBounds, m0Bounds) : storageBounds;