Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,20 @@ struct Builder {
return created;
}

IrValue& select(IrOpcode opcode, IrType type, IrValue& condition, IrValue& whenTrue, IrValue& whenFalse, SelectFlags flags = {}) {
auto& created = emit(opcode, type, {&condition, &whenTrue, &whenFalse});
created.SetFlags(flags);
return created;
}

IrValue& asF32(IrValue& bits) {
return emit(IrOpcode::BitCastF32U32, IrType::F32, {&bits});
}

IrValue& asU32(IrValue& value) {
return emit(IrOpcode::BitCastU32F32, IrType::U32, {&value});
}

void keep(IrValue& value) {
(void)emit(IrOpcode::ReferenceU32, IrType::Void, {&value});
}
Expand Down Expand Up @@ -150,6 +164,116 @@ int main() {
return b.eliminate() == 1u && !clamped.HasUses() && product.HasUses();
});

passed &= run("a min, max, min3, max3 or med3 of flushed values needs no flush, one with an unknown operand does", [] {
Builder b;
const auto value = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&bits}); };
const auto flushedResult = [&](IrOpcode opcode, std::initializer_list<IrValue*> arguments) -> IrValue& {
return b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(opcode, IrType::F32, arguments)}));
};
auto& x = value(b.flush(b.unknown()));
auto& y = value(b.flush(b.unknown()));
auto& z = value(b.flush(b.unknown()));
auto& raw = value(b.unknown());
IrValue* removed[] = {
&flushedResult(IrOpcode::FPMin32, {&x, &y}),
&flushedResult(IrOpcode::FPMax32, {&y, &x}),
&flushedResult(IrOpcode::FPMinTri32, {&x, &y, &z}),
&flushedResult(IrOpcode::FPMaxTri32, {&z, &y, &x}),
&flushedResult(IrOpcode::FPMedTri32, {&x, &z, &y}),
};
auto& mixedPair = flushedResult(IrOpcode::FPMax32, {&x, &raw});
auto& mixedMedian = flushedResult(IrOpcode::FPMedTri32, {&x, &y, &raw});
for (auto* flushed : removed) b.keep(*flushed);
b.keep(mixedPair);
b.keep(mixedMedian);
const bool counted = b.eliminate() == 5u;
bool kept = false;
for (auto* flushed : removed) kept |= flushed->HasUses();
return counted && !kept && mixedPair.HasUses() && mixedMedian.HasUses();
});

passed &= run("a floor or ceil needs no flush, even of an unknown value", [] {
Builder b;
auto& raw = b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&b.unknown()});
auto& floored = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::FPFloor32, IrType::F32, {&raw})}));
auto& ceiled = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::FPCeil32, IrType::F32, {&raw})}));
b.keep(floored);
b.keep(ceiled);
return b.eliminate() == 2u && !floored.HasUses() && !ceiled.HasUses();
});

passed &= run("a fract, sin or cos of a flushed value needs no flush, one of an unknown value does", [] {
Builder b;
auto& flushedInput = b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&b.flush(b.unknown())});
auto& raw = b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&b.unknown()});
const auto flushedResult = [&](IrOpcode opcode, IrValue& argument) -> IrValue& {
return b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(opcode, IrType::F32, {&argument})}));
};
IrValue* removed[] = {
&flushedResult(IrOpcode::FPFract32, flushedInput),
&flushedResult(IrOpcode::FPSin, flushedInput),
&flushedResult(IrOpcode::FPCos, flushedInput),
};
IrValue* kept[] = {
&flushedResult(IrOpcode::FPFract32, raw),
&flushedResult(IrOpcode::FPSin, raw),
&flushedResult(IrOpcode::FPCos, raw),
};
for (auto* flushed : removed) b.keep(*flushed);
for (auto* flushed : kept) b.keep(*flushed);
bool result = b.eliminate() == 3u;
for (auto* flushed : removed) result &= !flushed->HasUses();
for (auto* flushed : kept) result &= flushed->HasUses();
return result;
});

passed &= run("a max carried around a loop without a flush keeps its flush and the search ends", [] {
Builder b;
IrBlock& head = *b.block;
IrBlock& body = b.newBlock();
IrBlock& exit = b.newBlock();
head.AddBranch(&body);
body.AddBranch(&body);
body.AddBranch(&exit);
auto& initial = b.flush(b.unknown());
b.block = &body;
auto& carried = b.program.CreateValue(IrOpcode::Phi, IrType::U32);
body.AppendInstruction(&carried);
auto& step = b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&b.flush(b.unknown())});
auto& maximum = b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::FPMax32, IrType::F32, {&b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&carried}), &step})});
carried.AddPhiOperand(&head, &initial);
carried.AddPhiOperand(&body, &maximum);
auto& flushed = b.flush(maximum);
b.block = &exit;
b.keep(flushed);
return b.eliminate() == 0u && flushed.HasUses();
});

passed &= run("a truncated or round-even value, a square root or a reciprocal square root needs no flush", [] {
Builder b;
auto& unknown = b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&b.unknown()});
std::uint32_t kept = 0u;
for (const auto opcode : {IrOpcode::FPTrunc32, IrOpcode::FPRoundEven32, IrOpcode::FPSqrt, IrOpcode::FPRecipSqrt32}) {
b.keep(b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(opcode, IrType::F32, {&unknown})})));
++kept;
}
auto& reciprocal = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::FPRecip32, IrType::F32, {&unknown})}));
b.keep(reciprocal);
return b.eliminate() == kept && reciprocal.HasUses();
});

passed &= run("a sign bit alone or a value with an exponent bit set needs no flush, other masks do", [] {
Builder b;
auto& sign = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.unknown(), &b.constant(0x80000000u)}));
auto& infinity = b.flush(b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&b.constant(0x7f800000u), &b.unknown()}));
auto& lowExponent = b.flush(b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&b.unknown(), &b.constant(0x00800000u)}));
auto& quiet = b.flush(b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&b.flush(b.unknown()), &b.constant(0x00400000u)}));
auto& unknownOr = b.flush(b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&b.unknown(), &b.unknown()}));
auto& mantissa = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.unknown(), &b.constant(0x807fffffu)}));
for (auto* value : {&sign, &infinity, &lowExponent, &quiet, &unknownOr, &mantissa}) b.keep(*value);
return b.eliminate() == 3u && !sign.HasUses() && !infinity.HasUses() && !lowExponent.HasUses() && quiet.HasUses() && unknownOr.HasUses() && mantissa.HasUses();
});

passed &= run("a phi of flushed values needs no flush", [] {
Builder b;
IrBlock& head = *b.block;
Expand Down Expand Up @@ -192,5 +316,41 @@ int main() {
return loop(true) && !loop(false);
});

passed &= run("a product flushed when tiny needs no flush", [] {
Builder b;
auto& lhs = b.asF32(b.flush(b.unknown()));
auto& product = b.emit(IrOpcode::FPMul32, IrType::F32, {&lhs, &lhs});
auto& tiny = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.unknown(), &b.constant(0x207fffffu)});
auto& flushed = b.flush(b.asU32(b.select(IrOpcode::SelectF32, IrType::F32, tiny, b.asF32(b.constant(0u)), product, SelectFlags{true})));
auto& unflagged = b.flush(b.asU32(b.select(IrOpcode::SelectF32, IrType::F32, tiny, b.asF32(b.constant(0u)), product)));
b.keep(flushed);
b.keep(unflagged);
return b.eliminate() == 1u && !flushed.HasUses() && unflagged.HasUses();
});

passed &= run("a NaN choice of a quieted operand is as denormal-free as its fallback", [] {
Builder b;
auto& candidate = b.unknown();
auto& isNan = b.emit(IrOpcode::FPIsNan32, IrType::Bool, {&b.asF32(candidate)});
auto& quieted = b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&candidate, &b.constant(0x00400000u)});
auto& flushedFallback = b.flush(b.select(IrOpcode::SelectU32, IrType::U32, isNan, quieted, b.flush(b.unknown())));
auto& unknownFallback = b.flush(b.select(IrOpcode::SelectU32, IrType::U32, isNan, quieted, b.unknown()));
auto& otherValue = b.flush(b.select(IrOpcode::SelectU32, IrType::U32, isNan, b.unknown(), b.flush(b.unknown())));
b.keep(flushedFallback);
b.keep(unknownFallback);
b.keep(otherValue);
return b.eliminate() == 1u && !flushedFallback.HasUses() && unknownFallback.HasUses() && otherValue.HasUses();
});

passed &= run("a flushed value times a constant of at least one needs no flush, times a half it does", [] {
Builder b;
auto& value = b.asF32(b.flush(b.unknown()));
auto& scaled = b.flush(b.asU32(b.emit(IrOpcode::FPMul32, IrType::F32, {&value, &b.constant(0x40000000u)})));
auto& halved = b.flush(b.asU32(b.emit(IrOpcode::FPMul32, IrType::F32, {&b.constant(0x3f000000u), &value})));
b.keep(scaled);
b.keep(halved);
return b.eliminate() == 1u && !scaled.HasUses() && halved.HasUses();
});

return passed ? 0 : 1;
}
14 changes: 14 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1364,6 +1364,20 @@ void verifyHalfWaveReduction() {
{0u, 0x38000000u, 0x88000000u, spv::OpGroupNonUniformBitwiseOr, "half-wave reduction: the wave64 OR scan did not read lane 31 as a subgroup OR on 32 lanes"},
}};
for (const auto& reduction : scans) require(reductions(pixel(scan(reduction.identity, reduction.vector, reduction.scalar), 32u, true), reduction.reduce) == 1u, reduction.message);
auto comparedScan = scan(0u, 0x38000000u, 0x88000000u);
*std::find(comparedScan.begin(), comparedScan.end(), 0xbe98047eu) = 0x8a986a7eu;
comparedScan.insert(comparedScan.begin(), 0x7c080080u);
require(reductions(pixel(comparedScan, 32u, true), spv::OpGroupNonUniformBitwiseOr) == 1u, "half-wave reduction: an OR scan keyed by the lanes that failed a compare did not read lane 31 as a subgroup OR on 32 lanes");
auto narrowedScan = comparedScan;
std::vector<std::uint32_t> narrowing;
for (std::uint32_t step = 1; step <= 600u; ++step) {
const auto threshold = -static_cast<float>(step);
std::uint32_t bits = 0;
std::memcpy(&bits, &threshold, sizeof(bits));
narrowing.insert(narrowing.end(), {0x7c0800ffu, bits, 0x87fe6a7eu});
}
narrowedScan.insert(narrowedScan.begin(), narrowing.begin(), narrowing.end());
require(reductions(pixel(narrowedScan, 32u, true), spv::OpGroupNonUniformBitwiseOr) == 1u, "half-wave reduction: an OR scan keyed by lanes that 600 compares narrowed exec to did not read lane 31 as a subgroup OR on 32 lanes");
for (const auto& reduction : scans) {
const auto wide = "half-wave reduction: a wave64 scan reduced by SPIR-V opcode " + std::to_string(reduction.reduce) + " did not read lanes 31 and 63 as reductions of host invocations 0-31 and 32-63 on 64 lanes";
require(halves(pixel(scan(reduction.identity, reduction.vector, reduction.scalar), 64u, true), reduction.reduce, reduction.identity), wide.c_str());
Expand Down
129 changes: 129 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/execution/DsPermuteLanes.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "Recompiler.hpp"
#include "VulkanTestDevice.hpp"
#include <algorithm>
#include <array>
#include <cstdint>
#include <cstdio>
#include <iostream>
#include <numeric>
#include <random>
#include <string>
#include <vector>

namespace {

using AgcDriver::Graphics::Require;
using ShaderRecompiler::ShaderStage;

constexpr std::uint32_t Threads = 128;
constexpr std::uint32_t Lanes = 4096;
constexpr std::uint32_t Batches = 16;
constexpr std::uint32_t Offset = 0x74;
alignas(256) std::array<std::uint32_t, Lanes * 4> Input{};
alignas(256) std::array<std::uint32_t, Lanes> Output{};

alignas(256) constexpr std::array<std::uint32_t, 20> Wave32Code{
0x34020084, 0x8f098b08, 0x4a020209, 0x34060082, 0x8f0a8908, 0x4a06060a, 0xe0381000, 0x80000401,
0xbf8c3f70, 0x7e1002ff, 0xdeaddead, 0x7d8a0c80, 0xbe943c6a, 0xdac80000 | Offset, 0x08000504, 0xbefe0314,
0xbf8cc07f, 0xe0701000, 0x80010803, 0xbf810000,
};

alignas(256) constexpr std::array<std::uint32_t, 20> Wave64Code{
0x34020084, 0x8f098b08, 0x4a020209, 0x34060082, 0x8f0a8908, 0x4a06060a, 0xe0381000, 0x80000401,
0xbf8c3f70, 0x7e1002ff, 0xdeaddead, 0x7d8a0c80, 0xbe94246a, 0xdac80000 | Offset, 0x08000504, 0xbefe0414,
0xbf8cc07f, 0xe0701000, 0x80010803, 0xbf810000,
};

std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
const auto address = reinterpret_cast<std::uintptr_t>(data);
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
}

std::string Hex(std::uint32_t value) {
char text[16];
std::snprintf(text, sizeof(text), "0x%08x", value);
return text;
}

void Fill(std::mt19937_64& random, std::uint32_t batch) {
const std::uint32_t mode = batch % 4u;
for (std::uint32_t group = 0; group < Lanes / 32u; ++group) {
std::array<std::uint32_t, 32> order{};
std::iota(order.begin(), order.end(), 0u);
std::shuffle(order.begin(), order.end(), random);
for (std::uint32_t lane = 0; lane < 32u; ++lane) {
std::uint32_t* row = &Input[(group * 32u + lane) * 4u];
std::uint32_t target = static_cast<std::uint32_t>(random()) & 31u;
if (mode == 1u) target &= 3u;
if (mode == 2u) target = order[lane];
const std::uint32_t address = static_cast<std::uint32_t>(random()) + Offset;
row[0] = ((address & ~0x7cu) | (target << 2u)) - Offset;
row[1] = static_cast<std::uint32_t>(random());
row[2] = mode == 3u || random() % 4u != 0u ? 1u : 0u;
row[3] = 0u;
}
}
}

std::uint32_t Expected(std::uint32_t lane) {
const std::uint32_t base = lane & ~31u;
if (Input[lane * 4u + 2u] == 0u) return 0xdeaddeadu;
std::uint32_t result = 0u;
for (std::uint32_t source = base; source < base + 32u; ++source) {
const std::uint32_t* row = &Input[source * 4u];
if (row[2] != 0u && (((row[0] + Offset) >> 2u) & 31u) == (lane & 31u)) result = row[1];
}
return result;
}

void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::uint32_t waveSize, const char* name) {
std::vector<std::uint32_t> userData(8, 0u);
const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
std::copy(input.begin(), input.end(), userData.begin());
std::copy(output.begin(), output.end(), userData.begin() + 4);
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1};
ShaderRecompiler::RecompileRequest request{
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
{waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
device.Target(),
{0, 0, 0, 128}
};
request.useCache = false;
const auto result = ShaderRecompiler::Recompile(request);
std::mt19937_64 random{0x7065726d75u + waveSize};
for (std::uint32_t batch = 0; batch < Batches; ++batch) {
Fill(random, batch);
Output.fill(0xdeadbeefu);
device.Dispatch(result, Lanes / Threads, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
device.WaitIdle();
for (std::uint32_t lane = 0; lane < Lanes; ++lane) {
const std::uint32_t* row = &Input[lane * 4u];
const std::uint32_t expected = Expected(lane);
Require(Output[lane] == expected, std::string(name) + " batch " + std::to_string(batch) + ": lane " + std::to_string(lane) + " (address " + Hex(row[0]) + ", data " + Hex(row[1]) + ", exec " + std::to_string(row[2]) + ") is " + Hex(Output[lane]) + ", expected " + Hex(expected));
}
}
}

}

int main() {
try {
const auto device = OpenVulkanTestDevice();
if (!device) return VulkanTestSkipped;
if (device->Target().subgroupSize < 32u) {
std::printf("skipped, the device's subgroups are narrower than a wave (%u lanes)\n", device->Target().subgroupSize);
return VulkanTestSkipped;
}
Run(*device, Wave32Code, 32u, "wave32");
Run(*device, Wave64Code, 64u, "wave64");
std::printf("ds permute lane tests passed (%u results)\n", 2u * Batches * Lanes);
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
22 changes: 22 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,15 @@ constexpr std::uint32_t TinyLegacyExpected[32] = {
0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u,
0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u
};
constexpr std::uint32_t CancelRows[Threads][Inputs] = {
{0x00800001u, 0x3f800000u, 0x80800000u, 0x00000000u},
{0x80800001u, 0x3f800000u, 0x00800000u, 0x00000000u},
{0x3f800000u, 0x00800003u, 0x80800000u, 0x00000000u},
{0x00ffffffu, 0x3f800000u, 0x80800000u, 0x00000000u},
{0x3f000000u, 0x01000001u, 0x80800000u, 0x00000000u},
{0xbf000000u, 0x01000001u, 0x00800000u, 0x00000000u},
};
constexpr std::uint32_t CancelExpected[6] = {0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u};
constexpr const char* TinyLegacyNames[2] = {
"v_mad_legacy_f32 v18, v4, v5, v6",
"v_mac_legacy_f32 v19, v4, v5"
Expand Down Expand Up @@ -455,6 +464,13 @@ void CheckTinyLegacy(const std::string& mode) {
}
}

void CheckCancel(const std::string& mode) {
for (std::uint32_t tid = 0; tid < std::size(CancelExpected); ++tid) {
for (const std::uint32_t i : {2u, 3u, 6u}) Expect(tid, Output[tid * TinyResults + i], CancelExpected[tid], (mode + " " + TinyNames[i]).c_str());
for (std::uint32_t i = 0; i < 2; ++i) Expect(tid, Output[tid * TinyResults + 8u + i], CancelExpected[tid], (mode + " " + TinyLegacyNames[i]).c_str());
}
}

void CheckTinyDirected(std::uint32_t round) {
const std::string mode = "round mode " + std::to_string(round) + " ";
for (std::uint32_t tid = 0; tid < Threads; ++tid) {
Expand Down Expand Up @@ -494,6 +510,12 @@ int main() {
Run(*device, ShaderRecompiler::ShaderFloatMode{0x00u, false, true, false}, TinyCode, TinyRows);
CheckTinyLegacy("IEEE=1 all denormals flushed");
CheckTiny("IEEE=1 all denormals flushed");
Run(*device, std::nullopt, TinyCode, CancelRows);
CheckCancel("no float mode cancelled sum");
Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}, TinyCode, CancelRows);
CheckCancel("IEEE=0 cancelled sum");
Run(*device, ShaderRecompiler::ShaderFloatMode{0x00u, false, true, false}, TinyCode, CancelRows);
CheckCancel("IEEE=1 cancelled sum");
for (std::uint32_t round = 1u; round < 4u; ++round) {
Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u | round, true, false, false}, TinyCode, TinyRows);
CheckTinyLegacy("round mode " + std::to_string(round));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,11 @@ struct ExportFlags {
std::uint32_t pc = 0;
};

struct SelectFlags {
bool notDenormal = false;
bool rarelyTrue = false;
};

[[nodiscard]] std::size_t IrOpcodeOperandCount(IrOpcode opcode);

[[nodiscard]] IrType IrOpcodeType(IrOpcode opcode);
Expand Down
Loading
Loading