perf(recompiler): memoize SRT evaluation in a flat table and inline IrValue's hot accessors

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -4,6 +4,7 @@
 #include "Optimization/ResourceMaterializer.hpp"
 #include "Optimization/ResourceProgram.hpp"
 #include "Optimization/ShaderStageInputInfo.hpp"
+#include "Optimization/SrtWalker/SrtEvaluator.hpp"
 #include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp"
 #if ANYPS5_ENABLE_SPIRV_TOOLS
 #include "SpirvBackend/SpirvOptimizer.hpp"
@@ -79,6 +80,27 @@ void verifyRegisterSources() {
     require(!EquivalentValue(plan, &samplerRegister, &firstVector), "different register types were merged");
 }
 
+void verifyEvaluatedValues() {
+    using namespace ShaderRecompiler;
+    std::vector<std::unique_ptr<IrValue>> values;
+    for (std::uint32_t id = 0; id < 1000u; id++) {
+        values.push_back(std::make_unique<IrValue>(IrOpcode::Void, IrType::U32, id));
+    }
+    Detail::EvaluatedValues table;
+    std::uint64_t found = 0;
+    require(!table.Find(values.front().get(), found), "evaluated values: an empty table found a value");
+    for (std::uint32_t id = 0; id < values.size(); id++) {
+        table.Insert(values[id].get(), std::uint64_t{id} * 3u);
+    }
+    for (std::uint32_t id = 0; id < values.size(); id++) {
+        require(table.Find(values[id].get(), found) && found == std::uint64_t{id} * 3u, "evaluated values: a value was lost when the table grew");
+    }
+    table.Insert(values[7].get(), 0u);
+    require(table.Find(values[7].get(), found) && found == 21u, "evaluated values: a second insert replaced the first value");
+    IrValue absent(IrOpcode::Void, IrType::U32, 1000u);
+    require(!table.Find(&absent, found), "evaluated values: a value that was never inserted was found");
+}
+
 // The pure flat slots of a hand-built plan (Detail::ComputePureFlatSlots): a slot is pure unless
 // a descriptor dword, a condition, a uniform value or another slot's address cone reaches it.
 void verifyPureFlatSlots() {
@@ -603,6 +625,7 @@ int main() {
     try {
         using namespace ShaderRecompiler;
         verifyRegisterSources();
+        verifyEvaluatedValues();
         verifyPureFlatSlots();
         verifyBindlessTable();
         verifyProgramCounterRelativeData();

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrValue.hpp

@@ -106,6 +106,44 @@ class IrValue {
     IrBlock* parent;
 };
 
+inline IrOpcode IrValue::Opcode() const {
+    return opcode;
+}
+
+inline IrType IrValue::Type() const {
+    return type;
+}
+
+inline bool IrValue::HasImmediate() const {
+    return hasImmediate;
+}
+
+inline bool IrValue::IsIdentity() const {
+    return opcode == IrOpcode::Identity;
+}
+
+inline std::size_t IrValue::ArgumentCount() const {
+    return arguments.size();
+}
+
+inline IrValue* IrValue::Argument(std::size_t index) const {
+    if (index >= arguments.size()) {
+        throw std::out_of_range("IrValue::Argument index is out of range");
+    }
+    return arguments[index];
+}
+
+inline IrValue* IrValue::Resolve() const {
+    const IrValue* current = this;
+    while (current->IsIdentity()) {
+        if (current->arguments.empty() || current->arguments.front() == nullptr) {
+            throw std::runtime_error("IrValue::Resolve encountered a malformed Identity value");
+        }
+        current = current->arguments.front();
+    }
+    return const_cast<IrValue*>(current);
+}
+
 template<IrType TValueType>
 class IrTypedValue {
 public:

core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp

@@ -8,14 +8,6 @@ namespace ShaderRecompiler {
 IrValue::IrValue(IrOpcode opcode, IrType type, std::uint32_t id) : opcode(opcode), type(type), id(id), hasImmediate(false), parent(nullptr) {
 }
 
-IrOpcode IrValue::Opcode() const {
-    return opcode;
-}
-
-IrType IrValue::Type() const {
-    return type;
-}
-
 std::uint32_t IrValue::Id() const {
     return id;
 }
@@ -32,10 +24,6 @@ IrBlock* IrValue::Parent() const {
     return parent;
 }
 
-bool IrValue::HasImmediate() const {
-    return hasImmediate;
-}
-
 std::uint32_t IrValue::ImmediateU32() const {
     return static_cast<std::uint32_t>(immediateBits);
 }
@@ -96,25 +84,10 @@ bool IrValue::IsEmpty() const {
     return opcode == IrOpcode::Void;
 }
 
-bool IrValue::IsIdentity() const {
-    return opcode == IrOpcode::Identity;
-}
-
 bool IrValue::IsPhi() const {
     return opcode == IrOpcode::Phi;
 }
 
-IrValue* IrValue::Resolve() const {
-    const IrValue* current = this;
-    while (current->IsIdentity()) {
-        if (current->arguments.empty() || current->arguments.front() == nullptr) {
-            throw std::runtime_error("IrValue::Resolve encountered a malformed Identity value");
-        }
-        current = current->arguments.front();
-    }
-    return const_cast<IrValue*>(current);
-}
-
 bool IrValue::MayHaveSideEffects() const {
     return IrOpcodeHasSideEffects(opcode);
 }
@@ -127,21 +100,10 @@ std::size_t IrValue::UseCount() const {
     return uses.size();
 }
 
-std::size_t IrValue::ArgumentCount() const {
-    return arguments.size();
-}
-
 std::size_t IrValue::PhiBlockCount() const {
     return phiBlocks.size();
 }
 
-IrValue* IrValue::Argument(std::size_t index) const {
-    if (index >= arguments.size()) {
-        throw std::out_of_range("IrValue::Argument index is out of range");
-    }
-    return arguments[index];
-}
-
 IrBlock* IrValue::PhiBlock(std::size_t index) const {
     if (index >= phiBlocks.size()) {
         throw std::out_of_range("IrValue::PhiBlock index is out of range");

core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp

@@ -6,11 +6,66 @@
 
 #include <cstdint>
 #include <span>
-#include <unordered_map>
 #include <vector>
 
 namespace ShaderRecompiler::Detail {
 
+class EvaluatedValues {
+public:
+    bool Find(const IrValue* key, std::uint64_t& value) const {
+        if (_slots.empty()) {
+            return false;
+        }
+        for (std::size_t index = Home(key);; index = (index + 1u) & (_slots.size() - 1u)) {
+            const auto& slot = _slots[index];
+            if (slot.key == key) {
+                value = slot.value;
+                return true;
+            }
+            if (slot.key == nullptr) {
+                return false;
+            }
+        }
+    }
+    void Insert(const IrValue* key, std::uint64_t value) {
+        if ((_count + 1u) * 2u > _slots.size()) {
+            Grow();
+        }
+        for (std::size_t index = Home(key);; index = (index + 1u) & (_slots.size() - 1u)) {
+            auto& slot = _slots[index];
+            if (slot.key == key) {
+                return;
+            }
+            if (slot.key == nullptr) {
+                slot = {key, value};
+                ++_count;
+                return;
+            }
+        }
+    }
+
+private:
+    struct Slot {
+        const IrValue* key = nullptr;
+        std::uint64_t value = 0;
+    };
+    std::size_t Home(const IrValue* key) const {
+        return static_cast<std::size_t>((reinterpret_cast<std::uintptr_t>(key) >> 4u) * 0x9e3779b97f4a7c15ull >> 32u) & (_slots.size() - 1u);
+    }
+    void Grow() {
+        std::vector<Slot> previous(_slots.empty() ? 64u : _slots.size() * 2u);
+        previous.swap(_slots);
+        _count = 0;
+        for (const auto& slot : previous) {
+            if (slot.key != nullptr) {
+                Insert(slot.key, slot.value);
+            }
+        }
+    }
+    std::vector<Slot> _slots;
+    std::size_t _count = 0;
+};
+
 class Evaluator {
 public:
     Evaluator(const IrResourcePlan& program, const SrtRuntime& runtime, std::span<const std::uint8_t> cleanFlatSlots = {}, Evaluator* cleanEvaluator = nullptr, IrValue* activeMask = nullptr) : _program(program), _runtime(runtime), _cleanFlatSlots(cleanFlatSlots), _cleanEvaluator(cleanEvaluator), _activeMask(activeMask != nullptr ? activeMask->Resolve() : nullptr) {}
@@ -33,7 +88,7 @@ class Evaluator {
     std::span<const std::uint8_t> _cleanFlatSlots;
     Evaluator* _cleanEvaluator = nullptr;
     IrValue* _activeMask = nullptr;
-    std::unordered_map<IrValue*, std::uint64_t> _cache;
+    EvaluatedValues _cache;
     std::vector<IrValue*> _visiting;
 };
 

core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp

@@ -42,8 +42,7 @@ bool Evaluator::EvaluateWide(IrValue* raw, std::uint64_t& result) {
     if (_activeMask != nullptr && IsRuntimeSelect(inst->Opcode()) && inst->ArgumentCount() == 3 && inst->Argument(0)->Resolve() == _activeMask) {
         return EvaluateWide(inst->Argument(1), result);
     }
-    if (const auto found = _cache.find(inst); found != _cache.end()) {
-        result = found->second;
+    if (_cache.Find(inst, result)) {
         return true;
     }
     if (std::find(_visiting.begin(), _visiting.end(), inst) != _visiting.end()) {
@@ -58,7 +57,7 @@ bool Evaluator::EvaluateWide(IrValue* raw, std::uint64_t& result) {
         if (debug) std::fprintf(stderr, "[srt] cannot evaluate %s (%zu arguments)\n", std::string(IrOpcodeName(inst->Opcode())).c_str(), inst->ArgumentCount());
         return false;
     }
-    _cache.emplace(inst, out);
+    _cache.Insert(inst, out);
     result = out;
     return true;
 }