perf(graphics): retain vertex input layouts

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -440,6 +440,15 @@ target_link_libraries(agc_shader_memory_tests PRIVATE shader_recompiler_agc_driv
 set_target_properties(agc_shader_memory_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_shader_memory_tests)
 
+add_test_executable(agc_vertex_input_tests EXCLUDE_FROM_ALL tests/VertexInput.cpp)
+target_include_directories(agc_vertex_input_tests PRIVATE
+        ${LIBS_INCLUDE_DIR}
+        ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
+)
+target_link_libraries(agc_vertex_input_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_vertex_input_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_vertex_input_tests)
+
 add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp)
 target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc)
@@ -1402,7 +1411,7 @@ if(BUILD_TESTING)
     if(WIN32)
         set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
-    foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests})
+    foreach(agcTest IN ITEMS agc_vertex_input_tests agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests})
         if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests")
             continue()
         endif()

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -359,6 +359,7 @@ struct VulkanDevice::State {
     std::unique_ptr<Graphics::TextureCache> textureCache;
     std::unique_ptr<Graphics::PipelineCache> pipelineCache;
     std::unique_ptr<Graphics::DescriptorCache> descriptorCache;
+    std::unique_ptr<Graphics::VertexInputCache> vertexInputs;
     std::unique_ptr<Graphics::SamplerCache> samplerCache;
     Graphics::ResourceCache& resourceCache = Graphics::SharedResourceCache();
     // Recorded dispatches that write a copied buffer (their results reach guest memory by a CPU
@@ -706,6 +707,7 @@ struct VulkanDevice::State {
             workgroupMemory.reset();
             if (workgroupMemoryLayout != VK_NULL_HANDLE) reinterpret_cast<PFN_vkDestroyDescriptorSetLayout>(deviceProc(device, "vkDestroyDescriptorSetLayout"))(device, workgroupMemoryLayout, nullptr);
             descriptorCache.reset();
+            vertexInputs.reset();
             emptyBuffer.reset();
             placeholderImages.reset();
             samplerCache.reset();
@@ -1730,6 +1732,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     state->textureCache = std::make_unique<Graphics::TextureCache>(graphicsContext());
     state->colorTransfer = std::make_unique<Graphics::GpuColorTransfer>(graphicsContext());
     state->descriptorCache = std::make_unique<Graphics::DescriptorCache>(graphicsContext());
+    state->vertexInputs = std::make_unique<Graphics::VertexInputCache>(graphicsContext());
     state->samplerCache = std::make_unique<Graphics::SamplerCache>();
     state->CreateWorkgroupMemoryLayout();
     if (!state->shaderProfile->NullDescriptors()) {
@@ -3236,6 +3239,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.sparseQueueMutex = &state->sparseQueueMutex;
     context.recorder = state->recorder.get();
     context.descriptorCache = state->descriptorCache.get();
+    context.vertexInputs = state->vertexInputs.get();
     context.samplerCache = state->samplerCache.get();
     context.drawIndirectFirstInstance = state->drawIndirectFirstInstance;
     context.multiDrawIndirect = state->multiDrawIndirect;

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -31,6 +31,7 @@ class DescriptorCache;
 class SamplerCache;
 class ShaderResources;
 class PlaceholderImages;
+class VertexInputCache;
 
 class PipelineCacheSynchronization {
 public:
@@ -219,6 +220,7 @@ struct Context {
     PipelineCacheSynchronization::PipelineUseLock LockPipelineCacheForUse() const {
         return pipelineCacheSynchronization != nullptr ? pipelineCacheSynchronization->LockForPipelineUse() : PipelineCacheSynchronization::PipelineUseLock{};
     }
+    VertexInputCache* vertexInputs = nullptr;
 
     template<typename TFunction>
     TFunction Function(const char* name) const {

core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp

@@ -3,6 +3,10 @@
 
 #include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp"
 #include <algorithm>
+#include <atomic>
+#include <list>
+#include <mutex>
+#include <unordered_map>
 #include <cstring>
 #include <limits>
 #include <optional>
@@ -132,6 +136,94 @@ inline VertexInputLayout BuildVertexInputLayout(const Context& context, std::spa
     return result;
 }
 
+inline void ValidateVertexAddresses(std::span<const ShaderRecompiler::VertexAttribute> attributes) {
+    for (const auto& attribute : attributes) {
+        if (NullVertexDescriptor(attribute)) continue;
+        const auto& fields = attribute.resource.fields;
+        const auto address = fields[0] | (static_cast<std::uint64_t>(fields[1] & 0xffffu) << 32u);
+        if (address == 0 || address % DecodeVertexFormat(attribute).alignment != 0) Require(false, "unaligned vertex buffer");
+    }
+}
+
+class VertexInputCache {
+public:
+    explicit VertexInputCache(const Context& context, std::size_t capacity = 1024) : context(context), capacity(capacity) {
+        Require(capacity != 0, "vertex input cache capacity is zero");
+    }
+
+    std::shared_ptr<const VertexInputLayout> TryGet(std::uint64_t variant, std::span<const ShaderRecompiler::VertexAttribute> attributes) {
+        if (attributes.empty() || variant == 0 || bypassed(variant)) return nullptr;
+        return get(variant, attributes);
+    }
+
+    std::size_t Size() const {
+        std::lock_guard lock(mutex);
+        return entries.size();
+    }
+
+private:
+    std::shared_ptr<const VertexInputLayout> get(std::uint64_t variant, std::span<const ShaderRecompiler::VertexAttribute> attributes) {
+        std::lock_guard lock(mutex);
+        const auto found = entries.find(variant);
+        if (found != entries.end()) {
+            if (!sameShape(attributes, found->second.shape)) {
+                bypass(variant);
+                order.erase(found->second.order);
+                entries.erase(found);
+                return nullptr;
+            }
+            ValidateVertexAddresses(attributes);
+            order.splice(order.begin(), order, found->second.order);
+            return found->second.layout;
+        }
+        if (bypassed(variant)) return nullptr;
+        auto layout = std::make_shared<const VertexInputLayout>(BuildVertexInputLayout(context, attributes));
+        std::vector<ShaderRecompiler::VertexAttribute> shape(attributes.begin(), attributes.end());
+        order.push_front(variant);
+        try {
+            entries.emplace(variant, Entry{layout, std::move(shape), order.begin()});
+        } catch (...) {
+            order.pop_front();
+            throw;
+        }
+        if (entries.size() > capacity) {
+            bypass(order.back());
+            entries.erase(order.back());
+            order.pop_back();
+        }
+        return layout;
+    }
+
+    static bool sameShape(std::span<const ShaderRecompiler::VertexAttribute> a, std::span<const ShaderRecompiler::VertexAttribute> b) {
+        return std::equal(a.begin(), a.end(), b.begin(), b.end(), [](const auto& x, const auto& y) {
+            return x.location == y.location && x.components == y.components && x.formatComponents == y.formatComponents && x.fetchIndex == y.fetchIndex &&
+                (x.resource.fields[1] & 0xffff0000u) == (y.resource.fields[1] & 0xffff0000u) && x.resource.fields[3] == y.resource.fields[3] && NullVertexDescriptor(x) == NullVertexDescriptor(y);
+        });
+    }
+    static std::size_t bypassIndex(std::uint64_t variant) {
+        return static_cast<std::size_t>((variant ^ (variant >> 32u)) & 65535u);
+    }
+    bool bypassed(std::uint64_t variant) const {
+        const auto index = bypassIndex(variant);
+        return (bypassBits[index / 64u].load(std::memory_order_relaxed) & (std::uint64_t{1} << (index % 64u))) != 0;
+    }
+    void bypass(std::uint64_t variant) {
+        const auto index = bypassIndex(variant);
+        bypassBits[index / 64u].fetch_or(std::uint64_t{1} << (index % 64u), std::memory_order_relaxed);
+    }
+    struct Entry {
+        std::shared_ptr<const VertexInputLayout> layout;
+        std::vector<ShaderRecompiler::VertexAttribute> shape;
+        std::list<std::uint64_t>::iterator order;
+    };
+    Context context;
+    std::size_t capacity;
+    std::array<std::atomic<std::uint64_t>, 1024> bypassBits{};
+    mutable std::mutex mutex;
+    std::list<std::uint64_t> order;
+    std::unordered_map<std::uint64_t, Entry> entries;
+};
+
 // The whole byte range a vertex buffer descriptor covers (records * stride, or records when the
 // stride is 0): what an indirect draw, whose counts only the GPU knows, copies for the fetch.
 inline std::uint32_t VertexBufferOutOfBoundsSelect(const ShaderRecompiler::VertexAttribute& attribute) {
@@ -329,4 +421,3 @@ inline std::vector<std::size_t> SoloZeroPaddedFetchIndices(const std::vector<Ver
 }
 
 #endif
-

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -971,6 +971,7 @@ struct DrawInputs {
     VkDeviceSize indexOffset = 0;
     std::uint32_t maxIndex = 0;
     VertexInputLayout vertexInput;
+    std::shared_ptr<const VertexInputLayout> retainedVertexInput;
     std::vector<std::shared_ptr<Buffer>> vertexBuffers;
     std::vector<VkBuffer> vertexHandles;
     std::vector<VkDeviceSize> vertexOffsets;
@@ -979,6 +980,7 @@ struct DrawInputs {
     VkPipelineStageFlags shaderStages = 0;
     std::uint32_t meshGroups = 0;
     std::vector<std::uint32_t> restartTable;
+    const VertexInputLayout& VertexLayout() const { return retainedVertexInput != nullptr ? *retainedVertexInput : vertexInput; }
 };
 
 std::shared_ptr<Buffer> zeroVertexBuffer(const Context& context, const ShaderRecompiler::VertexAttribute& attribute) {
@@ -1119,8 +1121,12 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
     APS5_LOG_CHARS_OUT_DEBUG("Index validation OK");
     const auto& attributes = shaders.front().program->vertexAttributes;
     // Validates the vertex descriptors; the layout also keys and builds the pipeline.
-    if (recipe != nullptr) inputs.vertexInput = recipe->vertexInput;
-    else inputs.vertexInput = BuildVertexInputLayout(context, attributes);
+    if (recipe != nullptr) {
+        inputs.vertexInput = recipe->vertexInput;
+    } else if (!attributes.empty()) {
+        if (context.vertexInputs != nullptr) inputs.retainedVertexInput = context.vertexInputs->TryGet(shaders.front().program->variantId, attributes);
+        if (inputs.retainedVertexInput == nullptr) inputs.vertexInput = BuildVertexInputLayout(context, attributes);
+    }
     inputs.vertexOffsets.assign(attributes.size(), 0);
     // An indexed draw's vertex offset moves every fetch: the copy must reach the last one.
     if (draw.indexed) {
@@ -2119,7 +2125,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
         const auto built = resources->SetLayouts();
         Require(described.layouts.main == built.main && described.layouts.fragment == built.fragment && described.layouts.split == built.split && std::ranges::equal(described.mainKey, built.mainKey) && std::ranges::equal(described.fragmentKey, built.fragmentKey), "set layouts described from the shaders differ from the draw's");
     }
-    auto pipeline = CachedPipeline(context, pipelineState, inputs.vertexInput, *resources, shaders, lean ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+    auto pipeline = CachedPipeline(context, pipelineState, inputs.VertexLayout(), *resources, shaders, lean ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
     // Resident targets' views are stable while their storage image lives, so the framebuffer is
     // reused with the pipeline; a per-draw RenderTarget gets a framebuffer of its own.
     std::vector<std::shared_ptr<StorageTexture>> owners;
@@ -2165,7 +2171,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
             for (const auto& owner : owners) recipe->targets.emplace_back(owner);
             recipe->targetViews = targetViews;
             recipe->passKey = DrawRenderPassKey(context, state, targetViews);
-            recipe->vertexInput = inputs.vertexInput;
+            recipe->vertexInput = inputs.VertexLayout();
             recipe->pushStages = PushConstantStages(shaders);
             if (recipe->pushStages != 0) {
                 recipe->pushBytes = AssemblePushConstants(shaders);