From 1d7f8bd3654eac69abca46ca3d6c67f08150c86b Mon Sep 17 00:00:00 2001 From: poonasor Date: Sat, 10 Oct 2026 19:16:57 -0400 Subject: [PATCH] fix(agc): keep a shader library part's modules until the part is destroyed Pipeline destroyed its shader modules right after LinkPipelineFromLibraries returned, while the pre-rasterization and fragment shader library parts built from them stay cached and are relinked later with VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT. On AMD's Windows driver (RX 6700 XT, Adrenalin 25.10.45.05) that link faults inside amdvlk64.dll at varying offsets, reading freed heap memory, in 22 agc tests on a cold driver cache; all of them pass with APS5_NO_PIPELINE_LTO=1. Running the optimized link synchronously or without a pipeline cache did not help, keeping the modules alive did. When the optimized link is enabled, the library store takes over the modules of each newly built shader part and destroys them in ClearPipelineLibraries after the pending links have finished. Modules of a pipeline whose parts were all cached are destroyed as before. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/include/PipelineLibrary.hpp | 2 +- .../libSceAgcDriver/Graphics/src/Pipeline.cpp | 2 +- .../Graphics/src/PipelineLibrary.cpp | 17 +++++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp index 62c7e9359ef..f480042a306 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp @@ -31,7 +31,7 @@ struct OptimizedPipeline { void Release() noexcept; }; -VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized = nullptr); +VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized = nullptr, std::span modules = {}); void WaitForOptimizedPipelines(VkDevice device); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 2ab89e7d4f3..5bd89e287f8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -369,7 +369,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput pipelineInfo.renderPass = renderPass; if (dynamicRendering) pipelineInfo.pNext = &rendering; timing.Mark("modules_and_state"); - if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized); + if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, _modules); else Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); if (state.depthBiasPerFace) { raster.cullMode = VK_CULL_MODE_FRONT_BIT; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp index 5e142cbe42f..263880966ca 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp @@ -37,8 +37,10 @@ using Key = std::vector; struct DeviceLibraries { PFN_vkDestroyPipeline destroyPipeline = nullptr; PFN_vkDestroyPipelineLayout destroyLayout = nullptr; + PFN_vkDestroyShaderModule destroyModule = nullptr; std::map layouts; std::array, 4> parts; + std::vector modules; PipelineLibraryCounters counters; }; @@ -99,7 +101,7 @@ std::span PipelineLibraryDynamicStates() { return dynamicStates; } -VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized) { +VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized, std::span modules) { Require(info.renderPass == VK_NULL_HANDLE && info.pDynamicState != nullptr, "pipeline library parts need dynamic rendering and the library dynamic states"); auto& store = Libraries(); std::lock_guard lock(store.mutex); @@ -107,6 +109,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip if (device.destroyPipeline == nullptr) { device.destroyPipeline = context.Function("vkDestroyPipeline"); device.destroyLayout = context.Function("vkDestroyPipelineLayout"); + device.destroyModule = context.Function("vkDestroyShaderModule"); } auto pipelineLayout = device.layouts.find(keys.layout); if (pipelineLayout == device.layouts.end()) { @@ -135,6 +138,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip const std::array contextKeys{&none, &keys.layout, &keys.layout, &keys.renderPass}; const VkPipelineRenderingCreateInfoKHR shaderRendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR}; VkPipelineDepthStencilStateCreateInfo dynamicDepth{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO}; + static const bool optimize = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr; std::vector libraries; for (std::size_t part = 0; part < flags.size(); ++part) { if (part == 0 && mesh) continue; @@ -179,6 +183,15 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip VkPipeline handle = VK_NULL_HANDLE; Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &handle), "vkCreateGraphicsPipelines library"); cache.emplace(key, handle); + if (optimize) { + for (std::uint32_t stage = 0; stage < create.stageCount; ++stage) { + for (auto& module : modules) { + if (module == VK_NULL_HANDLE || module != create.pStages[stage].module) continue; + device.modules.push_back(module); + module = VK_NULL_HANDLE; + } + } + } ++device.counters.built[part]; libraries.push_back(handle); } @@ -191,7 +204,6 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip VkPipeline pipeline = VK_NULL_HANDLE; Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &pipeline), "vkCreateGraphicsPipelines link"); ++device.counters.linked; - static const bool optimize = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr; if (optimize && optimized != nullptr) { auto slot = std::make_shared(); slot->device = context.device; @@ -242,6 +254,7 @@ void ClearPipelineLibraries(VkDevice device) { for (auto& part : libraries.parts) { for (const auto& [key, handle] : part) libraries.destroyPipeline(device, handle, nullptr); } + for (const auto module : libraries.modules) libraries.destroyModule(device, module, nullptr); for (const auto& [key, handle] : libraries.layouts) libraries.destroyLayout(device, handle, nullptr); store.devices.erase(found); }