From 2a15be09c91d95ba69186abe2b05b943122e7c9d Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:32:20 +0200 Subject: [PATCH] perf(agc): pre-sync the DCC key writers of a dispatch's images before the GPU mutex The stage-A pre-sync of a dispatch listed only the surfaces that stage B reads on the CPU. The DCC keys of an image that a recorded kernel writes were not in the list, so stage B waited for that kernel in WaitForKeyWriters with the GPU mutex held and every other worker queued behind the wait. At the Minecraft Dungeons II title the pre-sync fired once in 296,000 dispatch checks while WaitForKeyWriters synchronized 400 times per 10 s. PresyncSurfaces now also returns the key range of every image binding (PresyncSurface::dccKeys). presync waits for a key range only while recorded work writes it, the key-store memo does not already give its value (DccKeyWritersPending) and, under the mutex it takes for a moment anyway, the pending stores do not give the keys either (DccKeyWriterNeedsSync, the test WaitForKeyWriters makes before it synchronizes). Which keys the dispatch reads, and when, is unchanged: the locked lookup runs as before and finds nothing pending. APS5_NO_DCC_PRESYNC=1 leaves the key ranges out. The [presync] line counts key ranges checked, pre-syncs that only the keys required and the outcomes of WaitForKeyWriters. The test agc_driver_dcc_key_presync covers a kernel that writes the keys, a recorded fill of the keys and nothing pending, and skips with the reasons check_skips.py expects when the device has no timeline semaphores or no host import. --- .../Execution/include/Recipe.hpp | 2 +- .../Execution/include/VulkanDevice.hpp | 6 +- .../Execution/src/VulkanDevice.cpp | 29 +- .../Graphics/include/DccMetadata.hpp | 10 + .../Graphics/include/ShaderResources.hpp | 8 +- .../Graphics/src/DccMetadata.cpp | 53 +++- .../Graphics/src/ShaderResources.cpp | 25 +- .../tests/execution/DccKeyPresync.cpp | 283 ++++++++++++++++++ docs/dev/ARCHITECTURE.md | 1 + 9 files changed, 401 insertions(+), 16 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/DccKeyPresync.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp index 9a4dc16db6c..9ba265842a3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp @@ -47,7 +47,7 @@ struct Recipe { std::uint64_t dataWordsHash = 0; // The template's PresyncSurfaces() as of the attach and the host-import identity they were // computed under (a surface is CPU-read only while its memory has no import). - std::vector> presyncSurfaces; + std::vector presyncSurfaces; Graphics::HostImportsProof presyncProof; // Both false for every recipe (only reusable objects get one); kept for the asserts. bool needsCompletion = false; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 08de3994f09..10258859ac6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -243,6 +243,10 @@ class VulkanDevice { double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs; }; static PresentStatistics PresentCounts(); + struct PresyncStatistics { + std::uint64_t checked, presyncs, keyChecks, keyWaits; + }; + static PresyncStatistics PresyncCounts(); // Stage A of a dispatch's resource build, run WITHOUT GuestMemory::GpuMutex before Dispatch or // DispatchIndirect: the binding plan, data buffers, imports already serving the guest buffers, // read-only copies and the descriptor set (see ShaderResources). The dispatch completes it @@ -335,7 +339,7 @@ class VulkanDevice { // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe); // The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none. - std::uint64_t presync(std::span> surfaces); + std::uint64_t presync(std::span surfaces); std::shared_ptr computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress); // A DISPATCH_INDIRECT's argument path (see DispatchIndirect): GPU-side from the host import, or // the counts read on the CPU (the record then continues as a direct dispatch). diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index c2ebf9a5ac2..0bfbc459da9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -3751,6 +3751,8 @@ struct PresyncCounters { std::atomic checked{0}; std::atomic presyncs{0}; std::atomic waitedUs{0}; + std::atomic keyChecks{0}; + std::atomic keyWaits{0}; std::atomic lastReport{0}; }; @@ -4133,23 +4135,31 @@ RecipeOutcome VulkanDevice::DrawFromRecipe(const Graphics::State& graphics, cons return RecipeOutcome::Recorded; } -std::uint64_t VulkanDevice::presync(std::span> surfaces) { +std::uint64_t VulkanDevice::presync(std::span surfaces) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; auto& counters = Presyncs(); bool overlaps = false; - for (const auto& [address, bytes] : surfaces) overlaps = overlaps || Graphics::Recorder::SnapshotWriteOverlaps(address, static_cast(bytes)); + for (const auto& surface : surfaces) { + const auto bytes = static_cast(surface.bytes); + if (surface.dccKeys) counters.keyChecks.fetch_add(1, std::memory_order_relaxed); + overlaps = overlaps || (surface.dccKeys ? Graphics::DccKeyWritersPending(surface.address, bytes) : Graphics::Recorder::SnapshotWriteOverlaps(surface.address, bytes)); + } std::uint64_t serial = 0; + bool texelsPending = false; if (overlaps) { // No reap here (SubmitAndEpoch's would run completions, which a compute worker must not do // on queue 0's behalf, see SubmitRecorded): the dispatch reaps after the wait. std::lock_guard lock(GuestMemory::GpuMutex()); if (state->recorder) { bool open = false; - for (const auto& [address, bytes] : surfaces) { - const auto info = state->recorder->DescribePendingWrite(address, static_cast(bytes)); + for (const auto& surface : surfaces) { + const auto bytes = static_cast(surface.bytes); + if (surface.dccKeys && !Graphics::DccKeyWriterNeedsSync(surface.address, bytes)) continue; + const auto info = state->recorder->DescribePendingWrite(surface.address, bytes); if (!info.has_value()) continue; open = open || info->open; serial = std::max(serial, info->serial); + texelsPending = texelsPending || !surface.dccKeys; } if (open) serial = state->recorder->SubmitAndEpoch(); } @@ -4158,16 +4168,25 @@ std::uint64_t VulkanDevice::presync(std::span(std::chrono::duration_cast(std::chrono::steady_clock::now() - waitStart).count()), std::memory_order_relaxed); } if (profile) { const auto nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); auto last = counters.lastReport.load(); - if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) AgcDriver::ProfilePrint_nid_no_patch("[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative)\n", static_cast(counters.checked.load()), static_cast(counters.presyncs.load()), counters.waitedUs.load() / 1000.0); + if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) { + const auto keyWaitCounts = Graphics::KeyWaitCounts(); + AgcDriver::ProfilePrint_nid_no_patch("[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative); key ranges %llu checked, %llu pre-syncs that only the keys required; WaitForKeyWriters %llu calls: %llu without a pending writer, %llu known keys, %llu fills leaving uncompressed, %llu synced\n", static_cast(counters.checked.load()), static_cast(counters.presyncs.load()), counters.waitedUs.load() / 1000.0, static_cast(counters.keyChecks.load()), static_cast(counters.keyWaits.load()), static_cast(keyWaitCounts.calls), static_cast(keyWaitCounts.noPending), static_cast(keyWaitCounts.known), static_cast(keyWaitCounts.fillsUncompressed), static_cast(keyWaitCounts.synced)); + } } return serial; } +VulkanDevice::PresyncStatistics VulkanDevice::PresyncCounts() { + const auto& counters = Presyncs(); + return {counters.checked.load(std::memory_order_relaxed), counters.presyncs.load(std::memory_order_relaxed), counters.keyChecks.load(std::memory_order_relaxed), counters.keyWaits.load(std::memory_order_relaxed)}; +} + std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots, std::uint64_t dispatchThreads, std::uint64_t programAddress) { ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); // APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp index d2e9dff3a6f..765d2acde71 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -99,6 +99,16 @@ DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceByt void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes); void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys); std::optional WaitForKeyWriters(const GuestTextureResource& resource, std::uint64_t guestBytes); +bool DccKeyWritersPending(std::uint64_t metaAddress, std::size_t count); +bool DccKeyWriterNeedsSync(std::uint64_t metaAddress, std::size_t count); +struct DccKeyWaitCounts { + std::uint64_t calls; + std::uint64_t noPending; + std::uint64_t known; + std::uint64_t fillsUncompressed; + std::uint64_t synced; +}; +DccKeyWaitCounts KeyWaitCounts(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index efefda3a7f1..305feabe99d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -122,6 +122,12 @@ struct ImageTableReuseCounts { }; ImageTableReuseCounts ImageTableReuses(); +struct PresyncSurface { + std::uint64_t address; + std::uint64_t bytes; + bool dccKeys; +}; + class ShaderResources { public: ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes); @@ -280,7 +286,7 @@ class ShaderResources { // Usable right after the deferred constructor, for address-based builds too (it decodes the // compiled shader's bindings itself when stage A has not run), and on a completed object from // the resource cache (its Revalidate repeats the same lookups), without the device lock. - std::vector> PresyncSurfaces() const; + std::vector PresyncSurfaces() const; private: friend class AgcDriver::VulkanDevice; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp index f8f4d371ecb..98449f5291d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -761,13 +761,62 @@ bool PendingFillsLeaveUncompressed(const Recorder& recorder, const GuestTextureR } +namespace { + +struct KeyWaitTotals { + std::atomic calls{0}; + std::atomic noPending{0}; + std::atomic known{0}; + std::atomic fillsUncompressed{0}; + std::atomic synced{0}; +}; + +KeyWaitTotals& KeyWaits() { + static KeyWaitTotals totals; + return totals; +} + +} + +DccKeyWaitCounts KeyWaitCounts() { + const auto& totals = KeyWaits(); + return {totals.calls.load(std::memory_order_relaxed), totals.noPending.load(std::memory_order_relaxed), totals.known.load(std::memory_order_relaxed), totals.fillsUncompressed.load(std::memory_order_relaxed), totals.synced.load(std::memory_order_relaxed)}; +} + +bool DccKeyWritersPending(std::uint64_t metaAddress, std::size_t count) { + if (metaAddress == 0 || count == 0 || !Recorder::SnapshotWriteOverlaps(metaAddress, count)) return false; + auto& memo = Memo(); + std::lock_guard lock(memo.mutex); + MemoRecorder(memo); + return std::none_of(memo.entries.begin(), memo.entries.end(), [&](const GpuKeyStore& store) { return store.note != 0 && store.begin <= metaAddress && metaAddress + count <= store.end; }); +} + +bool DccKeyWriterNeedsSync(std::uint64_t metaAddress, std::size_t count) { + if (metaAddress == 0 || count == 0 || !GuestMemory::GpuMutex().HeldByThisThread()) return false; + auto* recorder = Recorder::Active(); + return recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count) && !PendingStoreKeys(*recorder, metaAddress, count).has_value(); +} + std::optional WaitForKeyWriters(const GuestTextureResource& resource, std::uint64_t guestBytes) { const auto count = static_cast(guestBytes / KeyBytes); const auto metaAddress = resource.dccAddress; if (metaAddress == 0 || count == 0 || !GuestMemory::GpuMutex().HeldByThisThread()) return std::nullopt; + auto& waits = KeyWaits(); + waits.calls.fetch_add(1, std::memory_order_relaxed); auto* recorder = Recorder::Active(); - if (recorder == nullptr || !recorder->PendingWriteOverlaps(metaAddress, count) || PendingStoreKeys(*recorder, metaAddress, count).has_value()) return std::nullopt; - if (PendingFillsLeaveUncompressed(*recorder, resource, count)) return DccKeys::Uncompressed; + if (recorder == nullptr || !recorder->PendingWriteOverlaps(metaAddress, count)) { + waits.noPending.fetch_add(1, std::memory_order_relaxed); + return std::nullopt; + } + if (PendingStoreKeys(*recorder, metaAddress, count).has_value()) { + waits.known.fetch_add(1, std::memory_order_relaxed); + return std::nullopt; + } + if (PendingFillsLeaveUncompressed(*recorder, resource, count)) { + waits.fillsUncompressed.fetch_add(1, std::memory_order_relaxed); + return DccKeys::Uncompressed; + } + waits.synced.fetch_add(1, std::memory_order_relaxed); Recorder::CountSync(2); recorder->SyncThrough(metaAddress, count); return std::nullopt; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index e31084e0c4e..a2cba7b942b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -4337,15 +4337,20 @@ void ShaderResources::finishImageTables() { bda->SetImageTables(std::move(faults), std::move(ranges)); } -std::vector> ShaderResources::PresyncSurfaces() const { - std::vector> surfaces; +std::vector ShaderResources::PresyncSurfaces() const { + std::vector surfaces; // A surface's lookup reads guest memory on the CPU unless it is served GPU-direct from a host // import: a storage image of imported memory uploads and refreshes from the import, and a // sampled texture of one views that image (SampledFromStorageEligible), so neither waits for // the producer. const auto consider = [&](std::uint32_t format, std::uint64_t address, std::uint64_t guestBytes, bool sampled) { const bool gpuDirect = sampled ? SampledFromStorageEligible(context, format, address, guestBytes) : HostImportCovers(context, address, static_cast(guestBytes)); - if (!gpuDirect) surfaces.emplace_back(address, guestBytes); + if (!gpuDirect) surfaces.push_back({address, guestBytes, false}); + }; + static const bool keyRanges = std::getenv("APS5_NO_DCC_PRESYNC") == nullptr; + const auto considerKeys = [&](std::uint64_t dccAddress, std::uint64_t guestBytes) { + const auto keyBytes = DccKeyBytes(guestBytes); + if (keyRanges && dccAddress != 0 && keyBytes != 0) surfaces.push_back({dccAddress, keyBytes, true}); }; if (completed) { // A cached object (the resource cache): Revalidate repeats the lookups of the objects the @@ -4361,18 +4366,22 @@ std::vector> ShaderResources::PresyncSur const auto& texture = textures[textureIndex++]; if (texture->ViewsStorageImage()) consider(range.format, range.address, range.bytes, false); else consider(range.format, range.address, range.bytes, true); + considerKeys(range.dccAddress, range.bytes); } } for (std::size_t index = 0; index < storageTextures.size(); ++index) { if (index != 0 && storageTextures[index] == storageTextures[index - 1]) continue; const auto& image = *storageTextures[index]; consider(image.Descriptor().format, image.Descriptor().baseAddress, image.GuestBytes(), false); + considerKeys(image.Descriptor().dccAddress, image.GuestBytes()); } return surfaces; } if (!imageRecords.empty()) { for (const auto& record : imageRecords) { - if (record.decoded) consider(record.resource.format, record.resource.baseAddress, record.guestBytes, record.sampled); + if (!record.decoded) continue; + consider(record.resource.format, record.resource.baseAddress, record.guestBytes, record.sampled); + considerKeys(record.resource.dccAddress, record.guestBytes); } return surfaces; } @@ -4381,14 +4390,18 @@ std::vector> ShaderResources::PresyncSur forEachImageElement(*deferredCompute.program, [&](const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t, std::span words) { if (IsNullTextureResource(words)) return; const auto resource = DecodeTextureResource(words); - consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage); + const auto guestBytes = DescribeSurface(resource).guestBytes; + consider(resource.format, resource.baseAddress, guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage); + considerKeys(resource.dccAddress, guestBytes); }); for (const auto& binding : deferredCompute.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::ImageTable) continue; for (std::size_t element = 0; element < binding.guestDescriptor.size() / 8u; ++element) { try { const auto resource = DecodeTextureResource(std::span(binding.guestDescriptor).subspan(element * 8u, 8u)); - consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, true); + const auto guestBytes = DescribeSurface(resource).guestBytes; + consider(resource.format, resource.baseAddress, guestBytes, true); + considerKeys(resource.dccAddress, guestBytes); } catch (const std::runtime_error&) { } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DccKeyPresync.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DccKeyPresync.cpp new file mode 100644 index 00000000000..bed49aed8fc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DccKeyPresync.cpp @@ -0,0 +1,283 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Side = 128; +constexpr std::uint32_t Format8888Unorm = 56; +constexpr std::uint32_t TileRenderTarget64KB = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t SurfaceBytes = 65536; +constexpr std::size_t KeyBytes = SurfaceBytes / 256; +constexpr std::size_t KeysOffset = SurfaceBytes; +constexpr std::size_t SourceKeysOffset = SurfaceBytes + 4096; +constexpr std::size_t BlockBytes = 2 * 65536; + +alignas(256) constexpr std::array CopyKeysCode{ + 0x34020083, 0xe0341000, 0x80000201, 0xbf8c3f70, 0xe0741000, 0x80010201, 0xbf810000, +}; + +alignas(256) constexpr std::array SampleSecondTexelCode{ + 0x34060084, 0x7e2802ff, 0x3c400000, 0x7e2a02ff, 0x3b800000, 0xf09c0f08, 0x00610a14, 0xbf8c3f70, + 0xe0781000, 0x80000a03, 0xbf810000, +}; + +constexpr std::array PointSampler{0x00000092u, 0x00fff000u, 0u, 0u}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(65536, BlockBytes)); +#endif + Require(block != nullptr, "DCC key pre-sync: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +std::array TextureDescriptor(const std::uint8_t* texels, const std::uint8_t* keys) { + const auto address = AddressOf(texels); + const auto meta = AddressOf(keys) >> 8u; + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888Unorm << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileRenderTarget64KB << 20u) | (Type2D << 28u), + 0u, + 0u, + (1u << 20u) | (1u << 21u) | (static_cast(meta & 0xffu) << 24u), + static_cast(meta >> 8u), + }; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = AddressOf(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span code, const std::vector& userData) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +struct Scenario { + std::uint8_t* texels; + std::uint8_t* keys; + std::uint8_t* source; + std::uint32_t* output; + std::vector copyData; + std::vector sampleData; +}; + +Scenario Prepare(std::uint8_t* block, std::uint8_t* outputBlock) { + Scenario scenario{block, block + KeysOffset, block + SourceKeysOffset, reinterpret_cast(outputBlock), std::vector(16, 0u), std::vector(24, 0u)}; + const auto source = BufferDescriptor(scenario.source, KeyBytes); + const auto destination = BufferDescriptor(scenario.keys, KeyBytes); + std::copy(source.begin(), source.end(), scenario.copyData.begin()); + std::copy(destination.begin(), destination.end(), scenario.copyData.begin() + 4); + const auto results = BufferDescriptor(scenario.output, Threads * 16u); + const auto texture = TextureDescriptor(scenario.texels, scenario.keys); + std::copy(results.begin(), results.end(), scenario.sampleData.begin()); + std::copy(texture.begin(), texture.end(), scenario.sampleData.begin() + 4); + std::copy(PointSampler.begin(), PointSampler.end(), scenario.sampleData.begin() + 12); + return scenario; +} + +void Reset(const Scenario& scenario) { + std::memset(scenario.texels, 0x55, SurfaceBytes); + std::memset(scenario.keys, 0xff, KeyBytes); + std::memset(scenario.source, 0x00, KeyBytes); + std::fill(scenario.output, scenario.output + Threads * 4u, 0xdeadbeefu); + AgcDriver::GuestMemory::BumpCollectEpoch(); +} + +void RecordKeyCopy(AgcDriver::VulkanDevice& device, const Scenario& scenario) { + const auto copy = Compile(device, CopyKeysCode, scenario.copyData); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + device.Dispatch(copy, 1, 1, 1, {}, reinterpret_cast(CopyKeysCode.data())); + Require(AgcDriver::Graphics::DccKeyWritersPending(AddressOf(scenario.keys), KeyBytes), "a recorded kernel that copies data over the keys is not seen as a pending key writer"); +} + +void RequireSampledClear(const Scenario& scenario, const char* what) { + for (std::uint32_t index = 0; index < Threads * 4u; ++index) { + char message[256]; + std::snprintf(message, sizeof(message), "%s: component %u of thread %u is 0x%08x, expected 0 (keys stored 0000 by the kernel)", what, index % 4u, index / 4u, scenario.output[index]); + Require(scenario.output[index] == 0u, message); + } +} + +void RunLockedSync(AgcDriver::VulkanDevice& device, const Scenario& scenario) { + Reset(scenario); + RecordKeyCopy(device, scenario); + const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData); + const auto waits = AgcDriver::Graphics::KeyWaitCounts(); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast(SampleSecondTexelCode.data())); + device.WaitIdle(); + } + const auto after = AgcDriver::Graphics::KeyWaitCounts(); + Require(after.synced > waits.synced, "a dispatch built under the GPU mutex did not wait for the kernel that copies data over the keys"); + RequireSampledClear(scenario, "a dispatch built under the GPU mutex"); +} + +void RunPresync(AgcDriver::VulkanDevice& device, const Scenario& scenario) { + Reset(scenario); + RecordKeyCopy(device, scenario); + const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData); + const auto waits = AgcDriver::Graphics::KeyWaitCounts(); + const auto presyncs = AgcDriver::VulkanDevice::PresyncCounts(); + auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast(SampleSecondTexelCode.data())); + const auto stageA = AgcDriver::VulkanDevice::PresyncCounts(); + Require(stageA.keyChecks > presyncs.keyChecks, "stage A did not list the DCC keys of the sampled texture for the pre-sync"); + Require(stageA.keyWaits == presyncs.keyWaits + 1 && stageA.presyncs == presyncs.presyncs + 1, "stage A did not wait for the kernel that copies data over the keys before the GPU mutex"); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast(SampleSecondTexelCode.data()), std::move(prepared)); + device.WaitIdle(); + } + const auto after = AgcDriver::Graphics::KeyWaitCounts(); + Require(after.synced == waits.synced, "the dispatch waited for the key writer under the GPU mutex after the pre-sync waited for it"); + RequireSampledClear(scenario, "a dispatch whose keys were pre-synced"); +} + +bool CanFillKeys(AgcDriver::VulkanDevice& device, const Scenario& scenario) { + const std::array pattern{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + const bool filled = device.FillBuffer(AddressOf(scenario.keys), KeyBytes, pattern); + device.WaitIdle(); + return filled; +} + +void RunKnownKeys(AgcDriver::VulkanDevice& device, const Scenario& scenario) { + Reset(scenario); + const std::array pattern{0u, 0u, 0u, 0u}; + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Require(device.FillBuffer(AddressOf(scenario.keys), KeyBytes, pattern), "the fill of the keys was not recorded"); + Require(AgcDriver::Graphics::DccKeyWritersPending(AddressOf(scenario.keys), KeyBytes), "a recorded fill of the keys is not seen as a pending key writer"); + Require(!AgcDriver::Graphics::DccKeyWriterNeedsSync(AddressOf(scenario.keys), KeyBytes), "the keys of a recorded constant fill need a synchronization"); + } + const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData); + const auto waits = AgcDriver::Graphics::KeyWaitCounts(); + const auto before = AgcDriver::VulkanDevice::PresyncCounts(); + auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast(SampleSecondTexelCode.data())); + const auto after = AgcDriver::VulkanDevice::PresyncCounts(); + Require(after.keyChecks > before.keyChecks, "stage A did not list the DCC keys of the sampled texture for the pre-sync"); + Require(after.presyncs == before.presyncs, "stage A waited for a fill whose keys the locked lookup derives from the pending stores"); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast(SampleSecondTexelCode.data()), std::move(prepared)); + device.WaitIdle(); + } + Require(AgcDriver::Graphics::KeyWaitCounts().synced == waits.synced, "the dispatch synchronized with the GPU although the pending fill gives the keys"); + RequireSampledClear(scenario, "a dispatch behind a pending fill of the keys"); +} + +void RunNoWriter(AgcDriver::VulkanDevice& device, const Scenario& scenario) { + const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData); + const auto before = AgcDriver::VulkanDevice::PresyncCounts(); + Require(!AgcDriver::Graphics::DccKeyWritersPending(AddressOf(scenario.keys), KeyBytes), "the idle device still reports a pending key writer"); + auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast(SampleSecondTexelCode.data())); + const auto after = AgcDriver::VulkanDevice::PresyncCounts(); + Require(after.keyChecks > before.keyChecks, "stage A did not list the DCC keys of the sampled texture for the pre-sync"); + Require(after.keyWaits == before.keyWaits && after.presyncs == before.presyncs, "stage A waited although no recorded work writes the keys"); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast(SampleSecondTexelCode.data()), std::move(prepared)); + device.WaitIdle(); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->CanWaitUnlocked()) { + std::puts("skipped, the device has no timeline semaphores: the pre-sync is not tested"); + return VulkanTestSkipped; + } + GuestBlock locked; + GuestBlock lockedOutput; + GuestBlock presynced; + GuestBlock presyncedOutput; + GuestBlock known; + GuestBlock knownOutput; + const auto control = Prepare(locked.Data(), lockedOutput.Data()); + const auto changed = Prepare(presynced.Data(), presyncedOutput.Data()); + const auto fills = Prepare(known.Data(), knownOutput.Data()); + if (!CanFillKeys(*device, control)) { + std::puts("skipped, host imports are unavailable: the key writers of the test are copied writers"); + return VulkanTestSkipped; + } + RunLockedSync(*device, control); + RunPresync(*device, changed); + RunNoWriter(*device, changed); + RunKnownKeys(*device, fills); + std::puts("DCC key pre-sync tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index c51b0407c78..81124cc96ea 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -99,6 +99,7 @@ flowchart LR - A compute shader whose LDS plus its lock dword exceeds the host's `maxComputeSharedMemorySize` is not given a `Workgroup` array. `Recompile` sets `ShaderInfo::sharedMemoryBytes` for it and the SPIR-V backend addresses the LDS in a coherent storage buffer at descriptor set 1 (`WorkgroupMemoryDescriptorSet`), with a slice of `RecompileResult::workgroupMemoryDwords` per workgroup, indexed by `WorkgroupID` and `NumWorkGroups`. `VulkanDevice` keeps one such buffer, sized from the dispatch dimensions, and binds it after the resource set; programs that fit keep using shared memory. - `Driver::RegisterShader` splits shader preparation in two. `PlanRegistered` runs on the registering guest thread: it validates the registers and entry point, decodes the code and builds the recompile request, so invalid registrations still fail there. The RDNA to IR translation and resource plan (`PrepareShader`) run on a background pool of `hardware_concurrency() / 2` detached threads (at least 2, 64 MiB stacks). - Queue 0's packet worker decodes, captures and prepares a draw and hands its recording (`Graphics::Draw` under the GPU lock) to a committer thread in [`DrawPipeline.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp), so the next packet is decoded while the previous draw is recorded. Each queued item carries the guest ranges it will write (`DrawWriteRanges`: written buffers and storage images, color targets with their DCC and CMASK, depth and stencil). A later packet that reads or writes one of those ranges, or that the pipeline can't order, drains it first; labels, submission ends, buffer fills and (with `APS5_PIPELINE_DMA=1`) DMA packets are queued as items instead. With `APS5_PIPELINE_DISPATCH=1` (off by default), a `DISPATCH_DIRECT` whose dispatch-cache entry hits with a recipe is queued as an item too, carrying the dispatch's written buffers and storage images (`DispatchWriteRanges`); the capture and the cache lookup stay on the worker and the recipe is recorded on the committer. A dispatch still drains when its lookup misses, when what it reads overlaps a write still pending, when the hit is verified against data or when its writes cannot be ranged (bindless access, GDS, fault buffer). An exception thrown by a commit is rethrown on the worker at its next enqueue or drain, as an inline draw error would be. On by default; `APS5_PIPELINED_DRAWS=0` records every draw inline, and `APS5_EXACT_DRAW_WRITES=0` widens the color and depth ranges to estimates. +- A dispatch's resource build runs stage A (binding plan, buffers, descriptor set) without the GPU lock and stage B (image lookups) under it. Between them `VulkanDevice::presync` waits, still without the lock, for recorded work that writes a surface stage B would read on the CPU and for recorded work that writes the DCC keys of an image the dispatch binds (`ShaderResources::PresyncSurfaces`), so the locked `WaitForKeyWriters` finds nothing pending to synchronize with. A key range is waited for only when recorded work writes it, the key-store memo does not give its value (`DccKeyWritersPending`, without the mutex) and the pending stores do not give it either (`DccKeyWriterNeedsSync`, under the mutex, the same test `WaitForKeyWriters` makes before it synchronizes). Stage A's own key scan runs before that wait, and stage B compares it with the keys it reads under the lock as before. `APS5_NO_DCC_PRESYNC=1` leaves the key ranges out; `APS5_NO_PRESYNC=1` disables the whole pre-sync. - The shader registry maps a code address to every header registered for that code, in registration order. Static, graphics and rectangle ABI resolution pick the header at the bound address (or one equal to it, `SameHeader`, which also takes a hull header fused by `sceAgcFuseShaderHalves` (type 3) as the registered back half (type 7) it copies, the fused registers aside; graphics ABI resolution then uses the fused header's shader registers); draws and dispatches pick the program by the stage's binary type (`RegisteredProgram`), and a front entry matching both type 2 and type 4 throws. Within a type the last registration wins; re-registering an identical earlier header makes its snapshot current again without preparing it again, and changed code drops the older headers. - The snapshot is published as pending. `SourceHandleFor`, `InvocationFor` and `ShaderPreparationTransaction::Edit` / `Read` wait for the pending job before reading `PreparedShaders`, and rethrow its failure. `APS5_SYNC_SHADER_PREPARE=1` prepares on the registering thread instead. - `APS5_LAZY_SHADER_PREPARE=1` retains registration validation and decoding but discards the temporary plan without queueing preparation. The snapshot is deferred, not pending; draws and dispatches prepare and cache it on first use. Explicit ABI resolution still prepares its requested stages. This opt-in mode takes precedence over synchronous registration preparation and trades background CPU/memory use for possible first-use stalls; disk caches and shader output are unchanged.