perf(agc): reuse import decisions across mapping changes

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -387,6 +387,7 @@ target_link_libraries(agc_driver_recorder_tests PRIVATE
         libSceAgcDriver
         SDL2-static
         libc
+        libkernel
 )
 
 set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF)

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -521,6 +521,7 @@ struct VulkanDevice::State {
             // Every ShaderResources (kept by the recorder or the resource cache) is gone now, so the
             // sets and samplers they borrowed can go.
             copiedWriters->clear();
+            Graphics::DrawCopiedWriters()->clear();
             resourceCache.Clear();
             Graphics::ClearCachedTextures(device);
             Graphics::ClearImageMirrors(device);
@@ -530,6 +531,7 @@ struct VulkanDevice::State {
             samplerCache.reset();
             textureCache.reset();
             detiler.reset();
+            Graphics::ClearHostImports(device);
             colorTransfer.reset();
             scaler.reset();
             pipelineCache.reset();
@@ -1252,7 +1254,7 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
     if (recorder.Idle()) return 1;
     static const bool drain = std::getenv("APS5_DRAIN_COMPLETION_LABELS") != nullptr;
     const auto context = graphicsContext();
-    const auto* import = Graphics::HostImportFor(context, address, bytes.size());
+    const auto import = Graphics::HostImportFor(context, address, bytes.size());
     if (import == nullptr) {
         // Memory the GPU has no view of: the store is a completion action of the batch (it runs after
         // the recorded work, in order, like a label behind write-backs) instead of a device drain.
@@ -1269,7 +1271,10 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
     // pair for a whole group of labels recorded back to back (Recorder::RecordStore orders the
     // stores of a run against each other and against the work before and after, the queued DCC
     // key stores included).
-    const auto recordStore = [&] { recorder.RecordStore(import->buffer, address - import->base, bytes, address); };
+    const auto recordStore = [&] {
+        recorder.RecordStore(import->buffer, address - import->base, bytes, address);
+        if (Graphics::ShadowVerify()) recorder.Keep(import);
+    };
     // Only a write-back that can land over these bytes (a listed copied writer of the range, as
     // FillBuffer tests) puts the label behind the completions; any other pending completion
     // stores elsewhere. Debug aid: APS5_LABEL_GATE_ALL=1 gates on any completion, as before (a
@@ -1320,12 +1325,13 @@ bool VulkanDevice::DumpSamplesOnGpu(std::uint64_t address) {
     if (state->CopiedWriterOverlaps(address, bytes) || (Graphics::Recorder::PendingCompletionLabels() != 0 && recorder.CompletionLabelIn(address, bytes))) return false;
     const auto context = graphicsContext();
     Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "occlusion counter dump", Graphics::PublishScope::PartialUnits);
-    const auto* import = Graphics::HostImportFor(context, address, bytes);
+    const auto import = Graphics::HostImportFor(context, address, bytes);
     if (import == nullptr || import->address == 0) return false;
     if (Graphics::AnyShadowedOverlaps(address, bytes)) Graphics::PublishShadow(address, bytes, Graphics::PublishScope::PartialUnits, Graphics::PublishReason::Label);
     recorder.FlushStoresOverlapping(address, bytes);
     recorder.FlushKeyStoresOverlapping(address, bytes);
     if (!recorder.DumpSamples(import->address + (address - import->base))) return false;
+    if (Graphics::ShadowVerify()) recorder.Keep(import);
     recorder.NotePendingWrite(address, bytes);
     GuestMemory::MarkWritten(address, bytes);
     return true;
@@ -1340,7 +1346,7 @@ bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::spa
     // import table. Debug aid: APS5_NO_OPPORTUNISTIC_REAP=1 skips it.
     if (OpportunisticReap()) recorder.Reap();
     const auto context = graphicsContext();
-    const Graphics::HostImport* import = Graphics::HostImportFor(context, address, bytes);
+    std::shared_ptr<Graphics::HostImport> import = Graphics::HostImportFor(context, address, bytes);
     if (import == nullptr) return false;
     // Earlier recorded work that stores into the range must land before the fill, or it would
     // overwrite the fill later. Stores the GPU makes (direct dispatch writes, GPU labels, earlier
@@ -1699,8 +1705,8 @@ VulkanDevice::CopyOutcome VulkanDevice::CopyBuffer(std::uint64_t destination, st
     Graphics::StorageTexture::FlushPending(destination, bytes, nullptr, "buffer copy destination", Graphics::PublishScope::PartialUnits);
     // Looked up after the sync (its completions' write-backs can refresh the import table and
     // retire an import) and the flushes (which may import memory themselves).
-    const Graphics::HostImport* destinationImport = Graphics::HostImportFor(context, destination, bytes);
-    const Graphics::HostImport* sourceImport = destinationImport != nullptr ? Graphics::HostImportFor(context, source, bytes) : nullptr;
+    std::shared_ptr<Graphics::HostImport> destinationImport = Graphics::HostImportFor(context, destination, bytes);
+    std::shared_ptr<Graphics::HostImport> sourceImport = destinationImport != nullptr ? Graphics::HostImportFor(context, source, bytes) : nullptr;
     if (destinationImport == nullptr || sourceImport == nullptr) return outcome;
     using CommandClass = Graphics::Recorder::CommandClass;
     // A queued DCC key store over either range must land before the transfer reads or writes it.
@@ -1712,6 +1718,8 @@ VulkanDevice::CopyOutcome VulkanDevice::CopyBuffer(std::uint64_t destination, st
     VkAccessFlags covered = 0;
     const auto commands = recorder.Commands(&covered);
     // The class range covers the barriers too; the program-keyed range inside it is the transfer.
+    recorder.Keep(sourceImport);
+    recorder.Keep(destinationImport);
     const auto classTiming = recorder.BeginGpuTiming(CommandClass::Copy);
     if (Graphics::Recorder::BarrierValidate()) {
         const std::pair<std::uint64_t, std::uint64_t> read{source, source + bytes}, written{destination, destination + bytes};
@@ -2886,7 +2894,7 @@ struct RecordedDispatch {
     std::uint32_t x, y, z;
     // The DISPATCH_INDIRECT arguments (0: direct) and their host import when the GPU reads them.
     std::uint64_t arguments;
-    const Graphics::HostImport* argumentImport;
+    std::shared_ptr<Graphics::HostImport> argumentImport;
     std::uint64_t programAddress;
     // What decides the template's data refresh: None (the object was built by this call), Words
     // (a template hit: RefreshData compares every buffer's words), Hash (a recipe: the template's
@@ -3178,7 +3186,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     const auto& context = *record.context;
     auto& resources = *record.resources;
     const auto arguments = record.arguments;
-    const auto* argumentImport = record.argumentImport;
+    const auto argumentImport = record.argumentImport;
     // The record phase split (APS5_PROFILE_DRAW, rows "record: ..." of the [dispatch] totals):
     // opening the batch, the keeps, the data refresh, the barriers with the bind and the dispatch
     // itself, the pending-write notes and marks (MarkGpuWrites), and the completion registration.
@@ -3238,6 +3246,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
         recorder.NoteAccess(CommandClass::DispatchLeading, Graphics::Recorder::Access{reads, resources.GpuWrites(), images, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, resources.HoldsLease()});
     }
     if (argumentImport != nullptr) {
+        recorder.Keep(argumentImport);
         // The group counts were stored by earlier recorded work (a dispatch in place, a fill) or the
         // host; the indirect read follows all of it.
         const auto timing = recorder.BeginGpuTiming(CommandClass::IndirectArguments);

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -31,6 +31,8 @@ struct HostImport {
     // Identity for the life of this import (see HostImportSerial); 0 until first asked for.
     std::uint64_t serial = 0;
     bool unwatched = false;
+    bool shadowAllowed = true;
+    std::vector<std::weak_ptr<const GuestAllocations::Range>> ranges;
     bool dmaBuf = false;
 };
 
@@ -53,7 +55,8 @@ void SetImportWatch(const Context& context, ImportWatch watch);
 // The host import of the registered allocation containing [address, address + bytes), made on demand
 // (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in
 // the import's buffer.
-const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
+std::shared_ptr<HostImport> HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
+void ClearHostImports(VkDevice device);
 // Whether an existing import covers [address, address + bytes), without reconciling the imports
 // with the registry or making one (HostImportFor may take a registry lease): a hint for choices
 // made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken
@@ -262,7 +265,7 @@ class GuestBufferMemory {
         // Registered allocation that is imported: its bytes are read from live guest memory, not a snapshot.
         bool hostBacked = false;
         // Set when the region is served by an imported allocation; nothing is copied or written back.
-        const HostImport* direct = nullptr;
+        std::shared_ptr<HostImport> direct = nullptr;
         // Set when the region covers uncommitted pages: only the `backed` parts (possibly none) are guest
         // memory; the rest reads as zeros and is never stored.
         bool sparse = false;
@@ -285,6 +288,7 @@ class GuestBufferMemory {
         bool gpuCopy = false;
         VkBuffer copySource = VK_NULL_HANDLE;
         std::uint64_t copySourceBase = 0;
+        std::shared_ptr<HostImport> copyImport;
         bool copiedBack = false;
         // An element the shader updates atomically lies inside (AddWritable's `atomic`).
         bool atomic = false;

core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp

@@ -217,7 +217,7 @@ void TraceKeyStore(const char* path, std::uint64_t begin, std::size_t count) {
 // buffer of 0xff instead. Under GuestMemory::GpuMutex (it records). False when the keys cannot be
 // stored this way (memory the GPU has no view of): the caller stores them on the CPU.
 bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uint64_t begin, std::size_t count) {
-    const auto* import = HostImportFor(context, begin, count);
+    const auto import = HostImportFor(context, begin, count);
     if (import == nullptr) return false;
     const VkDeviceSize first = begin - import->base;
     const VkDeviceSize last = first + count;
@@ -238,6 +238,7 @@ bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uin
     // Queued on the open batch and recorded with the batch's other key stores as one run (or before
     // a later command writing the keys; see Recorder::QueueKeyStore).
     recorder.QueueKeyStore(import->buffer, first, last, std::move(seed), begin, begin + count);
+    if (ShadowVerify()) recorder.Keep(import);
     recorder.NotePendingFill(begin, count, 0xff);
     GuestMemory::MarkWritten(begin, count);
     // After the note: an unlocked memo lookup drops an entry the snapshot does not cover yet. The

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -999,8 +999,8 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat
 struct IndirectRecord {
     const Pm4::DrawParameters::IndirectDraw* args = nullptr;
     IndirectDrawPath path = IndirectDrawPath::Gpu;
-    const HostImport* argumentImport = nullptr;
-    const HostImport* countImport = nullptr;
+    std::shared_ptr<HostImport> argumentImport = nullptr;
+    std::shared_ptr<HostImport> countImport = nullptr;
     std::span<const Pm4::DrawArguments> records;
     double readMs = 0;
 };
@@ -1070,6 +1070,8 @@ bool recordIndirectArguments(const Context& context, VkCommandBuffer commands, R
     RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT);
     countBarrier(1);
     if (recorded) {
+        recorder->Keep(indirect.argumentImport);
+        if (indirect.countImport != nullptr) recorder->Keep(indirect.countImport);
         // Read in place from the import when the batch runs (a synchronous draw waits for its own).
         recorder->NotePendingRead(args->arguments, static_cast<std::size_t>(args->RangeBytes()), Recorder::ReadKind::Indirect);
         if (args->countIndirect) recorder->NotePendingRead(args->countAddress, 4, Recorder::ReadKind::Indirect);
@@ -1282,7 +1284,7 @@ void captureInputs(const Context& context, Recorder& recorder, VkCommandBuffer c
             CaptureTrace::Log("input-skip draw=%llu batch=%llu address=%llx bytes=%zu reason=gpu-writer", draw, batch, static_cast<unsigned long long>(begin), bytes);
             continue;
         }
-        const auto* imported = HostImportFor(context, begin, bytes);
+        const auto imported = HostImportFor(context, begin, bytes);
         Require(imported != nullptr, "capture input has no host import");
         addSample(begin, bytes, imported->buffer, begin - imported->base, reinterpret_cast<const std::byte*>(begin));
     }
@@ -1624,7 +1626,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
     std::vector<Pm4::DrawArguments> records;
     if (args != nullptr) {
         static const bool gpuIndirectDraws = std::getenv("APS5_NO_GPU_INDIRECT_DRAW") == nullptr;
-        const auto decide = [&](std::uint64_t address, std::size_t bytes, const HostImport*& import) {
+        const auto decide = [&](std::uint64_t address, std::size_t bytes, std::shared_ptr<HostImport>& import) {
             if (StorageTexture::FlushPending(address, bytes, nullptr, "indirect draw arguments")) {
                 if (recorder != nullptr) {
                     Recorder::CountSync(2);