perf(agc): answer Refresh's registry and host-import lookups from an index and a cache

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -410,6 +410,17 @@ if(BUILD_TESTING)
         set_tests_properties(agc_handoff_spin PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+
+    add_test_executable(agc_range_index_tests tests/RangeIndex.cpp)
+    target_include_directories(agc_range_index_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_range_index_tests PRIVATE libc)
+    configure_windows_unwind(agc_range_index_tests)
+    add_test(NAME agc_range_index COMMAND agc_range_index_tests)
+    set_tests_properties(agc_range_index PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_range_index PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
 endif()
 
 if(WIN32)

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -72,6 +72,14 @@ void SetImportWatch(const Context& context, ImportWatch watch);
 // address)` in the import's buffer.
 const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
 const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
+struct HostImportLookupCache {
+    const HostImport* import = nullptr;
+    std::uint64_t address = 0;
+    std::uint64_t bytes = 0;
+    std::uint64_t epoch = 0;
+    std::uint64_t inserts = 0;
+};
+const HostImport* HostImportForCached(const Context& context, std::uint64_t address, std::size_t bytes, HostImportLookupCache& cache);
 void SetHostImportWindow(std::uint64_t bytes);
 // Tests: the window of one large registered range (64 MiB) and the import budget (0: the configured one).
 void SetHostImportRangeWindow(std::uint64_t bytes);

core/libs/prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp

@@ -0,0 +1,93 @@
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <vector>
+
+namespace AgcDriver {
+
+template <class Item>
+class RangeIndex {
+public:
+    void Insert(Item item, std::uint64_t begin, std::uint64_t end) {
+        const auto at = std::upper_bound(entries.begin(), entries.end(), begin, [](std::uint64_t value, const Entry& entry) { return value < entry.begin; });
+        entries.insert(at, Entry{begin, end, item});
+        stale = true;
+    }
+
+    std::size_t Remove(Item item, std::uint64_t begin, bool all = false) {
+        auto first = std::lower_bound(entries.begin(), entries.end(), begin, [](const Entry& entry, std::uint64_t value) { return entry.begin < value; });
+        std::size_t removed = 0;
+        while (first != entries.end() && first->begin == begin) {
+            if (first->item == item) {
+                first = entries.erase(first);
+                ++removed;
+                if (!all) break;
+            } else {
+                ++first;
+            }
+        }
+        if (removed != 0) stale = true;
+        return removed;
+    }
+
+    template <class Visit>
+    bool Overlapping(std::uint64_t address, std::uint64_t bytes, Visit&& visit) {
+        if (entries.empty() || bytes == 0) return false;
+        if (stale) rebuild();
+        const auto end = bytes > std::numeric_limits<std::uint64_t>::max() - address ? std::numeric_limits<std::uint64_t>::max() : address + bytes;
+        return walk(0, entries.size(), address, end, visit);
+    }
+
+    template <class Visit>
+    bool StartingAt(std::uint64_t begin, Visit&& visit) const {
+        auto it = std::lower_bound(entries.begin(), entries.end(), begin, [](const Entry& entry, std::uint64_t value) { return entry.begin < value; });
+        for (; it != entries.end() && it->begin == begin; ++it) {
+            if (visit(it->item)) return true;
+        }
+        return false;
+    }
+
+    std::size_t Size() const { return entries.size(); }
+
+private:
+    struct Entry {
+        std::uint64_t begin;
+        std::uint64_t end;
+        Item item;
+    };
+
+    std::uint64_t build(std::size_t from, std::size_t to) {
+        if (from >= to) return 0;
+        const auto mid = from + (to - from) / 2;
+        const auto left = build(from, mid);
+        const auto right = build(mid + 1, to);
+        reach[mid] = std::max({entries[mid].end, left, right});
+        return reach[mid];
+    }
+
+    void rebuild() {
+        reach.assign(entries.size(), 0);
+        build(0, entries.size());
+        stale = false;
+    }
+
+    template <class Visit>
+    bool walk(std::size_t from, std::size_t to, std::uint64_t address, std::uint64_t end, Visit& visit) const {
+        if (from >= to) return false;
+        const auto mid = from + (to - from) / 2;
+        if (reach[mid] <= address) return false;
+        if (walk(from, mid, address, end, visit)) return true;
+        if (entries[mid].begin >= end) return false;
+        if (address < entries[mid].end && visit(entries[mid].item)) return true;
+        return walk(mid + 1, to, address, end, visit);
+    }
+
+    std::vector<Entry> entries;
+    std::vector<std::uint64_t> reach;
+    bool stale = false;
+};
+
+}

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -3,6 +3,7 @@
 
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
@@ -571,6 +572,7 @@ class StorageTexture : public std::enable_shared_from_this<StorageTexture> {
     std::weak_ptr<StorageTexture> borrowedFrom;
     std::uint64_t borrowedVersion = 0;
     std::vector<bool> borrowedUnits;
+    HostImportLookupCache importLookup;
     // Units of this image were borrowed since it was last written: the borrower's proofs
     // (the resource cache's serial memo) hold only while this image's version does, so the next
     // write here bumps the pending serial although the image was pending already.

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -218,6 +218,7 @@ struct HostImports {
     // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be
     // reused while it is unchanged.
     std::uint64_t epoch = 1;
+    std::uint64_t inserts = 0;
     VkDevice watchDevice = VK_NULL_HANDLE;
     bool unwatchImports = false;
     bool unwatchDmaBufImports = false;
@@ -1616,6 +1617,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
         }
     }
 #endif
+    ++state.inserts;
     const auto inserted = state.imports.emplace(base, entry);
     owner.armed = false;
     return &inserted.first->second;
@@ -2999,14 +3001,7 @@ std::uint64_t HostImportsSettled() {
     return settledImports.load(std::memory_order_relaxed);
 }
 
-const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) {
-    if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return nullptr;
-    ensurePinWaiter();
-    auto& state = Imports();
-    std::lock_guard lock(state.mutex);
-#ifdef _WIN32
-    settleUnwatchedImports(context, state);
-#endif
+static const HostImport* hostImportSettled(const Context& context, HostImports& state, std::uint64_t address, std::size_t bytes) {
     // A hit is only valid while the registry has not changed since the imports were reconciled.
     if (!importsStale(context, state)) {
         if (const auto* entry = findImport(state, address, address + bytes)) {
@@ -3019,6 +3014,36 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s
     return importAccess(context, state, address, address + bytes, lease);
 }
 
+const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) {
+    if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return nullptr;
+    ensurePinWaiter();
+    auto& state = Imports();
+    std::lock_guard lock(state.mutex);
+#ifdef _WIN32
+    settleUnwatchedImports(context, state);
+#endif
+    return hostImportSettled(context, state, address, bytes);
+}
+
+const HostImport* HostImportForCached(const Context& context, std::uint64_t address, std::size_t bytes, HostImportLookupCache& cache) {
+    if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return nullptr;
+    ensurePinWaiter();
+    auto& state = Imports();
+    std::lock_guard lock(state.mutex);
+#ifdef _WIN32
+    settleUnwatchedImports(context, state);
+#endif
+    if (cache.import != nullptr && cache.address == address && cache.bytes == bytes && cache.epoch == state.epoch && cache.inserts == state.inserts && !importsStale(context, state)) {
+        cache.import->lastUse = ++state.useCounter;
+        cache.import->lastUse = ++state.useCounter;
+        return cache.import;
+    }
+    const auto* entry = hostImportSettled(context, state, address, bytes);
+    if (entry != nullptr && !importsStale(context, state)) cache = {entry, address, bytes, state.epoch, state.inserts};
+    else cache = {};
+    return entry;
+}
+
 const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) {
     if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return nullptr;
     auto& state = Imports();

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -7,6 +7,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 // For the declaration of PendingStorageOverlaps, defined below.
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
@@ -1249,6 +1250,11 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
 
 namespace {
 
+bool RefreshLookupsEnabled() {
+    static const bool enabled = std::getenv("APS5_NO_REFRESH_LOOKUP_CACHE") == nullptr;
+    return enabled;
+}
+
 class PendingList {
 public:
     using iterator = std::vector<StorageTexture*>::iterator;
@@ -1257,14 +1263,23 @@ class PendingList {
     void push_back(StorageTexture* texture) {
         textures.push_back(texture);
         ++version;
+        if (RefreshLookupsEnabled()) index.Insert(texture, texture->Descriptor().baseAddress, texture->Descriptor().baseAddress + texture->GuestBytes());
     }
     iterator erase(iterator it) {
         ++version;
+        if (RefreshLookupsEnabled()) index.Remove(*it, (*it)->Descriptor().baseAddress);
         return textures.erase(it);
     }
     void remove(const StorageTexture* texture) {
-        if (std::erase(textures, texture) != 0) ++version;
+        if (std::erase(textures, texture) != 0) {
+            ++version;
+            if (RefreshLookupsEnabled()) index.Remove(const_cast<StorageTexture*>(texture), texture->Descriptor().baseAddress, true);
+        }
     }
+    template <class Visit>
+    bool Overlapping(std::uint64_t address, std::uint64_t bytes, Visit&& visit) { return index.Overlapping(address, bytes, visit); }
+    template <class Visit>
+    bool StartingAt(std::uint64_t address, Visit&& visit) const { return index.StartingAt(address, visit); }
     bool MayOverlap(std::uint64_t address, std::size_t bytes) {
         if (textures.empty() || bytes == 0) return false;
         if (indexed != version) rebuild();
@@ -1287,6 +1302,7 @@ class PendingList {
         indexed = version;
     }
     std::vector<StorageTexture*> textures;
+    RangeIndex<StorageTexture*> index;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
     std::uint64_t version = 0;
     std::uint64_t indexed = ~std::uint64_t{0};
@@ -1866,7 +1882,7 @@ bool StorageTexture::Refresh() {
     // since `generation` need no comparison.
     const auto current = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast<std::size_t>(guestBytes));
     // Per tracked layer: changed since its generation (a keys change makes every layer stale).
-    std::vector<bool> changed(trackedLayers, false);
+    std::vector<bool> changed;
     // Per 64 KiB block of the surface (blockGenerations): stamped since its layer's generation, and
     // by a CPU store; scanned once when anything changed.
     std::vector<std::uint64_t> generations;
@@ -1906,14 +1922,17 @@ bool StorageTexture::Refresh() {
         const auto keysStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
         keys = ProvedKeys();
         if (profile && descriptor.dccAddress != 0) LookupOutcomes::Add(LookupOutcomes::DccScan, keysStart);
+        const bool quickCandidate = RefreshLookupsEnabled() && alias == nullptr && keys == uploadedKeys && blockUnits;
+        const bool quick = quickCandidate && GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), generation);
+        if (!quick) changed.assign(trackedLayers, false);
         if (keys != uploadedKeys) {
             changed.assign(trackedLayers, true);
             unchanged = false;
             keysChanged = true;
             // Which units the CPU stamped still decides what the drop rule below may drop.
             if (blockUnits) scanBlocks();
         } else if (blockUnits) {
-            if (!GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), generation)) {
+            if (quickCandidate ? !quick : !GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), generation)) {
                 // One stamp scan for every unit (each is one stamp block); the unchanged units move
                 // to the current generation, so the whole-surface check answers next time.
                 scanBlocks();
@@ -1952,7 +1971,7 @@ bool StorageTexture::Refresh() {
         // the same bytes read differently, and a changed unit holding pending results must take
         // the drop rule below (the image holds the results, `original` what the CPU wrote back).
         bool pendingChanged = false;
-        for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
+        for (std::uint32_t layer = 0; !quick && layer < trackedLayers; ++layer) {
             if (changed[layer] && layerPending[layer]) pendingChanged = true;
         }
         if (!unchanged && !keysChanged && !pendingChanged && originalHolds(descriptor.baseAddress, static_cast<std::size_t>(guestBytes)) && equalsOriginal()) {
@@ -1961,7 +1980,7 @@ bool StorageTexture::Refresh() {
         }
         // Only the direct path uploads the selected layers alone (see upload); the others replace
         // the whole image, so every pending layer's results are stored first.
-        direct = keys == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes)) != nullptr;
+        direct = keys == DccKeys::Uncompressed && (RefreshLookupsEnabled() ? HostImportForCached(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes), importLookup) : HostImportFor(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) != nullptr;
         if (alias != nullptr && direct) {
             // The alias's pending units that nothing stamped since its generation (a CPU store or a
             // driver store there wins over its results, as at its own write-back) are taken from
@@ -2109,7 +2128,8 @@ bool StorageTexture::otherPendingOverlaps() const {
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
     const auto other = [&](const StorageTexture* texture) { return texture != this && texture->overlaps(descriptor.baseAddress, static_cast<std::size_t>(guestBytes)); };
-    return std::any_of(pending.textures.begin(), pending.textures.end(), other) || std::any_of(pending.flushing.begin(), pending.flushing.end(), other);
+    const bool pendingOverlap = RefreshLookupsEnabled() ? pending.textures.Overlapping(descriptor.baseAddress, guestBytes, [&](StorageTexture* texture) { return texture != this; }) : std::any_of(pending.textures.begin(), pending.textures.end(), other);
+    return pendingOverlap || std::any_of(pending.flushing.begin(), pending.flushing.end(), other);
 }
 
 bool StorageTexture::refreshProved() {
@@ -4486,9 +4506,17 @@ std::shared_ptr<StorageTexture> StorageTexture::pendingAlias() const {
     {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
-        for (auto* texture : pending.textures) {
-            if (texture == this || texture->descriptor.baseAddress != descriptor.baseAddress || texture->guestBytes != guestBytes || !texture->blockUnits) continue;
+        const auto consider = [&](StorageTexture* texture) {
+            if (texture == this || texture->descriptor.baseAddress != descriptor.baseAddress || texture->guestBytes != guestBytes || !texture->blockUnits) return;
             if (auto alive = texture->weak_from_this().lock()) candidates.push_back(std::move(alive));
+        };
+        if (RefreshLookupsEnabled()) {
+            pending.textures.StartingAt(descriptor.baseAddress, [&](StorageTexture* texture) {
+                consider(texture);
+                return false;
+            });
+        } else {
+            for (auto* texture : pending.textures) consider(texture);
         }
     }
     for (const auto& candidate : candidates) {