diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index b55e00f59a6..0e7a3d5e2ee 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1116,6 +1116,17 @@ if(BUILD_TESTING) set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + foreach(mode IN ITEMS map ring shared uncached) + add_test(NAME agc_collect_mapping_history_${mode} COMMAND agc_driver_recorder_tests --collect-mapping-history-only) + set_tests_properties(agc_collect_mapping_history_${mode} PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_collect_mapping_history_${mode} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() + set_tests_properties(agc_collect_mapping_history_ring PROPERTIES ENVIRONMENT "APS5_NO_EPOCH_COLLECT_MAP=1") + set_tests_properties(agc_collect_mapping_history_shared PROPERTIES ENVIRONMENT "APS5_SHARED_COLLECT_MEMO=1") + set_tests_properties(agc_collect_mapping_history_uncached PROPERTIES ENVIRONMENT "APS5_NO_COLLECT_MEMO=1") add_test(NAME agc_storage_kept_unit_settle COMMAND agc_driver_recorder_tests --kept-unit-settle-only) set_tests_properties(agc_storage_kept_unit_settle PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) if(WIN32) @@ -1870,6 +1881,13 @@ if(BUILD_TESTING) endif() endforeach() endforeach() + add_test(NAME agc_driver_array_layer_target_cpu COMMAND agc_driver_array_layer_target_tests) + set_tests_properties(agc_driver_array_layer_target_cpu PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_HOST_IMPORT=1" TIMEOUT 30 SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_driver_array_layer_target_cpu PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync lazy lazy_sync) set(preparedTest "agc_driver_prepared_shaders_${preparedMode}") if(preparedMode STREQUAL "indirect") diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index d0859ab0610..7088ff6aae3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -797,6 +797,7 @@ struct WriteTracker { std::uintptr_t end = 0; std::uint64_t epoch = 0; std::uint64_t unwatched = 0; + std::uint64_t mapping = 0; }; std::array memo{}; std::size_t nextMemo = 0; @@ -1184,19 +1185,20 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi // must not differ between them. const auto epoch = currentCollectEpoch(); const auto unwatched = unwatchSerial.load(std::memory_order_acquire); + const auto mapping = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); const bool useMemo = memoized && collectMemoEnabled() && bytes != 0; if (useMemo && !sharedCollectMemo()) { // An entry exists only for a completed walk of an in-arena range, so the tracker is // initialized and watched; nothing below the lock needs asking. if (epochCollectMapEnabled()) { const auto& map = epochCollectMap(epoch); - if (const auto found = map.ranges.find(first); found != map.ranges.end() && found->second.unwatched == unwatched && stop <= found->second.end) { + if (const auto found = map.ranges.find(first); found != map.ranges.end() && found->second.unwatched == unwatched && found->second.mapping == mapping && stop <= found->second.end) { collectMemoHits.fetch_add(1, std::memory_order_relaxed); return tracker.generation.load(std::memory_order_relaxed); } } for (const auto& entry : threadCollectMemo.entries) { - if (entry.epoch == epoch && entry.unwatched == unwatched && entry.begin <= first && stop <= entry.end) { + if (entry.epoch == epoch && entry.unwatched == unwatched && entry.mapping == mapping && entry.begin <= first && stop <= entry.end) { // The current generation, not the memoized one: blocks stamped since (MarkWritten, an // overlapping collect) were written before this caller reads, and an older value would // make every later UnchangedSince fail until the epoch ends. @@ -1211,7 +1213,7 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi auto cursor = first; if (useMemo && sharedCollectMemo()) { for (const auto& entry : tracker.memo) { - if (entry.epoch == epoch && entry.unwatched == unwatchSerial.load(std::memory_order_relaxed) && entry.begin <= cursor && stop <= entry.end) { + if (entry.epoch == epoch && entry.unwatched == unwatchSerial.load(std::memory_order_relaxed) && entry.mapping == mapping && entry.begin <= cursor && stop <= entry.end) { collectMemoHits.fetch_add(1, std::memory_order_relaxed); return tracker.generation; } @@ -1232,12 +1234,12 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi // Only a completed walk is remembered; a failed one (uncommitted pages) returned 0 above. if (collectMemoEnabled()) { const auto serial = unwatchSerial.load(std::memory_order_relaxed); - if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch, serial}; + if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch, serial, mapping}; else { - threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial}; + threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial, mapping}; if (epochCollectMapEnabled()) { auto& entry = epochCollectMap(epoch).ranges[first]; - if (entry.unwatched != serial || entry.end < stop) entry = {first, stop, epoch, serial}; + if (entry.unwatched != serial || entry.mapping != mapping || entry.end < stop) entry = {first, stop, epoch, serial, mapping}; } } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 62afeaea77b..582c5b48095 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -3038,8 +3038,9 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { // walk: the collect memo is per worker packet, so on this thread a memoized answer could // miss a CPU write that landed after a worker's walk of the same range. GuestMemory::CollectWritesUncached(buffer.address, bytes); - if (!GuestMemory::UnchangedSince(buffer.address, bytes, resident->Generation()) || (buffer.dccAddress != 0 && ResidentKeysMoved(*resident))) { - resident->Refresh(); + if (!GuestMemory::UnchangedSince(buffer.address, bytes, resident->GenerationFor(buffer.address, bytes)) || (buffer.dccAddress != 0 && ResidentKeysMoved(*resident))) { + if (resident->ImageLayers() > 1) resident->RefreshLayers(layer, 1); + else resident->Refresh(); ++refreshedPresents; } ++residentPresents; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 89a05894c19..bcf1d6632e2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -46,6 +46,7 @@ bool CompletePendingCpuWrites(const Context& context, std::uint64_t address, std // The cached storage image of a surface (render targets use it as their resident image); brought up // to date with guest memory before it is returned. std::shared_ptr CachedStorageSurface(const Context& context, const GuestTextureResource& resource); +std::shared_ptr CachedStorageLayers(const Context& context, const GuestTextureResource& resource, std::uint32_t firstLayer, std::uint32_t layerCount); // Whether `image` is still the storage cache's image of its surface (what CachedStorageSurface // would return); a true answer counts as a use for the cache's eviction order, as a lookup would. bool StorageImageCached(const Context& context, const StorageTexture* image); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index dfe074f738d..27ef571bfb4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -332,6 +332,7 @@ class StorageTexture : public std::enable_shared_from_this { // validates textures copied from this image. std::uint64_t Version() const { return version; } std::uint64_t Generation() const { return generation; } + std::uint64_t GenerationFor(std::uint64_t address, std::uint64_t bytes) const; // The DCC keys the content was uploaded under and the last scan of the surface's keys // (ProvedClearKeys): Refresh's own key rule for the fast revalidation. Under // GuestMemory::GpuMutex only, as Refresh is; never from a build's stage A. @@ -346,6 +347,7 @@ class StorageTexture : public std::enable_shared_from_this { // was still current (nothing uploaded). // Keeps the image current with guest memory (see GuestMemory::CollectWrites). bool Refresh(); + bool RefreshLayers(std::uint32_t first, std::uint32_t count); static std::uint64_t SinglePassMoves(); static std::uint64_t RefreshesProved(); static std::uint64_t UntrackedCompares(); @@ -357,9 +359,8 @@ class StorageTexture : public std::enable_shared_from_this { StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel); // The regions of every array layer, or of the tracked layers `layers` selects. std::vector CopyRegions(const std::vector* layers = nullptr) const; - // Uploads the surface, or only the tracked layers `layers` selects (the direct path; the others - // upload everything). void upload(const std::vector* layers = nullptr); + std::vector cpuUploadLayers(const std::vector& units) const; void captureGuestBytes(const std::vector* layers); bool originalHolds(std::uint64_t address, std::size_t bytes) const; bool compareUntracked(std::uint64_t address, std::size_t bytes, std::span changed, bool memoize = false) const; @@ -454,7 +455,10 @@ class StorageTexture : public std::enable_shared_from_this { // layer generation (the whole-surface fast check), `dirty` whether any layer is pending. std::uint64_t layerBegin(std::uint32_t layer) const { return descriptor.baseAddress + static_cast(layer) * trackedLayerBytes; } bool anyLayerPending() const; + bool refresh(std::uint32_t firstLayer, std::uint32_t lastLayer); + void noteLayersBehind(std::uint32_t firstLayer, std::uint32_t lastLayer, const std::vector& outside); void refreshGeneration(); + std::uint64_t collectedLayerGeneration(std::uint32_t layer, std::uint64_t whole, bool uncached = false) const; bool refreshProved(); void takeRefreshProof(bool aliased); bool otherPendingOverlaps() const; @@ -530,7 +534,13 @@ class StorageTexture : public std::enable_shared_from_this { std::uint64_t sliceLinearBytes = 0; SurfaceGeometry geometry; std::vector original; - mutable std::array comparedGuestBytes{}; + struct ComparedRange { + std::uint64_t address = 0; + std::size_t bytes = 0; + std::array stamp{}; + }; + mutable std::array comparedRanges{}; + mutable std::uint32_t nextComparedRange = 0; std::vector generationBaseline; // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. DccKeys uploadedKeys = DccKeys::Uncompressed; @@ -556,6 +566,7 @@ class StorageTexture : public std::enable_shared_from_this { std::uint64_t trackedLayerBytes = 0; std::vector layerGeneration; std::vector layerPending; + std::vector layerBehind; bool blockUnits = false; // Write-back hysteresis (block units, see writeBack): partial stores in the current window of // presents, its first present, and the present until which every pending unit is stored. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 5993201d607..3f2c3510d5a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -2030,7 +2030,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& Require(color.dccAddress == 0 && color.cmaskAddress == 0 && AttachmentProxyFormat(context, color.format) == VK_FORMAT_UNDEFINED, "a slice of a 2D array color target with DCC, CMASK or an attachment proxy has no resident array image"); const auto array = ArraySurfaceForTarget(color); Require(color.surfaceAddress == color.arrayAddress + color.arraySlice * DescribeSurface(array).layerBytes, "the array image's layer pitch differs from the color view's slice offset"); - auto resident = CachedStorageSurface(context, array); + auto resident = CachedStorageLayers(context, array, color.arraySlice, 1); Require(resident->Attachable(), "storage format cannot be a color attachment"); return resident; } @@ -2623,7 +2623,8 @@ DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, con auto stored = recipe.targets[index].lock(); if (stored == nullptr || !StorageImageCached(context, stored.get()) || !StorageImageServesKeys(*stored, state.colors[index].dccAddress)) return miss(DrawRecipeMiss::TargetGone); auto resident = refreshResidentTarget(context, state, state.colors[index], outcome, profile, [&] { - stored->Refresh(); + if (state.colors[index].arrayLastSlice != 0) stored->RefreshLayers(state.colors[index].arraySlice, 1); + else stored->Refresh(); return stored; }); timer.phase(PhaseReadTarget); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 44e75aeaeb1..865082aab96 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -396,7 +396,12 @@ void logLookup(const LookupRecord& record) { log.push_back(record); } -std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes = 0, bool written = false); +void refreshForView(StorageTexture& source, const GuestTextureResource& resource) { + if (resource.dimension == TextureDimension::k2D && source.ImageLayers() > 1) source.RefreshLayers(resource.baseArray, 1); + else source.Refresh(); +} + +std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes = 0, bool written = false, std::optional> layers = std::nullopt); std::uint64_t heldBytes(const StorageTexture& image) { return std::max(image.AllocationBytes(), image.GuestBytes()); @@ -661,9 +666,9 @@ std::shared_ptr cachedTexture(const Context& context, std::spanGeneration())) { + if (source != nullptr && !GuestMemory::UnchangedSince(address, bytes, source->GenerationFor(address, bytes))) { // The CPU wrote the memory while results were pending: Refresh merges them the usual way. - source->Refresh(); + refreshForView(*source, resource); } // Results stored to guest memory just now are stamped newer than `generation`; a snapshot read // after them is current at the generation of a second (memoized) collect. The store marks the @@ -720,7 +725,7 @@ std::shared_ptr cachedTexture(const Context& context, std::spansource) && (source != nullptr || *keys == DccKeys::Uncompressed) && StorageImageServesKeys(*it->source, resource.dccAddress)) { - if (!GuestMemory::UnchangedSince(address, bytes, it->source->Generation())) it->source->Refresh(); + if (!GuestMemory::UnchangedSince(address, bytes, it->source->GenerationFor(address, bytes))) refreshForView(*it->source, resource); if (it->texture->IsDepthCopy()) it->texture->RefreshDepthCopy(); it->keys = *keys; touchTexture(cache, it); @@ -934,7 +939,7 @@ GuestTextureResource StorageSurface(const Context& context, const GuestTextureRe } // `guestBytes` is the surface size when the caller described the surface already (0: described here). -std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& viewed, std::uint32_t mip, std::uint64_t guestBytes, bool written) { +std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& viewed, std::uint32_t mip, std::uint64_t guestBytes, bool written, std::optional> layers) { Require(viewed.dimension != TextureDimension::k2DMsaa, "storage image access to a multisampled texture is not implemented"); RequireNoMultisampleTarget(viewed.baseAddress, guestBytes != 0 ? static_cast(guestBytes) : 1u, "a storage image access"); if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth; @@ -964,7 +969,8 @@ std::shared_ptr cachedStorageTexture(const Context& context, std it = cache.entries.end(); } if (it != cache.entries.end()) { - it->texture->Refresh(); + if (layers) it->texture->RefreshLayers(layers->first, layers->second); + else it->texture->Refresh(); cache.entries.splice(cache.entries.begin(), cache.entries, it); texture = it->texture; counters.storageHits.fetch_add(1, std::memory_order_relaxed); @@ -1139,6 +1145,10 @@ std::shared_ptr CachedStorageSurface(const Context& context, con return cachedStorageTexture(context, {}, resource, 0); } +std::shared_ptr CachedStorageLayers(const Context& context, const GuestTextureResource& resource, std::uint32_t firstLayer, std::uint32_t layerCount) { + return cachedStorageTexture(context, {}, resource, 0, 0, false, std::pair{firstLayer, layerCount}); +} + bool StorageImageServesKeys(const StorageTexture& image, std::uint64_t dccAddress) { GuestTextureResource resource{}; resource.dccAddress = dccAddress; @@ -2593,7 +2603,7 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::spanCached()) return fail(FastFail::Evicted); - queries.push_back({address, bytes, source->Generation()}); + queries.push_back({address, bytes, source->GenerationFor(address, bytes)}); images.push_back(source); } else { queries.push_back({address, bytes, surface.generation}); @@ -2617,7 +2627,7 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::spanCached()) return fail(FastFail::Evicted); - queries.push_back({address, bytes, image->Generation()}); + queries.push_back({address, bytes, image->GenerationFor(address, bytes)}); images.push_back(image); } if (!pending.empty()) { @@ -2683,7 +2693,7 @@ bool ShaderResources::fastRevalidateEach() { if (surface.source != nullptr) { // The view follows the image: it needs the image current with guest memory, as the // lookup's Refresh would make it. - if (!StorageImageCached(context, surface.source) || !GuestMemory::UnchangedSince(address, bytes, surface.source->Generation())) return false; + if (!StorageImageCached(context, surface.source) || !GuestMemory::UnchangedSince(address, bytes, surface.source->GenerationFor(address, bytes))) return false; } else if (!GuestMemory::UnchangedSince(address, bytes, surface.generation)) { return false; } @@ -2713,7 +2723,7 @@ bool ShaderResources::fastRevalidateEach() { if (GuestMemory::CollectWrites(address, bytes) == 0) return false; if (PendingStorageOverlaps(address, bytes, image)) return false; if (!StorageImageCached(context, image)) return false; - if (!GuestMemory::UnchangedSince(address, bytes, image->Generation())) return false; + if (!GuestMemory::UnchangedSince(address, bytes, image->GenerationFor(address, bytes))) return false; } // Every check passed: snapshot textures are current at the collects issued before them. Storage // images keep their own generation: nothing was stamped above it, so the blocks a later write @@ -2747,9 +2757,9 @@ ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span if (found != nullptr && !Texture::CanCopyFrom(*found, surface.resource)) found.reset(); if (found != nullptr) { if (found != source) return OwnRefreshFallback::ForeignView; - if (!GuestMemory::UnchangedSince(address, bytes, source->Generation())) { + if (!GuestMemory::UnchangedSince(address, bytes, source->GenerationFor(address, bytes))) { countOwnRefresh(false); - source->Refresh(); + refreshForView(*source, surface.resource); } return OwnRefreshFallback::Count; } @@ -3960,7 +3970,7 @@ std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) // keys and stays valid while they are unchanged); a change is the full lookup's to judge. if (keys != record.entryKeys) return nullptr; if (record.source != nullptr) { - if (!StorageImageCached(context, record.source.get()) || !GuestMemory::UnchangedSince(address, bytes, record.source->Generation())) return nullptr; + if (!StorageImageCached(context, record.source.get()) || !GuestMemory::UnchangedSince(address, bytes, record.source->GenerationFor(address, bytes))) return nullptr; if (record.resource.dccAddress != 0 && IsDccClear(record.source->FilledKeys())) return nullptr; // Under fast-clear keys the view holds only while its image's results are still pending over // the surface; flushed, the clear the image cannot see makes the lookup take a snapshot. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index ec3a8317f30..58ab754e1b4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1727,11 +1727,31 @@ LookupOutcomes& ThreadLookupOutcomes() { } bool StorageTexture::Refresh() { + return refresh(0, trackedLayers); +} + +bool StorageTexture::RefreshLayers(std::uint32_t first, std::uint32_t count) { + if (count == 0 || geometry.thick || geometry.imageLayers <= 1 || first >= geometry.imageLayers || count > geometry.imageLayers - first || geometry.layerBytes == 0 || trackedLayerBytes == 0 || descriptor.dccAddress != 0) return Refresh(); + const auto begin = geometry.GuestLayerOffset(first); + const auto end = geometry.GuestLayerOffset(first + count - 1) + geometry.layerBytes; + const auto firstLayer = static_cast(begin / trackedLayerBytes); + const auto lastLayer = static_cast(std::min((end + trackedLayerBytes - 1) / trackedLayerBytes, trackedLayers)); + if (firstLayer >= lastLayer || (firstLayer == 0 && lastLayer == trackedLayers)) return Refresh(); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layerPending[layer] && (layer < firstLayer || layer >= lastLayer)) return Refresh(); + } + if (pendingAlias() != nullptr) return Refresh(); + return refresh(firstLayer, lastLayer); +} + +bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) { const bool profile = LookupOutcomes::Profiled(); auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const bool scoped = firstLayer != 0 || lastLayer != trackedLayers; CompletePendingCpuWrites(context, descriptor.baseAddress, static_cast(guestBytes)); if (descriptor.dccAddress != 0) CompletePendingCpuWrites(context, descriptor.dccAddress, DccKeyBytes(guestBytes)); if (refreshProved()) { + layerBehind.clear(); NoteProved(); refreshesProved.fetch_add(1, std::memory_order_relaxed); if (profile) LookupOutcomes::Add(LookupOutcomes::RefreshProved, start); @@ -1753,7 +1773,7 @@ bool StorageTexture::Refresh() { // generation, no results of this image's own there) or dead to its own write-back (tracked and // stamped); an untracked unit is stored whole by that write-back, and a unit pending in both // images takes the old order (store, then this image re-uploads it). - auto alias = pendingAlias(); + auto alias = scoped ? std::shared_ptr{} : pendingAlias(); if (alias != nullptr && !(ProvedClearKeys(descriptor, guestBytes, keyProof) == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr)) alias = nullptr; if (alias != nullptr) { std::vector aliasStamped(trackedLayers); @@ -1766,19 +1786,32 @@ bool StorageTexture::Refresh() { } } } + const auto scopeBegin = scoped ? layerBegin(firstLayer) : descriptor.baseAddress; + const auto scopeBytes = scoped ? static_cast(layerBegin(lastLayer - 1) + layerBytes(lastLayer - 1) - scopeBegin) : static_cast(guestBytes); // No publish: the upload below reads the stored units from the unit shadow itself. - const bool flushed = FlushPending(descriptor.baseAddress, static_cast(guestBytes), alias.get(), "storage refresh", PublishScope::None); + const bool flushed = FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None); if (profile && flushed) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start); // `original` holds the guest bytes the image was last uploaded from or written back as; while the // guest memory and the DCC keys still match, the image content is current. Pages nobody wrote // since `generation` need no comparison. const auto current = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + const auto scopedCurrent = current != 0 || !scoped ? current : GuestMemory::CollectWrites(scopeBegin, scopeBytes); + std::vector collected; + if (scopedCurrent == 0) { + collected.resize(trackedLayers); + for (auto layer = firstLayer; layer < lastLayer; ++layer) collected[layer] = collectedLayerGeneration(layer, 0); + } + const auto currentFor = [&](std::uint32_t layer) { + if (scoped && (layer < firstLayer || layer >= lastLayer)) return current; + return scopedCurrent != 0 ? scopedCurrent : collected[layer]; + }; // Per tracked layer: changed since its generation (a keys change makes every layer stale). std::vector changed(trackedLayers, false); // Per 64 KiB block of the surface (blockGenerations): stamped since its layer's generation, and // by a CPU store; scanned once when anything changed. std::vector generations; std::vector stampedBlocks, cpuBlocks; + std::vector outside; // Units taken from the alias's image instead of guest memory. std::vector borrowed; bool anyBorrowed = false; @@ -1794,6 +1827,7 @@ bool StorageTexture::Refresh() { const auto equalsOriginal = [&] { // Named for the [hooksync] attribution: the compare goes through the flush hook. const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); + if (scoped) return GuestMemory::EqualsCommitted(scopeBegin, std::span(original).subspan(static_cast(scopeBegin - descriptor.baseAddress), scopeBytes)); return GuestMemory::EqualsCommitted(descriptor.baseAddress, original); }; const auto scanBlocks = [&] { @@ -1802,13 +1836,19 @@ bool StorageTexture::Refresh() { cpuBlocks.assign(generations.size(), 0); tracked = GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), generations, stampedBlocks, cpuBlocks); if (!tracked) awaitStoredSave(nullptr); - compared = !tracked && compareUntracked(descriptor.baseAddress, static_cast(guestBytes), stampedBlocks, true); + const auto firstBlock = static_cast(scopeBegin / 65536 - descriptor.baseAddress / 65536); + const auto blocks = static_cast((scopeBegin + scopeBytes - 1) / 65536 - scopeBegin / 65536 + 1); + compared = !tracked && (scoped ? compareUntracked(scopeBegin, scopeBytes, std::span(stampedBlocks).subspan(firstBlock, blocks), true) : compareUntracked(descriptor.baseAddress, static_cast(guestBytes), stampedBlocks, true)); if (compared) { for (const auto& [from, unitGeneration] : keptAtUnwatch) { const auto unit = static_cast((from - descriptor.baseAddress) / trackedLayerBytes); if (keptAtUnwatchHolds(from, unit)) stampedBlocks[static_cast(from / 65536 - descriptor.baseAddress / 65536)] = GuestMemory::BlockWritten; } cpuBlocks = stampedBlocks; + if (scoped) { + std::fill(stampedBlocks.begin(), stampedBlocks.begin() + static_cast(firstBlock), GuestMemory::BlockWritten); + std::fill(stampedBlocks.begin() + static_cast(firstBlock + blocks), stampedBlocks.end(), GuestMemory::BlockWritten); + } } }; const auto keysStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -1827,7 +1867,7 @@ bool StorageTexture::Refresh() { scanBlocks(); for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { if (stampedBlocks[unit] == 0) { - layerGeneration[unit] = current; + layerGeneration[unit] = currentFor(unit); continue; } changed[unit] = true; @@ -1853,6 +1893,15 @@ bool StorageTexture::Refresh() { } unchanged = std::none_of(changed.begin(), changed.end(), [](bool value) { return value; }); } + if (scoped && !keysChanged) { + outside.assign(trackedLayers, false); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if ((layer >= firstLayer && layer < lastLayer) || (!changed[layer] && !compared)) continue; + outside[layer] = true; + changed[layer] = false; + } + unchanged = std::none_of(changed.begin(), changed.end(), [](bool value) { return value; }); + } for (std::size_t k = 0; k < cpuBlocks.size(); ++k) { if (cpuBlocks[k] != 0 && (compared || (tracked && generations[k] != 0))) cpuWrote = true; } @@ -1863,12 +1912,11 @@ bool StorageTexture::Refresh() { for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (changed[layer] && layerPending[layer]) pendingChanged = true; } - if (!unchanged && !keysChanged && !pendingChanged && originalHolds(descriptor.baseAddress, static_cast(guestBytes)) && equalsOriginal()) { + if (!unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast(guestBytes))) && equalsOriginal()) { unchanged = true; stamped = false; + if (!scoped) outside.clear(); } - // Only the direct path uploads the selected layers alone (see upload); the others replace - // the whole image, so every pending layer's results are stored first. direct = keys == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr; if (alias != nullptr && direct) { // The alias's pending units that nothing stamped since its generation (a CPU store or a @@ -1893,11 +1941,16 @@ bool StorageTexture::Refresh() { } } } + const bool outsideChanged = std::any_of(outside.begin(), outside.end(), [](bool value) { return value; }); + if (scoped) noteLayersBehind(firstLayer, lastLayer, outside); + else layerBehind.clear(); if (unchanged) { ++Profile().storageReused; - layerGeneration.assign(trackedLayers, current); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (!outsideChanged || !outside[layer]) layerGeneration[layer] = currentFor(layer); + } refreshGeneration(); - takeRefreshProof(alias != nullptr); + if (layerBehind.empty()) takeRefreshProof(alias != nullptr); if (profile) LookupOutcomes::Add(stamped ? LookupOutcomes::RefreshUnchanged : LookupOutcomes::RefreshCompared, start); return true; } @@ -1947,6 +2000,7 @@ bool StorageTexture::Refresh() { } std::fprintf(stderr, "[dcc-keys] refresh 0x%llx+0x%llx keys %s -> %s (dirty %d, %zu pending units, %zu cpu-stamped units%s)\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), DccKeysName(uploadedKeys), DccKeysName(keys), dirty ? 1 : 0, pendingUnits, stampedUnits, keyFlip ? ", flip keeps unstamped results" : ""); } + const bool selectedCpu = !direct && keys == DccKeys::Uncompressed && !cpuUploadLayers(changed).empty(); for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (!layerPending[layer]) continue; if (keyFlip && layerGeneration[layer] != 0 && stampedBlocks[layer] != GuestMemory::BlockWritten) { @@ -1965,7 +2019,7 @@ bool StorageTexture::Refresh() { dropped = true; continue; } - } else if (direct) { + } else if (direct || selectedCpu) { continue; } stored[layer] = true; @@ -1987,7 +2041,7 @@ bool StorageTexture::Refresh() { uploadReason = "alias"; borrowUnits(*alias, borrowed); for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { - if (borrowed[unit]) layerGeneration[unit] = current; + if (borrowed[unit]) layerGeneration[unit] = currentFor(unit); } refreshGeneration(); } @@ -2096,7 +2150,7 @@ void traceKeyStore(const char* path, const GuestTextureResource& descriptor, std void StorageTexture::captureGuestBytes(const std::vector* layers) { original.resize(static_cast(guestBytes)); - comparedGuestBytes = {}; + comparedRanges = {}; std::vector selected(trackedLayers, true); if (layers != nullptr) selected = *layers; for (const auto& [begin, end] : unitRuns(selected)) GuestMemory::FlushGpuWrites(descriptor.baseAddress + begin, static_cast(end - begin)); @@ -2133,11 +2187,10 @@ bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes, Require(changed.size() == (end - 1) / blockBytes - first + 1, "texture comparison block count differs"); const auto* recorder = Recorder::Active(); const auto epoch = GuestMemory::CollectEpoch(); - const bool whole = address == descriptor.baseAddress && bytes == guestBytes; - const auto hint = memoize && epoch != 0 && whole ? GuestMemory::UntrackedHint(address, bytes) : 0; + const auto hint = memoize && epoch != 0 ? GuestMemory::UntrackedHint(address, bytes) : 0; const std::array stamp{hint != 0, hint != 0 ? hint : epoch, GuestMemory::NewestStamp(address, bytes), GuestMemory::ForgetSerial(), recorder != nullptr ? recorder->NewestWriteNote(address, bytes) : 0}; - const bool cacheable = memoize && epoch != 0 && (stamp[3] & 1u) == 0 && whole; - if (cacheable && comparedGuestBytes == stamp) { + const bool cacheable = memoize && epoch != 0 && (stamp[3] & 1u) == 0; + if (cacheable && std::any_of(comparedRanges.begin(), comparedRanges.end(), [&](const ComparedRange& entry) { return entry.address == address && entry.bytes == bytes && entry.stamp == stamp; })) { std::fill(changed.begin(), changed.end(), GuestMemory::BlockUnchanged); return true; } @@ -2145,10 +2198,14 @@ bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes, const auto saved = std::span(original).subspan(static_cast(address - descriptor.baseAddress), bytes); if (GuestMemory::EqualsCommittedUnsynced(address, saved)) { std::fill(changed.begin(), changed.end(), GuestMemory::BlockUnchanged); - if (cacheable) comparedGuestBytes = stamp; + if (cacheable) { + auto found = std::find_if(comparedRanges.begin(), comparedRanges.end(), [&](const ComparedRange& entry) { return entry.address == address && entry.bytes == bytes; }); + if (found == comparedRanges.end()) found = comparedRanges.begin() + nextComparedRange++ % comparedRanges.size(); + *found = {address, bytes, stamp}; + } return true; } - comparedGuestBytes = {}; + comparedRanges = {}; for (std::size_t index = 0; index < changed.size(); ++index) { const auto begin = std::max(address, (first + index) * blockBytes); const auto stop = std::min(end, (first + index + 1) * blockBytes); @@ -2160,6 +2217,25 @@ bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes, constexpr std::uint64_t GenerationBaselineBytes = 1ull << 20; +std::vector StorageTexture::cpuUploadLayers(const std::vector& units) const { + if (descriptor.dimension != TextureDimension::k2DArray || descriptor.mipCount != 1 || descriptor.dccAddress != 0 || IsBlockCompressed(descriptor.format) || geometry.thick || arrayLayers <= 1 || geometry.imageLayers != arrayLayers || trackedLayerBytes == 0 || units.size() != trackedLayers) return {}; + std::vector selected(arrayLayers, false); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + const auto begin = geometry.GuestLayerOffset(layer); + const auto end = begin + geometry.layerBytes; + if (end <= begin || end > guestBytes || begin % trackedLayerBytes != 0 || (end % trackedLayerBytes != 0 && end != guestBytes)) return {}; + const auto first = static_cast(begin / trackedLayerBytes); + const auto last = static_cast((end + trackedLayerBytes - 1) / trackedLayerBytes); + if (first >= last || last > trackedLayers) return {}; + selected[layer] = units[first]; + for (auto unit = first + 1; unit < last; ++unit) { + if (units[unit] != selected[layer]) return {}; + } + } + if (std::none_of(selected.begin(), selected.end(), [](bool value) { return value; }) || std::all_of(selected.begin(), selected.end(), [](bool value) { return value; })) return {}; + return selected; +} + void StorageTexture::upload(const std::vector* layers) { const bool profile = LookupOutcomes::Profiled(); const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -2168,8 +2244,6 @@ void StorageTexture::upload(const std::vector* layers) { CaptureTrace::Log("upload image=%llx bytes=%llu generation=%llu reason=%s partial=%d", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(generation), uploadReason, layers != nullptr); const auto elementBytes = BytesPerElement(descriptor.format); const auto linearBytes = sliceLinearBytes * arrayLayers; - // A layer selection applies to the direct path alone (the others upload everything); it names - // array layers, which the tracked layers are when there are several. if (layers != nullptr && ((!blockUnits && trackedLayers != arrayLayers) || std::all_of(layers->begin(), layers->end(), [](bool selected) { return selected; }))) layers = nullptr; const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); const auto stampLayers = [&](bool selectedOnly) { @@ -2181,7 +2255,7 @@ void StorageTexture::upload(const std::vector* layers) { } for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (!whole && !(*layers)[layer]) continue; - layerGeneration[layer] = now; + layerGeneration[layer] = collectedLayerGeneration(layer, now); if (whole || generationBaseline.size() != guestBytes) continue; const auto offset = static_cast(layerBegin(layer) - descriptor.baseAddress); GuestMemory::ReadCommitted(layerBegin(layer), std::span(generationBaseline).subspan(offset, static_cast(layerBytes(layer)))); @@ -2376,15 +2450,36 @@ void StorageTexture::upload(const std::vector* layers) { if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start); return; } - layerOriginal.assign(trackedLayers, true); - cancelStoredSaves(); - forgetBorrowed(0, trackedLayers); - stampLayers(false); - original.resize(static_cast(guestBytes)); - GuestMemory::ReadCommitted(descriptor.baseAddress, original); + const auto cpuLayers = uploadedKeys == DccKeys::Uncompressed && layers != nullptr ? cpuUploadLayers(*layers) : std::vector{}; + const bool selectedCpu = !cpuLayers.empty(); + std::uint64_t uploadedBytes = guestBytes; + if (selectedCpu) { + captureGuestBytes(layers); + for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) { + if ((*layers)[unit]) forgetBorrowed(unit, 1); + } + stampLayers(true); + uploadedBytes = 0; + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (cpuLayers[layer]) uploadedBytes += geometry.layerBytes; + } + } else { + layerOriginal.assign(trackedLayers, true); + cancelStoredSaves(); + forgetBorrowed(0, trackedLayers); + stampLayers(false); + original.resize(static_cast(guestBytes)); + GuestMemory::ReadCommitted(descriptor.baseAddress, original); + } { Buffer staging(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); - if (uploadedKeys == DccKeys::Uncompressed) std::memcpy(staging.Bytes().data(), original.data(), original.size()); + if (selectedCpu) { + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (!cpuLayers[layer]) continue; + const auto offset = geometry.GuestLayerOffset(layer); + std::memcpy(staging.Bytes().data() + offset, original.data() + offset, static_cast(geometry.layerBytes)); + } + } else if (uploadedKeys == DccKeys::Uncompressed) std::memcpy(staging.Bytes().data(), original.data(), original.size()); else ReadTextureSurface(descriptor, uploadedKeys, staging.Bytes().first(original.size())); DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); DeviceBuffer linear(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); @@ -2392,9 +2487,16 @@ void StorageTexture::upload(const std::vector* layers) { CommandBatch batch(context); const auto commands = batch.Handle(); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); - CopyBuffer(context, commands, staging.Handle(), 0, tiled.Handle(), 0, original.size()); + if (selectedCpu) { + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (!cpuLayers[layer]) continue; + const auto offset = geometry.GuestLayerOffset(layer); + CopyBuffer(context, commands, staging.Handle(), offset, tiled.Handle(), offset, geometry.layerBytes); + } + } else CopyBuffer(context, commands, staging.Handle(), 0, tiled.Handle(), 0, original.size()); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (selectedCpu && !cpuLayers[layer]) continue; for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; @@ -2403,15 +2505,16 @@ void StorageTexture::upload(const std::vector* layers) { } const auto linearRead = WholeBufferBarrier(linear.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.srcAccessMask = selectedCpu ? VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT : 0u; toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toTransfer.oldLayout = selectedCpu ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED; toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toTransfer.image = image; toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); - const auto regions = CopyRegions(); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | (selectedCpu ? VK_PIPELINE_STAGE_ALL_COMMANDS_BIT : 0u), VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearRead, 1, &toTransfer); + const auto regions = CopyRegions(selectedCpu ? &cpuLayers : nullptr); context.Function("vkCmdCopyBufferToImage")(commands, linear.Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast(regions.size()), regions.data()); VkImageMemoryBarrier toGeneral = toTransfer; toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; @@ -2423,7 +2526,7 @@ void StorageTexture::upload(const std::vector* layers) { batch.SubmitAndWait(); APS5_LOG_CHARS_OUT("StorageTexture upload done"); } - countStorageUpload(2, guestBytes); + countStorageUpload(2, uploadedBytes); ++version; if (profile) LookupOutcomes::Add(LookupOutcomes::UploadCpu, start); } @@ -3119,6 +3222,51 @@ void StorageTexture::refreshGeneration() { generation = *std::min_element(layerGeneration.begin(), layerGeneration.end()); } +std::uint64_t StorageTexture::GenerationFor(std::uint64_t address, std::uint64_t bytes) const { + if (layerBehind.empty() && generation != 0) return generation; + std::optional oldest; + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + const auto begin = layerBegin(layer); + if (address >= begin + layerBytes(layer) || begin >= address + bytes) continue; + if (!layerBehind.empty() && layerBehind[layer]) return 0; + oldest = std::min(oldest.value_or(layerGeneration[layer]), layerGeneration[layer]); + } + return oldest.value_or(generation); +} + +std::uint64_t StorageTexture::collectedLayerGeneration(std::uint32_t layer, std::uint64_t whole, bool uncached) const { + if (whole != 0 || trackedLayers <= 1) return whole; + const auto begin = layerBegin(layer); + const auto bytes = static_cast(layerBytes(layer)); + return uncached ? GuestMemory::CollectWritesUncached(begin, bytes) : GuestMemory::CollectWrites(begin, bytes); +} + +void StorageTexture::noteLayersBehind(std::uint32_t firstLayer, std::uint32_t lastLayer, const std::vector& outside) { + if (layerBehind.size() != trackedLayers) layerBehind.assign(trackedLayers, false); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layer >= firstLayer && layer < lastLayer) layerBehind[layer] = false; + else if (layer < outside.size() && outside[layer]) layerBehind[layer] = true; + } + const auto layersEnd = layerBegin(lastLayer - 1) + layerBytes(lastLayer - 1); + const std::array, 2> ranges{{{descriptor.baseAddress, layerBegin(firstLayer)}, {layersEnd, descriptor.baseAddress + guestBytes}}}; + for (const auto& [from, to] : ranges) { + if (to <= from) continue; + for (const auto& other : overlappingPending(from, static_cast(to - from))) { + const bool whole = other->descriptor.dccAddress != 0 && other->uploadedKeys != DccKeys::Uncompressed; + for (std::uint32_t unit = 0; unit < other->trackedLayers; ++unit) { + if (!whole && !other->layerPending[unit]) continue; + const auto begin = std::max(from, other->layerBegin(unit)); + const auto end = std::min(to, other->layerBegin(unit) + other->layerBytes(unit)); + if (end <= begin) continue; + const auto first = static_cast((begin - descriptor.baseAddress) / trackedLayerBytes); + const auto last = static_cast(std::min((end - descriptor.baseAddress + trackedLayerBytes - 1) / trackedLayerBytes, trackedLayers)); + for (auto layer = first; layer < last; ++layer) layerBehind[layer] = true; + } + } + } + if (std::none_of(layerBehind.begin(), layerBehind.end(), [](bool behind) { return behind; })) layerBehind.clear(); +} + void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count) { CaptureTrace::Log("image-write image=%llx first=%u count=%u generation=%llu", static_cast(descriptor.baseAddress), first, count, static_cast(generation)); static const bool eager = std::getenv("APS5_EAGER_WRITEBACK") != nullptr || std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; @@ -3338,7 +3486,7 @@ void StorageTexture::saveBeforeUnwatch() { const auto runs = unitRuns(units); if (runs.empty()) return; original.resize(static_cast(guestBytes)); - comparedGuestBytes = {}; + comparedRanges = {}; for (const auto& [begin, end] : runs) GuestMemory::ReadCommittedUnsynced(descriptor.baseAddress + begin, std::span(original).subspan(static_cast(begin), static_cast(end - begin))); for (const auto& [begin, end] : runs) GuestMemory::CollectWritesUncached(descriptor.baseAddress + begin, static_cast(end - begin)); constexpr std::uint64_t block = 65536; @@ -3399,7 +3547,7 @@ void StorageTexture::applyStoredSave(std::uint64_t ticket, const std::vector(guestBytes)); - comparedGuestBytes = {}; + comparedRanges = {}; const auto offset = static_cast(layerBegin(unit) - descriptor.baseAddress); std::memcpy(original.data() + offset, stored.data() + offset, static_cast(layerBytes(unit))); layerOriginal[unit] = true; @@ -4022,7 +4170,8 @@ bool StorageTexture::clearByKeysFill(DccKeys keys, std::uint8_t key) { filledKeys = DccKeys::Uncompressed; keyProof = {}; forgetBorrowed(0, trackedLayers); - layerGeneration.assign(trackedLayers, GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes))); + const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) layerGeneration[layer] = collectedLayerGeneration(layer, now); refreshGeneration(); ++version; if (traceKeys) std::fprintf(stderr, "[dcc-keys] key fill %s clears 0x%llx+0x%llx at once (keys 0x%llx, %zu pending units dropped, dirty %d)\n", DccKeysName(keys), static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(descriptor.dccAddress), droppedUnits, wasDirty ? 1 : 0); @@ -4183,7 +4332,8 @@ bool StorageTexture::FillClear(std::span pattern, std::u return true; } if (!watched) captureGuestBytes(nullptr); - layerGeneration.assign(trackedLayers, GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes))); + const auto now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) layerGeneration[layer] = collectedLayerGeneration(layer, now, true); refreshGeneration(); forgetBorrowed(0, trackedLayers); MarkDirty(); @@ -4305,7 +4455,8 @@ bool StorageTexture::CopyFrom(StorageTexture& source, const char*& refusal) { captureGuestBytes(nullptr); } uploadedKeys = DccKeys::Uncompressed; - layerGeneration.assign(trackedLayers, GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes))); + const auto now = GuestMemory::CollectWritesUncached(descriptor.baseAddress, static_cast(guestBytes)); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) layerGeneration[layer] = collectedLayerGeneration(layer, now, true); refreshGeneration(); forgetBorrowed(0, trackedLayers); MarkDirty(); @@ -4670,7 +4821,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { if (!advances && adjacent.empty() && shadowed.empty()) return; const auto now = memoizedCollect ? GuestMemory::CollectWrites(firstStored, static_cast(lastStored - firstStored)) : GuestMemory::CollectWritesUncached(firstStored, static_cast(lastStored - firstStored)); for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { - if (layers[layer] && !skippedLayer[layer]) layerGeneration[layer] = now; + if (layers[layer] && !skippedLayer[layer]) layerGeneration[layer] = collectedLayerGeneration(layer, now, !memoizedCollect); } refreshGeneration(); advanceAdjacent(adjacent, now, firstBlock, lastBlock); @@ -4850,7 +5001,8 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { overhang = covered >= end; } } - if (unregisteredDrop && !overhang && !RegisteredReadableCovers(descriptor.baseAddress, static_cast(guestBytes))) { + const bool registered = std::all_of(keep.begin(), keep.end(), [](const auto& range) { return RegisteredReadableCovers(range.first, static_cast(range.second - range.first)); }); + if (unregisteredDrop && !overhang && !registered) { static std::atomic reports{0}; if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx+0x%llx: memory no longer registered; %zu pending ranges dropped\n", static_cast(descriptor.baseAddress), static_cast(guestBytes), keep.size()); unregisteredDropped.fetch_add(1, std::memory_order_relaxed); @@ -4868,7 +5020,13 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { Require(!unsavedOutsideWatch, "storing storage image results pending over memory outside the write watch without the guest bytes saved before them is not implemented"); std::vector unsaved = layerOriginal; unsaved.flip(); - for (const auto& [begin, end] : unitRuns(unsaved)) GuestMemory::ReadCommitted(descriptor.baseAddress + begin, std::span(original).subspan(static_cast(begin), static_cast(end - begin))); + for (const auto& [begin, end] : unitRuns(unsaved)) { + for (const auto& [from, to] : keep) { + const auto first = std::max(descriptor.baseAddress + begin, from); + const auto last = std::min(descriptor.baseAddress + end, to); + if (first < last) GuestMemory::ReadCommitted(first, std::span(original).subspan(static_cast(first - descriptor.baseAddress), static_cast(last - first))); + } + } DeviceBuffer linear(context, static_cast(sliceLinearBytes * arrayLayers), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); DeviceBuffer tiled(context, original.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); Buffer host(context, original.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 580996f9212..8248bf5e811 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -109,6 +109,32 @@ void ReleaseWatched(void* block, std::size_t bytes) { #endif } +bool collectMappingHistoryTests() { + constexpr std::size_t bytes = 131072; + auto* memory = static_cast(AllocateWatched(bytes, 65536)); + if (memory == nullptr) return false; + GuestAllocations::Mutation().Add(memory, bytes, true, true, false); + struct Cleanup { + std::uint8_t* memory; + ~Cleanup() { + GuestAllocations::Mutation().Remove(memory); + ReleaseWatched(memory, bytes); + } + } cleanup{memory}; + const auto address = reinterpret_cast(memory); + std::memset(memory, 0x11, bytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + const auto before = AgcDriver::GuestMemory::CollectWrites(address, bytes); + Require(before != 0, "the mapping fixture has no complete write history"); + memory[4096] = 0x22; + GuestAllocations::Mutation().Remove(memory); + GuestAllocations::Mutation().Add(memory, bytes, true, true, false); + Require(AgcDriver::GuestMemory::CollectWrites(address, bytes) != 0, "the replacement mapping lost write tracking"); + Require(!AgcDriver::GuestMemory::UnchangedSince(address, bytes, before), "a cached collect hid a CPU store across allocation replacement"); + Require(memory[4096] == 0x22, "collecting the replacement mapping changed its CPU bytes"); + return true; +} + std::optional selectDevice(std::span devices, std::string_view request) { const auto rankDeviceType = [](VkPhysicalDeviceType type) { switch (type) { @@ -8176,6 +8202,11 @@ void Run(Context context) { int main(int argc, char** argv) { try { + if (argc == 2 && std::string_view(argv[1]) == "--collect-mapping-history-only") { + if (!collectMappingHistoryTests()) return 77; + std::cout << "collect mapping history tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--device-selection-only") { deviceSelectionTests(); std::cout << "recorder device selection tests passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ArrayLayerTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ArrayLayerTarget.cpp index 18dcef360cc..aed0ffc37ad 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ArrayLayerTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ArrayLayerTarget.cpp @@ -68,6 +68,8 @@ alignas(256) std::array PixelCode{}; alignas(256) std::array Output{}; void SampleSliceZero(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t swizzleMode); +void OverlappingArrays(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t swizzleMode, bool cpuWrite = false); +void OverlappingArrayLastSlice(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t swizzleMode); std::uint64_t AddressOf(const void* data) { return static_cast(reinterpret_cast(data)); @@ -81,27 +83,29 @@ std::string Hex(std::uint32_t value) { class GuestBlock { public: - GuestBlock() { + explicit GuestBlock(std::size_t registeredBytes = SurfaceBytes, std::size_t registeredOffset = 0, bool commitAll = true) : registeredOffset(registeredOffset) { #ifdef _WIN32 block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(SurfaceBytes, LayerBytes)); - if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, SurfaceBytes, PAGE_READWRITE, SurfaceBytes); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block + (commitAll ? 0 : registeredOffset), commitAll ? SurfaceBytes : registeredBytes, PAGE_READWRITE, commitAll ? SurfaceBytes : registeredBytes); #else - void* mapped = mmap(nullptr, SurfaceBytes + LayerBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void* mapped = mmap(nullptr, SurfaceBytes + LayerBytes, commitAll ? PROT_READ | PROT_WRITE : PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (mapped != MAP_FAILED) { const auto begin = reinterpret_cast(mapped); const auto aligned = (begin + LayerBytes - 1) & ~(static_cast(LayerBytes) - 1); if (aligned != begin) munmap(mapped, aligned - begin); if (aligned + SurfaceBytes != begin + SurfaceBytes + LayerBytes) munmap(reinterpret_cast(aligned + SurfaceBytes), begin + LayerBytes - aligned); block = reinterpret_cast(aligned); - if (AgcDriver::GuestMemory::WriteWatched()) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, SurfaceBytes); + if (!commitAll) Require(mprotect(block + registeredOffset, registeredBytes, PROT_READ | PROT_WRITE) == 0, "cannot commit the active array slice"); + if (AgcDriver::GuestMemory::WriteWatched()) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block + (commitAll ? 0 : registeredOffset), commitAll ? SurfaceBytes : registeredBytes); } #endif Require(block != nullptr, "array layer target: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, SurfaceBytes, true, true, true); + if (!commitAll) GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(block), SurfaceBytes); + GuestAllocations::Mutation().Add(block + registeredOffset, registeredBytes, true, true, true); } ~GuestBlock() { - GuestAllocations::Mutation().Remove(block); + if (registered) GuestAllocations::Mutation().Remove(block + registeredOffset); #ifdef _WIN32 GuestArena::GuestArenaReset_nid_postfix(block, SurfaceBytes); GuestArena::GuestArenaRelease_nid_postfix(block, SurfaceBytes); @@ -115,9 +119,15 @@ class GuestBlock { GuestBlock& operator=(const GuestBlock&) = delete; std::uint8_t* Data() { return block; } + void Unregister() { + GuestAllocations::Mutation().Remove(block + registeredOffset); + registered = false; + } private: std::uint8_t* block = nullptr; + std::size_t registeredOffset = 0; + bool registered = true; }; std::array SurfaceDescriptor(std::uint64_t address, std::uint32_t swizzleMode, std::uint32_t type, std::uint32_t lastLayer) { @@ -164,10 +174,10 @@ void Load(AgcDriver::VulkanDevice& device, std::span code, Dispatch(device, code, userData); } -void LoadLayer(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint32_t layer, std::uint32_t swizzleMode) { +void LoadLayer(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint32_t layer, std::uint32_t swizzleMode, std::uint32_t lastLayer = Layers - 1u) { auto& code = LoadCodes[layer]; code = {0x7e020280, 0x7e040280u | layer, 0xf0001128, 0x00010300, 0x34080082, 0xbf8c3f70, 0xe0701000, 0x80000304, 0xbf810000}; - Load(device, code, SurfaceDescriptor(surface, swizzleMode, Type2DArray, Layers - 1u)); + Load(device, code, SurfaceDescriptor(surface, swizzleMode, Type2DArray, lastLayer)); } void Sample2D(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint32_t swizzleMode) { @@ -181,23 +191,23 @@ void RequireOutput(std::uint32_t inside, std::uint32_t outside, const std::strin } } -AgcDriver::Graphics::ColorTarget DecodeTarget(std::uint64_t address, std::uint32_t slice, std::uint32_t swizzleMode, bool collapsed) { +AgcDriver::Graphics::ColorTarget DecodeTarget(std::uint64_t address, std::uint32_t slice, std::uint32_t swizzleMode, bool collapsed, std::uint32_t lastSlice) { AgcDriver::Registers cx; cx[0x318] = static_cast(address >> 8u); cx[0x31b] = slice | (slice << 13u); cx[0x31c] = Color32FloatInfo; cx[0x31d] = 0; cx[0x3b0] = ((Side - 1u) << 14u) | (Side - 1u); - cx[0x3b8] = (TiledAttrib3 & ~(0x1fu << 14u)) | (swizzleMode << 14u) | (collapsed ? 0u : Layers - 1u); + cx[0x3b8] = (TiledAttrib3 & ~(0x1fu << 14u)) | (swizzleMode << 14u) | (collapsed ? 0u : lastSlice); cx[0x390] = static_cast(address >> 40u); const auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); Require(color.format == VK_FORMAT_R32_SFLOAT && color.bytes == LayerBytes && color.address == address + slice * LayerBytes, "the COLOR_32 float view did not decode as a 128x128 R32 slice of the array"); if (collapsed) Require(color.arrayAddress == 0 && color.arrayLastSlice == 0, "a collapsed slice view unexpectedly describes a whole array"); - else Require(color.arrayAddress == address && color.arrayLastSlice == Layers - 1u && color.arraySlice == slice, "the color view did not keep its array and slice"); + else Require(color.arrayAddress == address && color.arrayLastSlice == lastSlice && color.arraySlice == slice, "the color view did not keep its array and slice"); return color; } -void DrawSlice(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint32_t slice, std::uint32_t swizzleMode, bool collapsed) { +void DrawSlice(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint32_t slice, std::uint32_t swizzleMode, bool collapsed, std::uint32_t lastSlice = Layers - 1u, std::uint32_t width = ScissorWidth) { const auto target = device.Target(); constexpr std::uint32_t waveSize = 64; std::size_t at = 0; @@ -224,7 +234,7 @@ void DrawSlice(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint const auto vertexResult = ShaderRecompiler::Recompile(vertex); const auto vertexPush = static_cast(vertexResult.pushConstants.size()); - const auto color = DecodeTarget(surface, slice, swizzleMode, collapsed); + const auto color = DecodeTarget(surface, slice, swizzleMode, collapsed, lastSlice); VkPipelineColorBlendAttachmentState blend{}; blend.colorWriteMask = 0xf; AgcDriver::Graphics::State state{}; @@ -236,7 +246,7 @@ void DrawSlice(AgcDriver::VulkanDevice& device, std::uint64_t surface, std::uint state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; state.viewport = {0, static_cast(Side), static_cast(Side), -static_cast(Side), 0, 1}; state.negativeOneToOne = false; - state.scissor = {{0, 0}, {ScissorWidth, Side}}; + state.scissor = {{0, 0}, {width, Side}}; state.cullMode = VK_CULL_MODE_NONE; state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; state.blends = {blend}; @@ -290,6 +300,9 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t sw RequireOutput(drawn, Initial, "slice " + std::to_string(slice) + " uploaded again from guest memory"); AgcDriver::Graphics::ClearCachedTextures(device.Device()); SampleSliceZero(device, texels, swizzleMode); + OverlappingArrays(device, texels, swizzleMode); + OverlappingArrays(device, texels, swizzleMode, true); + OverlappingArrayLastSlice(device, texels, swizzleMode); } void SampleSliceZero(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t swizzleMode) { @@ -309,6 +322,140 @@ void SampleSliceZero(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std: AgcDriver::Graphics::ClearCachedTextures(device.Device()); } +void OverlappingArrayLastSlice(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t swizzleMode) { + const auto neighbourAddress = AddressOf(texels); + const auto surface = neighbourAddress + LayerBytes; + const auto cpuValue = std::bit_cast(0.125f); + const auto drawn = std::bit_cast(Drawn); + std::fill_n(reinterpret_cast(texels), SurfaceBytes / sizeof(std::uint32_t), Initial); + AgcDriver::GuestMemory::CollectWrites(neighbourAddress, SurfaceBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + DrawSlice(device, surface, 1u, swizzleMode, false, 1u); + DrawSlice(device, neighbourAddress, 1u, swizzleMode, false, 1u, Side); + const auto neighbour = StorageTexture::FindPending(neighbourAddress, 2u * LayerBytes); + Require(neighbour != nullptr && neighbour->Descriptor().baseAddress == neighbourAddress, "the preceding target has no pending results"); + std::fill_n(reinterpret_cast(texels + 2u * LayerBytes), LayerBytes / sizeof(std::uint32_t), cpuValue); + AgcDriver::GuestMemory::CollectWrites(surface + LayerBytes, LayerBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + DrawSlice(device, surface, 1u, swizzleMode, false, 1u); + Require(StorageTexture::FindPending(neighbourAddress, 2u * LayerBytes) == neighbour, "CPU refresh of the last slice stored the preceding target's results"); + LoadLayer(device, surface, 1u, swizzleMode, 1u); + RequireOutput(drawn, cpuValue, "the refreshed last slice keeps the CPU's untouched pixels"); + LoadLayer(device, surface, 0u, swizzleMode, 1u); + RequireOutput(drawn, drawn, "the first slice sees the preceding target's pending results"); + StorageTexture::FlushPending(neighbourAddress, SurfaceBytes, nullptr, "test"); + device.WaitIdle(); + AgcDriver::Graphics::ClearCachedTextures(device.Device()); +} + +void UnregisteredUnusedLayer(AgcDriver::VulkanDevice& device, std::uint32_t slice) { + GuestBlock block(LayerBytes, slice * LayerBytes); + auto* texels = block.Data(); + const auto surface = AddressOf(texels); + const auto active = surface + slice * LayerBytes; + std::fill_n(reinterpret_cast(texels), SurfaceBytes / 4u, Initial); + AgcDriver::GuestMemory::CollectWrites(surface, SurfaceBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + DrawSlice(device, surface, slice, TileR64KBX, false); + Require(StorageTexture::FindPending(surface, SurfaceBytes) != nullptr, "the registered slice was not pending before its CPU write-back"); + StorageTexture::FlushPending(active, LayerBytes, nullptr, "test"); + device.WaitIdle(); + const auto* first = reinterpret_cast(active); + const auto* last = reinterpret_cast(active + LayerBytes); + const auto unchanged = [](std::uint32_t value) { return value == Initial; }; + const auto drawn = std::count(first, last, std::bit_cast(Drawn)); + const bool untouched = std::all_of(reinterpret_cast(texels), first, unchanged) && std::all_of(last, reinterpret_cast(texels + SurfaceBytes), unchanged); + AgcDriver::Graphics::ClearCachedTextures(device.Device()); + Require(drawn == ScissorWidth * Side, "CPU write-back discarded the registered slice because an unused array slice was unregistered"); + Require(untouched, "CPU write-back modified an unregistered unused slice"); +} + +void RetiredArrayLayer(AgcDriver::VulkanDevice& device) { + GuestBlock block(LayerBytes); + auto* texels = block.Data(); + const auto surface = AddressOf(texels); + std::fill_n(reinterpret_cast(texels), SurfaceBytes / 4u, Initial); + AgcDriver::GuestMemory::CollectWrites(surface, SurfaceBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + DrawSlice(device, surface, 0u, TileR64KBX, false); + Require(StorageTexture::FindPending(surface, SurfaceBytes) != nullptr, "the retired slice was not pending before it was unregistered"); + block.Unregister(); + StorageTexture::FlushPending(surface, LayerBytes, nullptr, "test"); + device.WaitIdle(); + const bool untouched = std::all_of(reinterpret_cast(texels), reinterpret_cast(texels + SurfaceBytes), [](std::uint32_t value) { return value == Initial; }); + AgcDriver::Graphics::ClearCachedTextures(device.Device()); + Require(untouched, "CPU write-back stored results over a retired allocation"); +} + +void UncommittedUnusedLayers(AgcDriver::VulkanDevice& device, std::uint32_t slice) { + if (!AgcDriver::GuestMemory::WriteWatched()) return; + GuestBlock block(LayerBytes, slice * LayerBytes, false); + const auto surface = AddressOf(block.Data()); + const auto active = surface + slice * LayerBytes; + auto* texels = reinterpret_cast(active); + std::fill_n(texels, LayerBytes / sizeof(std::uint32_t), Initial); + Require(AgcDriver::GuestMemory::CollectWrites(active, LayerBytes) != 0, "the committed array slice has no write history"); + Require(AgcDriver::GuestMemory::CollectWritesUncached(surface, SurfaceBytes) == 0, "the uncommitted array fixture unexpectedly has complete backend write history"); + Require(AgcDriver::GuestMemory::CollectWrites(surface, SurfaceBytes) == 0, "the unused array slices unexpectedly have complete write history"); + AgcDriver::GuestMemory::BumpCollectEpoch(); + DrawSlice(device, surface, slice, TileR64KBX, false); + const auto pending = StorageTexture::FindPending(surface, SurfaceBytes); + Require(pending != nullptr, "the committed array slice has no pending draw results"); + Require(pending->GenerationFor(active, LayerBytes) != 0, "an uncommitted unused slice erased the active slice's write history"); + const auto version = pending->Version(); + for (unsigned repeat = 0; repeat < 3; ++repeat) { + AgcDriver::GuestMemory::BumpCollectEpoch(); + Require(pending->RefreshLayers(slice, 1), "an unchanged committed slice was uploaded again"); + Require(pending->GenerationFor(active, LayerBytes) != 0 && pending->Version() == version, "repeated refresh erased the committed slice's write history"); + Require(StorageTexture::FindPending(surface, SurfaceBytes) == pending, "refresh settled unchanged GPU results for an uncommitted unused slice"); + } + const auto cpu = std::bit_cast(0.125f); + std::fill_n(texels, LayerBytes / sizeof(std::uint32_t), cpu); + AgcDriver::GuestMemory::BumpCollectEpoch(); + Require(!pending->RefreshLayers(slice, 1), "the active array slice ignored a genuine CPU write"); + LoadLayer(device, surface, slice, TileR64KBX); + RequireOutput(cpu, cpu, "a committed array slice after a genuine CPU write"); + DrawSlice(device, surface, slice, TileR64KBX, false); + StorageTexture::FlushPending(active, LayerBytes, nullptr, "test"); + device.WaitIdle(); + Require(std::count(texels, texels + LayerBytes / sizeof(std::uint32_t), std::bit_cast(Drawn)) == ScissorWidth * Side, "write-back lost results in the committed array slice"); + AgcDriver::Graphics::ClearCachedTextures(device.Device()); +} + +void OverlappingArrays(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint32_t swizzleMode, bool cpuWrite) { + const auto surface = AddressOf(texels); + const auto next = surface + LayerBytes; + const auto drawn = std::bit_cast(Drawn); + std::fill_n(reinterpret_cast(texels), SurfaceBytes / 4u, Initial); + AgcDriver::GuestMemory::CollectWrites(surface, SurfaceBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + DrawSlice(device, surface, 0, swizzleMode, false, 1u); + DrawSlice(device, next, 0, swizzleMode, false, 1u, Side); + const auto neighbour = StorageTexture::FindPending(next, 2u * LayerBytes); + Require(neighbour != nullptr && neighbour->Descriptor().baseAddress == next && neighbour->Descriptor().dimension == TextureDimension::k2DArray, "the next target did not render into the image of its own array"); + if (cpuWrite) { + std::fill_n(reinterpret_cast(texels), LayerBytes / sizeof(std::uint32_t), std::bit_cast(0.125f)); + AgcDriver::GuestMemory::CollectWrites(surface, LayerBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + } + DrawSlice(device, surface, 0, swizzleMode, false, 1u); + Require(StorageTexture::FindPending(next, 2u * LayerBytes) == neighbour, "a draw into slice 0 of the first array stored the next target's results, which only its unused second slice overlaps"); + Sample2D(device, surface, swizzleMode); + RequireOutput(drawn, cpuWrite ? std::bit_cast(0.125f) : Initial, "slice 0 of the first array sampled as a 2D texture after both draws"); + Require(StorageTexture::FindPending(next, 2u * LayerBytes) == neighbour, "a 2D texture over slice 0 of the first array stored the next target's results, which only its unused second slice overlaps"); + LoadLayer(device, next, 0, swizzleMode, 1u); + RequireOutput(drawn, drawn, "slice 0 of the next array after the draw into the first"); + LoadLayer(device, surface, 1, swizzleMode, 1u); + RequireOutput(drawn, drawn, "slice 1 of the first array (the next target's slice 0) read while the next target's results were pending"); + LoadLayer(device, surface, 0, swizzleMode, 1u); + RequireOutput(drawn, cpuWrite ? std::bit_cast(0.125f) : Initial, "slice 0 of the first array after both draws"); + LoadLayer(device, next, 1, swizzleMode, 1u); + RequireOutput(Initial, Initial, "slice 1 of the next array, which nothing drew"); + StorageTexture::FlushPending(surface, SurfaceBytes, nullptr, "test"); + device.WaitIdle(); + AgcDriver::Graphics::ClearCachedTextures(device.Device()); +} + } int main() { @@ -319,6 +466,11 @@ int main() { Run(*device, block.Data(), TileR64KBX); Run(*device, block.Data(), static_cast(AgcDriver::Graphics::ColorTileMode::D64KBX)); for (std::uint32_t slice = 1; slice < Layers; ++slice) Run(*device, block.Data(), TileR64KBX, true, slice); + UnregisteredUnusedLayer(*device, 0u); + UnregisteredUnusedLayer(*device, Layers - 1u); + RetiredArrayLayer(*device); + UncommittedUnusedLayers(*device, 0u); + UncommittedUnusedLayers(*device, Layers - 1u); std::puts("array layer target tests passed"); return 0; } catch (const std::exception& error) {