diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 92b900e7dad..e9ff7f60963 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -36,6 +36,8 @@ #include #include #include +#include +#include namespace AgcDriver::Graphics { @@ -597,12 +599,26 @@ std::uint64_t heapMirrorBudget() { return bytes; } -[[noreturn]] void heapMirrorFatal(const GuestAllocations::Range& range, std::uint64_t held, const char* reason) { - std::fprintf(stderr, "FATAL: heap mirror of 0x%llx+0x%llx: %s; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)\n", static_cast(range.address), static_cast(range.bytes), reason, static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); - std::fflush(stderr); - std::abort(); +std::string heapMirrorError(const GuestAllocations::Range& range, std::uint64_t held, std::string_view reason) { + constexpr std::string_view prefix = "AGC graphics: "; + if (reason.starts_with(prefix)) reason.remove_prefix(prefix.size()); + char where[64]; + std::snprintf(where, sizeof(where), "heap mirror of 0x%llx+0x%llx: ", static_cast(range.address), static_cast(range.bytes)); + char holding[96]; + std::snprintf(holding, sizeof(holding), "; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)", static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); + return std::string(prefix) + where + std::string(reason) + holding; } +std::uint64_t heapMirrorsBeside(const GuestAllocations::Range& range) { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + std::uint64_t held = state.heapBytes; + if (const auto found = state.entries.find(range.address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; + return held; +} + +void sweepMirrors(); + bool sameRange(const std::weak_ptr& mirrored, const std::shared_ptr& range) { return !mirrored.owner_before(range) && !range.owner_before(mirrored); } @@ -844,19 +860,20 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh const auto generation = GuestMemory::CollectWrites(range->address, range->bytes); constexpr std::uint64_t block = 65536; mirror->generations.assign(static_cast(((range->address + range->bytes + block - 1) / block) - range->address / block), generation); - std::lock_guard lock(state.mutex); if (generation == 0) { + std::lock_guard lock(state.mutex); ++state.heapUnwatched; return nullptr; } - std::uint64_t held = state.heapBytes; - if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; - if (held + range->bytes > heapMirrorBudget()) heapMirrorFatal(*range, held, "past the budget"); + if (heapMirrorsBeside(*range) + range->bytes > heapMirrorBudget()) { + sweepMirrors(); + if (const auto held = heapMirrorsBeside(*range); held + range->bytes > heapMirrorBudget()) throw std::runtime_error(heapMirrorError(*range, held, "past the budget")); + } } try { mirror->buffer = std::make_shared(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); } catch (const std::runtime_error& error) { - if (heap) heapMirrorFatal(*range, state.heapBytes, error.what()); + if (heap) throw std::runtime_error(heapMirrorError(*range, heapMirrorsBeside(*range), error.what())); std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast(range->address), static_cast(range->bytes), error.what()); std::lock_guard lock(state.mutex); state.failed.insert(range->address); diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index f803903fc0a..fadd6b8d413 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -11,8 +11,10 @@ #endif #include #include +#include #include #include +#include #include #include #include @@ -229,6 +231,71 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { GuestArena::GuestArenaRelease_nid_postfix(raw, size); } #endif + { + constexpr std::size_t large = 36 * 65536; + constexpr std::size_t small = 17 * 65536; + constexpr std::size_t total = 2 * large + small; + void* raw = GuestArena::GuestArenaAllocate_nid_postfix(total, 65536); +#ifdef _WIN32 + GuestArena::GuestArenaCommit_nid_postfix(raw, total, PAGE_READWRITE, total); +#endif + auto* const heap = static_cast(raw); + const std::array heaps{heap, heap + large, heap + 2 * large}; + const std::array sizes{large, large, small}; + for (std::size_t index = 0; index < heaps.size(); ++index) std::memset(heaps[index], 0x21 + static_cast(index), sizes[index]); + const auto change = [&](std::initializer_list added, std::initializer_list removed) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + for (const auto index : removed) GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, heaps[index]); + for (const auto index : added) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, heaps[index], sizes[index], true, false); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + const auto mirrored = [&](std::size_t index) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto begin = reinterpret_cast(heaps[index]); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& entry) { return entry.begin == begin && entry.end == begin + sizes[index]; }); + Require(found != ranges.end(), "a mirrored heap range is missing from the BDA table"); + const auto last = access.addressBytes(found->deviceAddress)[sizes[index] - 1]; + leased.WriteBack(); + return last; + }; + sweep(); + const auto before = MirrorCounters(); + change({0}, {}); + Require(mirrored(0) == std::byte{0x21}, "the first heap range was not mirrored"); + const auto one = MirrorCounters(); + access.limitMemory(large - 1); + change({1}, {0}); + Require(mirrored(1) == std::byte{0x22}, "a heap mirror past APS5_HEAP_MIRROR_MIB was not made in the memory of the expired mirror"); + const auto swept = MirrorCounters(); + Require(swept.heapMirrors == one.heapMirrors && swept.heapBytes == one.heapBytes, "the expired heap mirror was not swept before the allocation"); + access.limitMemory(small - 1); + change({2}, {}); + char expected[64]; + std::snprintf(expected, sizeof(expected), "heap mirror of 0x%llx+0x%llx: ", static_cast(reinterpret_cast(heaps[2])), static_cast(small)); + const auto attempts = access.allocationAttempts(); + bool refused = false; + try { + mirrored(2); + } catch (const std::runtime_error& error) { + const std::string what = error.what(); + refused = what.find(expected) != std::string::npos && what.find("Vulkan result -2") != std::string::npos; + } + Require(refused, "a heap mirror the memory cannot hold did not fail its build with the Vulkan result"); + Require(access.allocationAttempts() == attempts + 1, "a refused heap mirror was allocated again"); + Require(MirrorCounters().heapMirrors == swept.heapMirrors && MirrorCounters().heapBytes == swept.heapBytes, "a refused heap mirror was registered"); + access.limitMemory(std::nullopt); + Require(mirrored(2) == std::byte{0x23} && MirrorCounters().heapMirrors == swept.heapMirrors + 1, "a heap mirror was not made once the memory was free again"); + change({}, {1, 2}); + sweep(); + Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the heap mirrors outlived their ranges"); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(raw, total); +#endif + GuestArena::GuestArenaRelease_nid_postfix(raw, total); + } #ifdef _WIN32 GuestArena::GuestArenaReset_nid_postfix(block, bytes); #endif diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp index c18bd2d529d..0d90f1c5839 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp @@ -3,12 +3,15 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include +#include #include struct BdaTestAccess { std::function(VkBuffer)> bytes; std::function descriptor; std::function(VkDeviceAddress)> addressBytes; + std::function)> limitMemory; + std::function allocationAttempts; }; void RunBdaResourceTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index a035d3bf68b..5272700b5fd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1178,6 +1178,10 @@ struct MockVulkan { std::map allocationFlags; std::map bufferMemory; std::map> memories; + std::map allocationSizes; + VkDeviceSize allocatedBytes = 0; + std::optional memoryLimit; + std::uint64_t allocationAttempts = 0; std::vector layoutBindings; std::vector poolSizes; std::uint32_t poolMaxSets = 0; @@ -1217,8 +1221,12 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu } VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + ++mock.allocationAttempts; + if (mock.memoryLimit.has_value() && mock.allocatedBytes + info->allocationSize > *mock.memoryLimit) return VK_ERROR_OUT_OF_DEVICE_MEMORY; *memory = makeHandle(); mock.memories[*memory] = std::vector(info->allocationSize); + mock.allocationSizes[*memory] = info->allocationSize; + mock.allocatedBytes += info->allocationSize; if (info->pNext != nullptr) { const auto* flags = static_cast(info->pNext); mock.allocationFlags[*memory] = flags->flags; @@ -1246,7 +1254,11 @@ VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice, VkBuffer, const VkAllocat --mock.live; } -VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) { +VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory memory, const VkAllocationCallbacks*) { + if (const auto found = mock.allocationSizes.find(memory); found != mock.allocationSizes.end()) { + mock.allocatedBytes -= found->second; + mock.allocationSizes.erase(found); + } --mock.live; } @@ -2772,8 +2784,10 @@ void vertexCopyTests() { int main() { #ifdef _WIN32 _putenv_s("APS5_PIN_WAIT_MS", "200"); + _putenv_s("APS5_HEAP_MIRROR_MIB", "4"); #else setenv("APS5_PIN_WAIT_MS", "200", 1); + setenv("APS5_HEAP_MIRROR_MIB", "4", 1); #endif try { { @@ -2858,7 +2872,11 @@ int main() { const auto offset = address - 0x100000000000ULL; const auto buffer = reinterpret_cast(offset / 0x10000); return std::span(mock.memories.at(mock.bufferMemory.at(buffer))).subspan(offset % 0x10000); - } + }, + [](std::optional headroom) { + mock.memoryLimit = headroom.has_value() ? std::optional(mock.allocatedBytes + *headroom) : std::nullopt; + }, + [] { return mock.allocationAttempts; } }); Require(mock.live == 0, "BDA resources leaked Vulkan objects"); RunGuestAllocationTests();