From 249ef873b3842e61c686db0d00f708c64b1f72d2 Mon Sep 17 00:00:00 2001 From: blawesom Date: Sat, 17 Oct 2026 10:28:37 +0200 Subject: [PATCH 1/2] feat(libSceAgcDriver): import scattered direct memory as a dma-buf run list and probe its coherence Ports #4449 onto main's sharedBacking import structure, per the maintainer direction that this needs a port rather than a rebase: - GuestArenaSharedBackingRuns resolves a guest range into the runs of its direct memory backing (one per physical allocation, up to udmabuf's 1024), registered by DirectMemory; a single run still takes main's chunked and sparse paths, several runs create one udmabuf buffer per run through UDMABUF_CREATE_LIST - the start-up probe stores to the import from the GPU after CPU stores to the same pages and counts pages the CPU still reads stale (i915 maps dma-buf without snooping CPU caches); an incoherent measurement keeps the fallback off - decideImportWatch no longer returns early when there is no write watch: the coherence probe still runs there (its stale measurement is a plain CPU read after a GPU fill), so such devices measure instead of skipping the dma-buf fallback - the single shared backing resolver stays as a fallback for registrations that do not provide runs Co-authored-by: Mouad-Scriptz-0 <188722908+Mouad-Scriptz-0@users.noreply.github.com> --- .../Graphics/include/GuestBufferMemory.hpp | 1 + .../Graphics/src/GuestBufferMemory.cpp | 147 ++++++++++++------ core/libs/prx/libc/include/GuestArena.hpp | 7 + core/libs/prx/libc/src/GuestArena.cpp | 10 ++ .../libkernel/DirectMemory/DirectMemory.cpp | 34 ++++ core/libs/tests/GuestMemory.cpp | 52 +++++++ docs/dev/debt/libSceAgcDriver.md | 1 + 7 files changed, 201 insertions(+), 51 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 9622fb93a1b..4bc35c2dc69 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -60,6 +60,7 @@ struct ImportProbe { std::uint32_t writtenAfterSettle = 0; std::uint32_t writtenByCpu = 0; std::uint32_t writtenAfterFill = 0; + std::uint32_t staleAfterFill = 0; }; ImportProbe ProbeImportWriteProtection(const Context& context); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 6f6b03fb6e4..cc09947ab6a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -23,6 +23,8 @@ #include #endif #include +#include +#include #include #include #include @@ -221,6 +223,9 @@ struct HostImports { VkDevice watchDevice = VK_NULL_HANDLE; bool unwatchImports = false; bool unwatchDmaBufImports = false; + // dma-buf imports are assumed CPU-coherent until the probe measures otherwise, so the + // fallback keeps main's pre-probe behaviour and only an incoherent measurement skips it. + bool dmaBufImportsCoherent = true; bool watchPrivateImports = false; std::map settles; #ifdef _WIN32 @@ -480,18 +485,30 @@ int udmabufDevice() { return device; } -const char* createDmaBufImport(const Context& context, HostImport& entry, int file, std::uint64_t offset, VkResult& failure) { - udmabuf_create request{}; - request.memfd = static_cast(file); - request.flags = UDMABUF_FLAGS_CLOEXEC; - request.offset = offset; - request.size = entry.bytes; +constexpr std::size_t DmaBufRunLimit = 1024; + +const char* createDmaBufImport(const Context& context, HostImport& entry, int file, std::span runs, VkResult& failure) { const int device = udmabufDevice(); - const int buffer = device < 0 ? -1 : ioctl(device, UDMABUF_CREATE, &request); + int buffer = -1; + if (device >= 0 && runs.size() == 1) { + udmabuf_create request{}; + request.memfd = static_cast(file); + request.flags = UDMABUF_FLAGS_CLOEXEC; + request.offset = runs[0].offset; + request.size = runs[0].bytes; + buffer = ioctl(device, UDMABUF_CREATE, &request); + } else if (device >= 0) { + std::vector storage(sizeof(udmabuf_create_list) + runs.size() * sizeof(udmabuf_create_item)); + auto* request = reinterpret_cast(storage.data()); + request->flags = UDMABUF_FLAGS_CLOEXEC; + request->count = static_cast(runs.size()); + for (std::size_t index = 0; index < runs.size(); ++index) request->list[index] = udmabuf_create_item{static_cast(file), 0, runs[index].offset, runs[index].bytes}; + buffer = ioctl(device, UDMABUF_CREATE_LIST, request); + } close(file); if (buffer < 0) { failure = VK_ERROR_INVALID_EXTERNAL_HANDLE; - return device < 0 ? "open /dev/udmabuf" : "UDMABUF_CREATE"; + return device < 0 ? "open /dev/udmabuf" : runs.size() == 1 ? "UDMABUF_CREATE" : "UDMABUF_CREATE_LIST"; } VkMemoryFdPropertiesKHR properties{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR}; if (const auto result = context.Function("vkGetMemoryFdPropertiesKHR")(context.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, buffer, &properties); result != VK_SUCCESS) { @@ -752,35 +769,38 @@ bool createSparseDmaBufImport(const Context& context, HostImport& entry, int fil #endif #ifndef _WIN32 -struct BackingFile { - BackingFile() = default; - BackingFile(const BackingFile&) = delete; - BackingFile& operator=(const BackingFile&) = delete; - ~BackingFile() { - if (file >= 0) close(file); - } - int file = -1; - std::uint64_t offset = 0; -}; - -bool sharedBacking(const Context& context, const HostImport& entry, BackingFile& backing) { - return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &backing.file, &backing.offset); -} - -const char* createBackingImport(const Context& context, HostImport& entry, BackingFile& backing, VkResult& failure) { +const char* createBackingImport(const Context& context, HostImport& entry, int file, const GuestArena::SharedBackingRun* runs, std::size_t count, VkResult& failure) { bool fits = false; VkResult chunkFailure = VK_SUCCESS; - if (createChunkedDmaBufImport(context, entry, backing.file, backing.offset, fits, chunkFailure) == nullptr && fits) return nullptr; - if (!fits && createSparseDmaBufImport(context, entry, backing.file, backing.offset)) return nullptr; - return createDmaBufImport(context, entry, std::exchange(backing.file, -1), backing.offset, failure); + if (count == 1 && createChunkedDmaBufImport(context, entry, file, runs[0].offset, fits, chunkFailure) == nullptr && fits) return nullptr; + if (count == 1 && !fits && createSparseDmaBufImport(context, entry, file, runs[0].offset)) return nullptr; + return createDmaBufImport(context, entry, file, std::span(runs, count), failure); +} + +const char* createSharedBackingImport(const Context& context, HostImport& entry, VkResult& failure) { + int file = -1; + std::array runs{}; + std::size_t count = 0; + if (GuestArena::GuestArenaSharedBackingRuns_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &file, runs.data(), runs.size(), &count)) { + return createBackingImport(context, entry, file, runs.data(), count, failure); + } + // No runs resolver is registered: fall back to the single shared backing as one run. + std::uint64_t offset = 0; + if (GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &file, &offset)) { + const GuestArena::SharedBackingRun run{offset, static_cast(entry.bytes)}; + return createBackingImport(context, entry, file, &run, 1, failure); + } + failure = VK_ERROR_INVALID_EXTERNAL_HANDLE; + return "shared backing runs"; } #endif -const char* createImport(const Context& context, HostImport& entry, VkResult& failure) { +const char* createImport(const Context& context, HostImport& entry, VkResult& failure, bool dmaBufFallback) { const char* step = createHostPointerImport(context, entry, failure); #ifndef _WIN32 - BackingFile backing; - if (step != nullptr && sharedBacking(context, entry, backing)) step = createBackingImport(context, entry, backing, failure); + if (step != nullptr && dmaBufFallback && context.dmaBufImport) step = createSharedBackingImport(context, entry, failure); +#else + static_cast(dmaBufFallback); #endif return step; } @@ -1118,7 +1138,19 @@ void decideImportWatch(const Context& context, HostImports& state) { state.watchDevice = context.device; state.unwatchImports = false; state.unwatchDmaBufImports = false; - if (context.hostImportAlignment == 0 || !GuestMemory::WriteWatched()) return; + if (context.hostImportAlignment == 0) return; + if (!GuestMemory::WriteWatched()) { + // Without a write watch the write-protection probes cannot measure, but the dma-buf + // coherence probe still runs: its stale measurement is a plain CPU read after a GPU + // fill. Leaving it unmeasured kept dmaBufImportsCoherent at its default and skipped + // the dma-buf fallback, which forced the copy path on these devices. + if (context.dmaBufImport) { + const auto dmaBuf = ProbeDmaBufImportWriteProtection(context); + if (dmaBuf.failure == nullptr) state.dmaBufImportsCoherent = dmaBuf.staleAfterFill == 0; + std::fprintf(stderr, "[write-watch] no write watch: host imports are compared; dma-buf imports %s CPU-coherent (%u of %u pages stale after a GPU store over CPU writes)\n", state.dmaBufImportsCoherent ? "are" : "are not", dmaBuf.staleAfterFill, dmaBuf.pages); + } + return; + } if (request == ImportWatchRequest::Watch) { std::fprintf(stderr, "[write-watch] host imports stay watched (APS5_WRITE_WATCH_IMPORTS=watch)\n"); return; @@ -1146,7 +1178,8 @@ void decideImportWatch(const Context& context, HostImports& state) { return; } state.unwatchDmaBufImports = dmaBuf.writtenAfterSubmit != 0 || dmaBuf.writtenAfterFill != 0 || dmaBuf.writtenByCpu == 0; - std::fprintf(stderr, "[write-watch] dma-buf imports keep write protection: %s (%u of %u scratch pages written after a GPU read, %u after a GPU write, %u after the import, %u seen after a CPU store); imported ranges %s\n", state.unwatchDmaBufImports ? "no" : "yes", dmaBuf.writtenAfterSubmit, dmaBuf.pages, dmaBuf.writtenAfterFill, dmaBuf.writtenAtImport, dmaBuf.writtenByCpu, state.unwatchDmaBufImports ? "are compared" : "stay watched"); + state.dmaBufImportsCoherent = dmaBuf.staleAfterFill == 0; + std::fprintf(stderr, "[write-watch] dma-buf imports keep write protection: %s (%u of %u scratch pages written after a GPU read, %u after a GPU write, %u after the import, %u seen after a CPU store, %u stale after a GPU store over CPU writes); imported ranges %s%s\n", state.unwatchDmaBufImports ? "no" : "yes", dmaBuf.writtenAfterSubmit, dmaBuf.pages, dmaBuf.writtenAfterFill, dmaBuf.writtenAtImport, dmaBuf.writtenByCpu, dmaBuf.staleAfterFill, state.unwatchDmaBufImports ? "are compared" : "stay watched", state.dmaBufImportsCoherent ? "" : "; dma-buf imports are not CPU-coherent and are not used"); #endif } @@ -1539,7 +1572,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import const bool watchedDmaBuf = !state.unwatchDmaBufImports; #endif GuestMemory::ImportWatched(base, bytes, [&] { - step = watchedDmaBuf == watchedHostPointer ? createImport(context, entry, result) : createHostPointerImport(context, entry, result); + step = watchedDmaBuf == watchedHostPointer ? createImport(context, entry, result, state.dmaBufImportsCoherent) : createHostPointerImport(context, entry, result); #ifdef _WIN32 if (step == nullptr && settled && (step = settleImport(context, state, entry, result)) != nullptr) { destroyImport(context, entry); @@ -1550,11 +1583,16 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import return step == nullptr; }, watchedHostPointer); #ifndef _WIN32 - if (BackingFile backing; step != nullptr && watchedDmaBuf != watchedHostPointer && sharedBacking(context, entry, backing)) { - GuestMemory::ImportWatched(base, bytes, [&] { - step = createBackingImport(context, entry, backing, result); - return step == nullptr; - }, watchedDmaBuf); + if (step != nullptr && watchedDmaBuf != watchedHostPointer) { + int runsFile = -1; + std::array runs{}; + std::size_t count = 0; + if (GuestArena::GuestArenaSharedBackingRuns_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &runsFile, runs.data(), runs.size(), &count)) { + GuestMemory::ImportWatched(base, bytes, [&] { + step = createBackingImport(context, entry, runsFile, runs.data(), count, result); + return step == nullptr; + }, watchedDmaBuf); + } } if (step == nullptr && entry.dmaBuf && bytes >= settleFloorBytes && context.settleQueue != VK_NULL_HANDLE && context.settleQueueMutex != nullptr) { static const bool traceSettle = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; @@ -2749,7 +2787,7 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by HostImport import{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; VkBuffer destination = VK_NULL_HANDLE; VkDeviceMemory destinationMemory = VK_NULL_HANDLE; - VkCommandBuffer commands[2]{}; + VkCommandBuffer commands[3]{}; VkFence fence = VK_NULL_HANDLE; bool submitted = false; const auto run = [&]() -> const char* { @@ -2780,7 +2818,7 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; allocate.commandPool = context.pool; allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - allocate.commandBufferCount = 2; + allocate.commandBufferCount = 3; if ((probe.result = context.Function("vkAllocateCommandBuffers")(context.device, &allocate, commands)) != VK_SUCCESS) return "vkAllocateCommandBuffers"; VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; if ((probe.result = context.Function("vkBeginCommandBuffer")(commands[0], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer"; @@ -2791,6 +2829,9 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by if ((probe.result = context.Function("vkBeginCommandBuffer")(commands[1], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer"; context.Function("vkCmdFillBuffer")(commands[1], import.buffer, 0, bytes, 0x02020202u); if ((probe.result = context.Function("vkEndCommandBuffer")(commands[1])) != VK_SUCCESS) return "vkEndCommandBuffer"; + if ((probe.result = context.Function("vkBeginCommandBuffer")(commands[2], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer"; + context.Function("vkCmdFillBuffer")(commands[2], import.buffer, 0, bytes, 0x05050505u); + if ((probe.result = context.Function("vkEndCommandBuffer")(commands[2])) != VK_SUCCESS) return "vkEndCommandBuffer"; const VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; if ((probe.result = context.Function("vkCreateFence")(context.device, &fenceInfo, nullptr, &fence)) != VK_SUCCESS) return "vkCreateFence"; const auto execute = [&](VkCommandBuffer buffer) -> const char* { @@ -2812,12 +2853,17 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by if (!collect(probe.writtenAfterFill)) return "the collect after the fill"; *reinterpret_cast(base) = 2; if (!collect(probe.writtenByCpu)) return "the collect after a CPU store"; + for (std::uint64_t offset = 0; offset < bytes; offset += 4096) reinterpret_cast(base)[offset] = 4; + if (const char* step = execute(commands[2])) return step; + for (std::uint64_t offset = 0; offset < bytes; offset += 4096) { + if (reinterpret_cast(base)[offset] != 5) ++probe.staleAfterFill; + } return nullptr; }; probe.failure = run(); if (submitted) context.Function("vkQueueWaitIdle")(context.queue); if (fence != VK_NULL_HANDLE) context.Function("vkDestroyFence")(context.device, fence, nullptr); - if (commands[0] != VK_NULL_HANDLE) context.Function("vkFreeCommandBuffers")(context.device, context.pool, 2, commands); + if (commands[0] != VK_NULL_HANDLE) context.Function("vkFreeCommandBuffers")(context.device, context.pool, 3, commands); if (destination != VK_NULL_HANDLE) context.Function("vkDestroyBuffer")(context.device, destination, nullptr); if (destinationMemory != VK_NULL_HANDLE) context.Function("vkFreeMemory")(context.device, destinationMemory, nullptr); if (import.buffer != VK_NULL_HANDLE) destroyImport(context, import); @@ -2862,7 +2908,7 @@ ImportProbe ProbeImportWriteProtection(const Context& context) { std::uint32_t touched = 0; if (!collect(touched)) probe.failure = "GetWriteWatch"; else if (context.settleQueue == VK_NULL_HANDLE || context.settleQueueMutex == nullptr) probe.failure = "the settle queue"; - else runImportProbe(context, base, bytes, [&](HostImport& import, VkResult& result) { return createImport(context, import, result); }, collect, probe, [&](HostImport& import, VkResult& result) { return settleImport(context, Imports(), import, result); }); + else runImportProbe(context, base, bytes, [&](HostImport& import, VkResult& result) { return createImport(context, import, result, false); }, collect, probe, [&](HostImport& import, VkResult& result) { return settleImport(context, Imports(), import, result); }); VirtualFree(raw, 0, MEM_RELEASE); return probe; #else @@ -2870,10 +2916,8 @@ ImportProbe ProbeImportWriteProtection(const Context& context) { probe.failure = "host import support"; return probe; } - if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { - probe.failure = "the write watch"; - return probe; - } + // Without a write watch the write-protection fields stay empty (the collect reports none), + // but the coherence measurement still runs: a plain CPU read after a GPU fill. constexpr std::uint64_t page = 4096; const std::uint64_t alignment = std::max(context.hostImportAlignment, page); const std::uint64_t bytes = (4 * page + alignment - 1) / alignment * alignment; @@ -2886,9 +2930,9 @@ ImportProbe ProbeImportWriteProtection(const Context& context) { const auto base = (reinterpret_cast(raw) + alignment - 1) / alignment * alignment; auto* scratch = reinterpret_cast(base); for (std::uint64_t offset = 0; offset < bytes; offset += page) scratch[offset] = 1; - GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), bytes); - runImportProbe(context, base, bytes, [&](HostImport& import, VkResult& result) { return createImport(context, import, result); }, [&](std::uint32_t& pages) { return collectWriteWatch(base, bytes, pages); }, probe); - GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), bytes); + if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), bytes); + runImportProbe(context, base, bytes, [&](HostImport& import, VkResult& result) { return createImport(context, import, result, false); }, [&](std::uint32_t& pages) { return collectWriteWatch(base, bytes, pages); }, probe); + if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), bytes); munmap(raw, bytes + alignment); return probe; #endif @@ -2935,8 +2979,9 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context) { result = VK_ERROR_INITIALIZATION_FAILED; return "dup"; } - return createDmaBufImport(context, import, copy, 0, result); - }, [&](std::uint32_t& pages) { return collectWriteWatch(base, bytes, pages); }, probe); + const GuestArena::SharedBackingRun whole{0, import.bytes}; + return createDmaBufImport(context, import, copy, std::span(&whole, 1), result); + }, [&](std::uint32_t& pages) { return GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix() ? collectWriteWatch(base, bytes, pages) : (pages = 0, true); }, probe); GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), bytes); munmap(mapped, bytes); close(file); diff --git a/core/libs/prx/libc/include/GuestArena.hpp b/core/libs/prx/libc/include/GuestArena.hpp index a1ad24d339e..76e8a3e6085 100644 --- a/core/libs/prx/libc/include/GuestArena.hpp +++ b/core/libs/prx/libc/include/GuestArena.hpp @@ -11,6 +11,11 @@ namespace GuestArena { #ifndef _WIN32 using SharedBackingResolver = bool (*)(std::uintptr_t address, std::size_t bytes, int* file, std::uint64_t* offset); using SharedBackingWriter = bool (*)(std::uintptr_t address, const void* source, std::size_t bytes); +struct SharedBackingRun { + std::uint64_t offset; + std::size_t bytes; +}; +using SharedBackingRunsResolver = bool (*)(std::uintptr_t address, std::size_t bytes, int* file, SharedBackingRun* runs, std::size_t capacity, std::size_t* count); #endif extern "C" { @@ -52,6 +57,8 @@ void GuestArenaSetSharedBacking_nid_postfix(SharedBackingResolver resolver); void GuestArenaSetSharedBackingWriter_nid_no_patch(SharedBackingWriter writer); bool GuestArenaWriteSharedBacking_nid_no_patch(std::uintptr_t address, const void* source, std::size_t bytes); bool GuestArenaSharedBacking_nid_postfix(std::uintptr_t address, std::size_t bytes, int* file, std::uint64_t* offset); +void GuestArenaSetSharedBackingRuns_nid_postfix(SharedBackingRunsResolver resolver); +bool GuestArenaSharedBackingRuns_nid_postfix(std::uintptr_t address, std::size_t bytes, int* file, SharedBackingRun* runs, std::size_t capacity, std::size_t* count); #endif bool GuestArenaBeginHostWrite_nid_postfix(void* pointer, std::size_t bytes); void GuestArenaEndHostWrite_nid_postfix(void* pointer, std::size_t bytes); diff --git a/core/libs/prx/libc/src/GuestArena.cpp b/core/libs/prx/libc/src/GuestArena.cpp index ca580fc5bdd..6bb82c3d538 100644 --- a/core/libs/prx/libc/src/GuestArena.cpp +++ b/core/libs/prx/libc/src/GuestArena.cpp @@ -500,6 +500,7 @@ namespace { std::atomic sharedBackingResolver{nullptr}; std::atomic sharedBackingWriter{nullptr}; +std::atomic sharedBackingRunsResolver{nullptr}; } @@ -520,6 +521,15 @@ bool GuestArenaSharedBacking_nid_postfix(std::uintptr_t address, std::size_t byt const auto resolver = sharedBackingResolver.load(std::memory_order_acquire); return resolver != nullptr && resolver(address, bytes, file, offset); } + +void GuestArenaSetSharedBackingRuns_nid_postfix(SharedBackingRunsResolver resolver) { + sharedBackingRunsResolver.store(resolver, std::memory_order_release); +} + +bool GuestArenaSharedBackingRuns_nid_postfix(std::uintptr_t address, std::size_t bytes, int* file, SharedBackingRun* runs, std::size_t capacity, std::size_t* count) { + const auto resolver = sharedBackingRunsResolver.load(std::memory_order_acquire); + return resolver != nullptr && resolver(address, bytes, file, runs, capacity, count); +} #endif } diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index 4ceb6dae706..e66b5627848 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -546,6 +546,39 @@ bool SharedBacking(std::uintptr_t address, std::size_t bytes, int* file, std::ui return true; } +bool SharedBackingRuns(std::uintptr_t address, std::size_t bytes, int* file, GuestArena::SharedBackingRun* runs, std::size_t capacity, std::size_t* count) { + std::lock_guard lock(g_directLock); + *count = 0; + const auto next = g_directMappings.upper_bound(address); + if (bytes == 0 || bytes > std::numeric_limits::max() - address || next == g_directMappings.begin()) return false; + auto it = std::prev(next); + if (address >= it->second.end) return false; + const int backingFile = it->second.backing->File(); + const auto end = address + bytes; + std::size_t filled = 0; + for (auto cursor = address; cursor < end;) { + if (it == g_directMappings.end() || it->first > cursor || it->second.backing->File() != backingFile) return false; + const auto phys = it->second.phys + (cursor - it->first); + const auto page = g_physPages.find(phys - phys % PS5_PAGE_SIZE); + if (page == g_physPages.end() || page->second.backing != it->second.backing) return false; + const auto stop = std::min(end, static_cast(it->second.end)); + const auto offset = it->second.backing->FileOffset() + page->second.offset + phys % PS5_PAGE_SIZE; + if (filled != 0 && runs[filled - 1].offset + runs[filled - 1].bytes == offset) { + runs[filled - 1].bytes += stop - cursor; + } else { + if (filled == capacity) return false; + runs[filled++] = GuestArena::SharedBackingRun{offset, static_cast(stop - cursor)}; + } + cursor = stop; + ++it; + } + const int duplicate = fcntl(backingFile, F_DUPFD_CLOEXEC, 0); + if (duplicate < 0) throw std::system_error(errno, std::generic_category(), "duplicate direct memory backing"); + *file = duplicate; + *count = filled; + return true; +} + bool WriteSharedBacking(std::uintptr_t address, const void* source, std::size_t bytes) { if (bytes == 0 || bytes > std::numeric_limits::max() - address) return false; std::lock_guard lock(g_directLock); @@ -1210,6 +1243,7 @@ void CreateDirectMemoryBacking(int64_t start, size_t len, int memoryType) { static std::once_flag registered; std::call_once(registered, [] { GuestArena::GuestArenaSetSharedBacking_nid_postfix(&SharedBacking); + GuestArena::GuestArenaSetSharedBackingRuns_nid_postfix(&SharedBackingRuns); GuestArena::GuestArenaSetSharedBackingWriter_nid_no_patch(&WriteSharedBacking); }); #endif diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 21394b5e72c..2f74df99492 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -2202,6 +2202,57 @@ static void CheckDirectMemoryBackingNeedsNoFilesystem() { Require(sceKernelReleaseDirectMemory(phys, page) == 0); } +static void CheckDirectMemorySharedBackingRuns() { + constexpr std::size_t page = 0x4000; + std::int64_t first = 0; + std::int64_t second = 0; + std::int64_t third = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 2, 0, 0, &first) == 0); + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &second) == 0); + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &third) == 0); + void* reserved = nullptr; + Require(sceKernelReserveVirtualRange(&reserved, page * 4, 0, 0) == 0 && reserved != nullptr); + auto* bytes = static_cast(reserved); + constexpr int fixed = 0x10; + void* piece = bytes; + Require(sceKernelMapDirectMemory(&piece, page * 2, 3, fixed, first, 0) == 0 && piece == bytes); + piece = bytes + page * 2; + Require(sceKernelMapDirectMemory(&piece, page, 3, fixed, second, 0) == 0 && piece == bytes + page * 2); + piece = bytes + page * 3; + Require(sceKernelMapDirectMemory(&piece, page, 3, fixed, third, 0) == 0 && piece == bytes + page * 3); + bytes[page * 2 + 7] = 0x6b; + const auto address = reinterpret_cast(reserved); + int file = -1; + GuestArena::SharedBackingRun runs[4]{}; + std::size_t count = 0; + Require(GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, page * 4, &file, runs, 4, &count)); + const bool adjacent = static_cast(second) == static_cast(first) + page * 2; + Require(count == (adjacent ? 2u : 3u) || (adjacent && static_cast(third) == static_cast(second) + page && count == 1)); + std::uint64_t total = 0; + for (std::size_t index = 0; index < count; ++index) total += runs[index].bytes; + Require(total == page * 4 && runs[0].offset == static_cast(first)); + unsigned char byte = 0; + std::uint64_t secondOffset = 0; + std::uint64_t covered = 0; + for (std::size_t index = 0; index < count; ++index) { + if (covered <= page * 2 && page * 2 < covered + runs[index].bytes) secondOffset = runs[index].offset + (page * 2 - covered); + covered += runs[index].bytes; + } + Require(secondOffset == static_cast(second)); + Require(pread(file, &byte, 1, static_cast(secondOffset + 7)) == 1 && byte == 0x6b); + close(file); + Require(GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address + page, page * 3, &file, runs, 4, &count) && runs[0].offset == static_cast(first) + page); + close(file); + Require(!GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, page * 4, &file, runs, 1, &count) || count == 1); + Require(!GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, page * 5, &file, runs, 4, &count)); + Require(!GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, 0, &file, runs, 4, &count)); + Require(sceKernelMunmap(reserved, page * 4) == 0); + Require(sceKernelReleaseDirectMemory(first, page * 2) == 0); + Require(sceKernelReleaseDirectMemory(second, page) == 0); + Require(sceKernelReleaseDirectMemory(third, page) == 0); +} + + static void CheckRegularFileMapping() { constexpr std::size_t page = 0x4000; const auto path = std::filesystem::path("anyps5-file-mapping-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".bin"); @@ -2847,6 +2898,7 @@ int main(int argc, char** argv) { CheckDirectMemoryWriteWatchFollowsRemovedMappings(); CheckMappingCostFollowsTheMappingCount(); CheckDirectMemoryBackingNeedsNoFilesystem(); + CheckDirectMemorySharedBackingRuns(); CheckDirectMemorySharedBacking(); CheckVirtualQueryFindsNeighbours(); CheckReleaseFindsEveryDirectMapping(); diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index bc49a0d84ad..4363e5340b8 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -177,3 +177,4 @@ - [Unclamped normalized color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - CB_COLOR_INFO targets with BLEND_CLAMP clear (bit 15) on UNORM and SNORM formats are accepted when blending is disabled (ENABLE clear or BLEND_BYPASS set). Vulkan converts unblended shader values and clamps to the normalized range automatically; blending into an unclamped normalized target remains unmodeled because Vulkan always clamps inputs prior to blending. - [Thick SW_4KB_S_X volumes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp) use AMD addrlib's GFX10 non-RB+ 16-pipe `GFX10_SW_4K_S3_X_PATINFO` equations for 1, 2, 4, 8 and 16-byte elements, including packed mip tails. The layout follows [AMD PAL c5e80007](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/gfx10/gfx10addrlib.cpp); it has not been measured on console hardware. Other thick swizzle modes outside the supported families still throw. Mipmapped 3D color targets remain refused by the color-register decoder. +- [dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) - the udmabuf of a guest range lists the runs of the direct memory backing file behind it (`SharedBackingRuns`, one per physical allocation the range was mapped from, up to udmabuf's 1024), where it took the first run's offset for the whole range. The start-up probe also stores to the import from the GPU after CPU stores to the same pages and counts pages the CPU still reads stale: i915 maps a dma-buf without snooping the CPU caches (4 of 4 pages stale on Meteor Lake with Mesa 26.2.3 and main), so such a device does not fall back to dma-buf imports and copies instead. The memory type the import reports is host-coherent for the driver's own mappings; the guest's mapping of the same pages is what goes stale. From ca1d4821163b8704a61c5f3353bf7ef5daea5a98 Mon Sep 17 00:00:00 2001 From: Mouad-Scriptz-0 Date: Sun, 18 Oct 2026 12:15:28 +0000 Subject: [PATCH 2/2] fix(libSceAgcDriver): run the dma-buf coherence probe without a write watch and gate the mixed-watch import on it ProbeDmaBufImportWriteProtection returned before measuring when the write watch was unavailable, so on Linux the no-write-watch branch of decideImportWatch never measured and dmaBufImportsCoherent kept its default. The probe now registers and collects the watch only where it exists, as the host probe does, and that branch logs when the probe failed instead of reporting an unmeasured device as coherent. The mixed-watch branch of importRunAt created a dma-buf import without consulting dmaBufImportsCoherent; it now requires it. CheckDirectMemorySharedBackingRuns closes the duplicated file of the capacity-1 call when that call succeeds. Co-Authored-By: Claude Fable 5.1 --- .../Graphics/src/GuestBufferMemory.cpp | 18 +++++++++--------- core/libs/tests/GuestMemory.cpp | 4 +++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index cc09947ab6a..936392ca91c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -1146,8 +1146,12 @@ void decideImportWatch(const Context& context, HostImports& state) { // the dma-buf fallback, which forced the copy path on these devices. if (context.dmaBufImport) { const auto dmaBuf = ProbeDmaBufImportWriteProtection(context); - if (dmaBuf.failure == nullptr) state.dmaBufImportsCoherent = dmaBuf.staleAfterFill == 0; - std::fprintf(stderr, "[write-watch] no write watch: host imports are compared; dma-buf imports %s CPU-coherent (%u of %u pages stale after a GPU store over CPU writes)\n", state.dmaBufImportsCoherent ? "are" : "are not", dmaBuf.staleAfterFill, dmaBuf.pages); + if (dmaBuf.failure != nullptr) { + std::fprintf(stderr, "[write-watch] no write watch: host imports are compared; dma-buf coherence not measured (probe failed at %s, %d), dma-buf imports %s used\n", dmaBuf.failure, static_cast(dmaBuf.result), state.dmaBufImportsCoherent ? "are" : "are not"); + } else { + state.dmaBufImportsCoherent = dmaBuf.staleAfterFill == 0; + std::fprintf(stderr, "[write-watch] no write watch: host imports are compared; dma-buf imports %s CPU-coherent (%u of %u pages stale after a GPU store over CPU writes)\n", state.dmaBufImportsCoherent ? "are" : "are not", dmaBuf.staleAfterFill, dmaBuf.pages); + } } return; } @@ -1583,7 +1587,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import return step == nullptr; }, watchedHostPointer); #ifndef _WIN32 - if (step != nullptr && watchedDmaBuf != watchedHostPointer) { + if (step != nullptr && watchedDmaBuf != watchedHostPointer && state.dmaBufImportsCoherent) { int runsFile = -1; std::array runs{}; std::size_t count = 0; @@ -2949,10 +2953,6 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context) { probe.failure = "dma-buf import support"; return probe; } - if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { - probe.failure = "the write watch"; - return probe; - } constexpr std::uint64_t page = 4096; const std::uint64_t alignment = std::max(context.hostImportAlignment, page); const std::uint64_t bytes = (4 * page + alignment - 1) / alignment * alignment; @@ -2972,7 +2972,7 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context) { const auto base = reinterpret_cast(mapped); auto* scratch = reinterpret_cast(base); for (std::uint64_t offset = 0; offset < bytes; offset += page) scratch[offset] = 1; - GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), bytes); + if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), bytes); runImportProbe(context, base, bytes, [&](HostImport& import, VkResult& result) -> const char* { const int copy = dup(file); if (copy < 0) { @@ -2982,7 +2982,7 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context) { const GuestArena::SharedBackingRun whole{0, import.bytes}; return createDmaBufImport(context, import, copy, std::span(&whole, 1), result); }, [&](std::uint32_t& pages) { return GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix() ? collectWriteWatch(base, bytes, pages) : (pages = 0, true); }, probe); - GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), bytes); + if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), bytes); munmap(mapped, bytes); close(file); return probe; diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 2f74df99492..237af8552c5 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -2243,7 +2243,9 @@ static void CheckDirectMemorySharedBackingRuns() { close(file); Require(GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address + page, page * 3, &file, runs, 4, &count) && runs[0].offset == static_cast(first) + page); close(file); - Require(!GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, page * 4, &file, runs, 1, &count) || count == 1); + const bool oneRun = GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, page * 4, &file, runs, 1, &count); + if (oneRun) close(file); + Require(!oneRun || count == 1); Require(!GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, page * 5, &file, runs, 4, &count)); Require(!GuestArena::GuestArenaSharedBackingRuns_nid_postfix(address, 0, &file, runs, 4, &count)); Require(sceKernelMunmap(reserved, page * 4) == 0);