diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 029af5f1c8c..56eb2d534e1 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -703,6 +703,15 @@ if(BUILD_TESTING) add_test(NAME agc_robust_buffer_access COMMAND agc_robust_buffer_access_tests) endif() +add_test_executable(agc_draw_inputs_in_place_driver_tests EXCLUDE_FROM_ALL tests/DrawInputsInPlace.cpp) +target_include_directories(agc_draw_inputs_in_place_driver_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) +set_target_properties(agc_draw_inputs_in_place_driver_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_draw_inputs_in_place_driver_tests) +if(BUILD_TESTING) + set_target_properties(agc_draw_inputs_in_place_driver_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_draw_inputs_in_place_driver COMMAND agc_draw_inputs_in_place_driver_tests) +endif() + add_test_executable(agc_device_fault_tests EXCLUDE_FROM_ALL tests/DeviceFault.cpp) target_include_directories(agc_device_fault_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) set_target_properties(agc_device_fault_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -1422,6 +1431,8 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() set_tests_properties(agc_draw_inputs_in_place PROPERTIES SKIP_RETURN_CODE 77) + add_test(NAME agc_draw_inputs_copied COMMAND agc_driver_recorder_tests --draw-inputs-copied-only) + set_tests_properties(agc_draw_inputs_copied PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME agc_resident_present COMMAND agc_driver_recorder_tests --resident-present-only) set_tests_properties(agc_resident_present PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME agc_queued_write_groups COMMAND agc_driver_recorder_tests --queued-write-groups-only) @@ -1432,7 +1443,7 @@ if(BUILD_TESTING) add_test(NAME agc_metadata_pass COMMAND agc_driver_recorder_tests --metadata-pass-only) set_tests_properties(agc_metadata_pass PROPERTIES SKIP_RETURN_CODE 77) if(WIN32) - set_tests_properties(agc_recorder_read_list agc_host_import_unmap agc_parked_import_unmap agc_host_import_retirement agc_sparse_run_import agc_sparse_run_import_copied agc_unit_shadow agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_draw_snapshot_epoch agc_multisample_guest_access agc_draw_inputs_in_place agc_resident_present agc_slice_dirty agc_queued_write_groups agc_metadata_pass agc_tiled_resolve PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_recorder_read_list agc_host_import_unmap agc_parked_import_unmap agc_host_import_retirement agc_sparse_run_import agc_sparse_run_import_copied agc_unit_shadow agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_draw_snapshot_epoch agc_multisample_guest_access agc_draw_inputs_in_place agc_draw_inputs_copied agc_resident_present agc_slice_dirty agc_queued_write_groups agc_metadata_pass agc_tiled_resolve PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") add_test(NAME agc_host_import_second_mapping COMMAND agc_driver_recorder_tests --host-import-second-mapping-only) set_tests_properties(agc_host_import_second_mapping PROPERTIES SKIP_RETURN_CODE 77 ENVIRONMENT_MODIFICATION diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp new file mode 100644 index 00000000000..e43c70bfff2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp @@ -0,0 +1,14 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRAWINPUTSINPLACE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRAWINPUTSINPLACE_HPP + +#include + +namespace AgcDriver { + +inline bool DrawInputsInPlaceUsable(VkDriverId driver, bool windows) { + return !windows || driver != VK_DRIVER_ID_NVIDIA_PROPRIETARY; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 3326b66c8d1..c99661ca6c3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -16,6 +16,7 @@ #include "prx/libSceAgcDriver/Execution/include/DeviceFault.hpp" #include "prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp" #include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp" +#include "prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp" #include "prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp" #include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" @@ -343,6 +344,7 @@ struct VulkanDevice::State { bool dualSrcBlend = false; bool occlusionQueryPrecise = false; VkDeviceSize hostImportAlignment = 0; + bool drawInputsInPlace = true; bool dmaBufImport = false; bool sparseBinding = false; std::uint32_t queueFamily = 0; @@ -1375,6 +1377,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); state->hostImportAlignment = hostProperties.minImportedHostPointerAlignment; +#ifdef _WIN32 + constexpr bool windows = true; +#else + constexpr bool windows = false; +#endif + state->drawInputsInPlace = std::getenv("APS5_NO_DRAW_INPUTS_IN_PLACE") == nullptr && (std::getenv("APS5_DRAW_INPUTS_IN_PLACE") != nullptr || DrawInputsInPlaceUsable(driverProperties.driverID, windows)); if (!compileOnly) Graphics::SetHostImportWindow(8ull << 20u); #ifdef _WIN32 // Imported pages are locked; locking is bounded by the process working-set minimum. @@ -3679,6 +3687,7 @@ Graphics::Context VulkanDevice::buildContext() const { context.provokingVertexLast = state->provokingVertexLast; context.graphicsPipelineLibrary = state->graphicsPipelineLibrary; context.pipelineLinkTimeOptimization = state->pipelineLinkTimeOptimization; + context.drawInputsInPlace = state->drawInputsInPlace; context.provokingVertexModePerPipeline = state->provokingVertexModePerPipeline; context.drawIndirectCount = state->drawIndirectCount; context.occlusionQueryPrecise = state->occlusionQueryPrecise; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index eb28fe29226..9663e67853e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -161,6 +161,7 @@ struct Context { bool depthClamp = false; // Nonzero when VK_EXT_external_memory_host is enabled: the required host pointer alignment. VkDeviceSize hostImportAlignment = 0; + bool drawInputsInPlace = true; bool dmaBufImport = false; bool sparseBinding = false; std::uint32_t queueFamily = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 4a98d4d701d..1fb537a91b3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -976,7 +976,7 @@ void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCop const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment, bool* behindGpuWrite) { auto* recorder = context.recorder; - if (recorder == nullptr || bytes == 0) return nullptr; + if (recorder == nullptr || bytes == 0 || !context.drawInputsInPlace) return nullptr; bool behind = false; if (behindGpuWrite == nullptr) { GuestMemory::FlushGpuWrites(address, bytes); diff --git a/core/libs/prx/libSceAgcDriver/tests/DrawInputsInPlace.cpp b/core/libs/prx/libSceAgcDriver/tests/DrawInputsInPlace.cpp new file mode 100644 index 00000000000..5a329bb0dbb --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/DrawInputsInPlace.cpp @@ -0,0 +1,32 @@ +#include "prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp" +#include + +namespace { + +struct Case { + VkDriverId driver; + bool windows; + bool inPlace; +}; + +constexpr Case Cases[]{ + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, false}, + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, false, true}, + {VK_DRIVER_ID_AMD_PROPRIETARY, true, true}, + {VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, true, true}, + {VK_DRIVER_ID_MESA_RADV, false, true}, + {VK_DRIVER_ID_MESA_LLVMPIPE, false, true}, +}; + +} + +int main() { + int failures = 0; + for (const auto& item : Cases) { + if (AgcDriver::DrawInputsInPlaceUsable(item.driver, item.windows) != item.inPlace) { + std::fprintf(stderr, "driver %d on %s: expected draw inputs %s\n", static_cast(item.driver), item.windows ? "Windows" : "another host", item.inPlace ? "in place" : "copied"); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index ede470a6b52..ee1bce17320 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -4205,7 +4205,7 @@ void drawInputReuseTests(const Device& device, Recorder& recorder) { recorder.Sync(); } -void drawInputInPlaceTests(const Device& device, Recorder& recorder) { +void drawInputInPlaceTests(const Device& device, Recorder& recorder, bool copied = false) { auto context = device.GetContext(); constexpr std::size_t bytes = 65536; void* block = context.hostImportAlignment != 0 ? AllocateWatched(bytes, 65536) : nullptr; @@ -4240,6 +4240,12 @@ void drawInputInPlaceTests(const Device& device, Recorder& recorder) { context.recorder = &recorder; recorder.Sync(); const auto vertices = address + 4096; + context.drawInputsInPlace = !copied; + if (copied) { + Require(InPlaceDrawInput(context, vertices, 256, 4) == nullptr, "an imported draw input was bound in place where draw inputs are copied"); + Require(!recorder.PendingReadOverlaps(vertices, 256), "a copied draw input's read was noted as an in-place read"); + return; + } const auto* import = InPlaceDrawInput(context, vertices, 256, 4); Require(import != nullptr && import->base <= vertices && vertices + 256 <= import->base + import->bytes, "an imported draw input was not bound in place"); const auto read = recorder.DescribePendingRead(vertices, 256); @@ -9582,6 +9588,11 @@ int main(int argc, char** argv) { std::cout << "In-place draw input tests passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--draw-inputs-copied-only") { + drawInputInPlaceTests(device, recorder, true); + std::cout << "Copied draw input tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--multisample-only") { multisampleGuestAccessTests(device, recorder); fmaskDecompressMsaaTests(device, recorder); diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 23591563fbf..a54c68ac3bb 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -130,6 +130,7 @@ flowchart LR - A texture over a depth surface samples the depth image itself when it views one 2D slice of the plane's own format (`DepthSurfaceTexture` in [`DepthSurface.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp)). A view of several slices (a 2D array or a cube) or an integer view of a D16 plane is served by a `DepthPlaneCopy`, an image the slices are copied into at every use: from the depth surface bound at each slice's address, else from guest memory. Such a texture is never kept in the resource templates (`RefreshedPerUse`). - Guest memory reaches the GPU through host imports (`VK_EXT_external_memory_host`) where the driver accepts them, and through copies otherwise ([`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). A registered range larger than the import window (64 MiB) is imported per window for buffer and fill accesses and whole for an address-based build. Live imports stay under a budget, with the least recently used import retired first (see TechnicalDebt, Host import windows and budget). - A guest protect or unmap over part of a plain host-pointer import (one registered range, no alias, dma-buf, chunked or joined import, no pending settle) trims the import instead of retiring it (`retireOrTrim` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)): its buffer is destroyed and it waits in `HostImports::trimmed` with the side of its range the change left, the larger side for a change strictly inside the range, while its `VkDeviceMemory` keeps the whole range it was made for pinned (`HostImport::memoryBase` and `pinnedBytes`, which `liveBytes` and the eviction count), also after the guest protects, unmaps or maps again the pages that were cut off. The next `refreshImports` binds a new `VkBuffer` to the same memory at the offset of the largest registered range inside the kept side (`reattachTrimmed`) when that range still has the import's backing serial, overlaps no other import, is what a new import would cover and the buffer accepts the offset; otherwise, or when an import of a range overlapping the kept side is made first, the trimmed import is released and held through the recorder (`KeepThroughHold`) as a retired import is. `GuestAllocations::Range::backingSerial` tells whether a range's pages are still the ones that were imported: a new range gets a fresh serial, the untouched pieces of a split keep theirs, and every span a mutation passes to `RequireUnpinned` (a protect or unmap, a fixed mapping over registered memory, the other mappings of remapped direct memory, and for an unmap or remap the whole 2 MiB resident runs it touches, which it may demote to shared pages) gets a fresh one when the mutation ends. +- A draw binds its vertex and index ranges in place from a host import, except with NVIDIA's Windows driver, which reads a host import of guest memory far slower than a buffer it allocated (Astro Bot at Crash Site on an RTX 5070 Ti: 164 against 45 ms of draws per frame), so they are copied there (`DrawInputsInPlaceUsable`, `Context::drawInputsInPlace`). `APS5_DRAW_INPUTS_IN_PLACE=1` binds them in place and `APS5_NO_DRAW_INPUTS_IN_PLACE=1` copies them on any device; the second wins. - On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the settle queue, and once the batch's fence has signalled a later lookup returns the import's whole 64 KiB blocks to the write watch (`GuestMemory::Rewatch` drops their marks and stamps every block as a CPU store), unless a live storage image overlaps them. Such an import stays compared: an image that took its generation before the stamp would take the stamp for a CPU store over the results it produces after it, and drop them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. `APS5_NO_DEFERRED_SETTLES` (any value) keeps every import under 1 MiB compared. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. On Linux the settle queue is made when the device imports dma-buf memory, and every dma-buf import of 1 MiB or more is settled right after it is made, for another reason: the kernel maps an import's pages for the GPU at the first submission that uses it, which took about 2 s for a 3 GiB import on RADV and held up the title's frame. The import's buffer is shared with the settle queue's family instead of aliased, since a sparse import is bound to many memory objects. ### Log output diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 6c0cd7f8150..ace7ce8e5c7 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -114,6 +114,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_DRAIN_COMPLETION_LABELS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp` | | records completion labels immediately instead of deferring them with the batch | | `APS5_DRAW_CACHE_ENTRIES` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | 4096 | number of entries in the draw cache; 0 keeps the default 4096 | | `APS5_DRAW_DRAIN` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp` | | treats draw packets as drains, waiting for them instead of recording ahead | +| `APS5_DRAW_INPUTS_IN_PLACE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | binds draw vertex and index ranges in place from host imports also with NVIDIA's Windows driver, which copies them by default | | `APS5_DRAW_KEY_ALL_USER_WORDS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp` | | includes all 32 user words in a draw's register key instead of the used count | | `APS5_DRAW_SNAPSHOT_MAX_MIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | 64 | upper bound in MiB on the snapshot bytes a draw's bindings may take | | `APS5_DRAW_TRANSITIONS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | keeps the per-draw upload and download barriers and layout transitions instead of the lean recorded path | @@ -207,6 +208,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_DISPATCH_MOVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | leaves a dispatch-cache hit from moving its read-only buffers into the template, so it prepares the dispatch again; also off when APS5_NO_TEMPLATE_DATA_REFRESH is set | | `APS5_NO_DISPATCH_RECIPE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | builds no dispatch recipes, so dispatch-cache hits repeat the recorded work | | `APS5_NO_DRAW_DATA_HITS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | | disables draw-cache hits that differ only in ShaderData words | +| `APS5_NO_DRAW_INPUTS_IN_PLACE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | copies draw vertex and index ranges instead of binding them in place from host imports, on any device; wins over `APS5_DRAW_INPUTS_IN_PLACE` | | `APS5_NO_DRAW_KEY` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | | registers no draw key, so every draw decodes its programs instead of hitting the draw cache | | `APS5_NO_DRAW_KEY_TRIM` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | keys the render target and index ranges into the draw resource key instead of repeating the alias checks on a hit | | `APS5_NO_DRAW_PRECHECK` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp` | | disables the draw precheck that rejects empty or invalid draws early |