diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 65f1aff6c34..6970183d983 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2060,6 +2060,24 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_partially_resident_pending_stores PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_shader_disk_cache_large_file COMMAND agc_shader_disk_cache_tests --large-file) + if(Python3_Interpreter_FOUND) + set(housekeepingModes plain) + if(WIN32) + list(APPEND housekeepingModes extended extended-long device) + else() + list(APPEND housekeepingModes plain-long) + endif() + foreach(housekeepingMode IN LISTS housekeepingModes) + set(housekeepingTest agc_shader_cache_housekeeping_${housekeepingMode}) + add_test(NAME ${housekeepingTest} COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_shader_cache_housekeeping.py + $ ${housekeepingMode}) + set_tests_properties(${housekeepingTest} PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(${housekeepingTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() + endforeach() + endif() set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES SKIP_RETURN_CODE 77 RUN_SERIAL TRUE WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 240) if(WIN32) set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index f335fb2c69d..67aa7463438 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -2259,6 +2259,14 @@ void verifyDefaultDirectory(const char* self) { int main(int argc, char** argv) { try { + if (argc == 2 && std::string_view(argv[1]) == "--housekeeping-safety") { + const auto directory = ShaderDiskCache::EntryDirectory(); + require(!directory.empty(), "the housekeeping fixture cache is disabled"); + ShaderDiskCache::Shutdown(); + require(ShaderDiskCache::Writers() == 0, "the housekeeping writer did not stop"); + std::cout << "housekeeping version " << directory.filename().string() << '\n'; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--resource-plan-codec") { verifyResultRoundTrip(); verifyEntryRoundTrip(); diff --git a/core/libs/prx/libSceAgcDriver/tests/test_shader_cache_housekeeping.py b/core/libs/prx/libSceAgcDriver/tests/test_shader_cache_housekeeping.py new file mode 100644 index 00000000000..ac8853b3c1d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/test_shader_cache_housekeeping.py @@ -0,0 +1,100 @@ +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +def native(path): + return Path('\\\\?\\' + str(path)) if os.name == 'nt' else path + + +def snapshot(directory): + return {str(path.relative_to(directory)): None if path.is_dir() else path.read_bytes() + for path in directory.rglob('*')} + + +def main(): + executable = str(Path(sys.argv[1]).resolve()) + mode = sys.argv[2] + assert mode in ('plain', 'plain-long', 'extended', 'extended-long', 'device') + assert os.name == 'nt' or mode in ('plain', 'plain-long') + sandbox = Path(tempfile.mkdtemp(prefix='aps5-housekeeping-')).resolve() + temporary = Path(tempfile.gettempdir()).resolve() + try: + outside = sandbox / 'outside' + outside.mkdir() + environment = dict(os.environ, ANYPS5_NO_SHADER_CACHE='0', + ANYPS5_SHADER_CACHE_DIR=str(sandbox / 'bootstrap')) + + def run(): + return subprocess.run([executable, '--housekeeping-safety'], cwd=outside, + env=environment, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors='replace', timeout=5) + + bootstrap = run() + assert bootstrap.returncode == 0, bootstrap.stdout + match = re.search(r'housekeeping version ([0-9a-f]{16})', bootstrap.stdout) + assert match, bootstrap.stdout + version = match.group(1) + siblings = [digit + version[1:] for digit in '0123' if digit != version[0]] + root = sandbox / 'cache-\u00e4\u6f22' + if mode.endswith('long'): + while len(str(root)) < 230: + root /= 'd' * 40 + root /= 'd' * max(1, 269 - len(str(root))) + assert len(str(root)) >= 270 + root = native(root) + current = root / version + current.mkdir(parents=True) + old = time.time() - 30 * 24 * 3600 + for parent in (root, outside): + for index, name in enumerate(siblings): + folder = parent / name + folder.mkdir() + (folder / 'sentinel').write_bytes(b'preserve outside the cache root') + if index != 1: + (folder / 'last-used').write_bytes(b'') + os.utime(folder / 'last-used', (old if index == 0 else time.time(),) * 2) + os.utime(folder, (old, old)) + (outside / 'outside.tmp.1').write_bytes(b'not a cache temporary') + os.utime(outside / 'outside.tmp.1', (old, old)) + for name in ('shader.bin', 'old.tmp.1', 'fresh.tmp.1', 'last-used'): + (current / name).write_bytes(b'cached bytes') + for name in ('shader.bin', 'old.tmp.1', 'last-used'): + os.utime(current / name, (old, old)) + before = snapshot(outside) + if mode.startswith('extended'): + environment['ANYPS5_SHADER_CACHE_DIR'] = str(root) + elif mode == 'device': + environment['ANYPS5_SHADER_CACHE_DIR'] = '\\\\.\\' + str(root)[4:] + else: + environment['ANYPS5_SHADER_CACHE_DIR'] = str(root)[4:] if os.name == 'nt' else str(root) + started = time.time() + try: + result = run() + except subprocess.TimeoutExpired as error: + assert snapshot(outside) == before, 'housekeeping changed files outside its root' + raise AssertionError('housekeeping did not return within five seconds; child terminated') from error + assert snapshot(outside) == before, 'housekeeping changed files outside its root' + assert result.returncode == 0, result.stdout + assert 'housekeeping version ' + version in result.stdout, result.stdout + assert (current / 'last-used').stat().st_mtime >= started - 2, 'active cache stamp was not refreshed' + skipped = mode.startswith('extended') or mode == 'device' + for name in siblings[:2]: + assert (root / name).exists() == skipped, 'unexpected stale cache directory eviction' + assert (root / siblings[2] / 'sentinel').read_bytes() == b'preserve outside the cache root' + assert (current / 'old.tmp.1').exists() == skipped, 'unexpected old temporary removal' + assert (current / 'fresh.tmp.1').read_bytes() == b'cached bytes' + assert (current / 'shader.bin').read_bytes() == b'cached bytes' + print(mode + ': returned, active stamp refreshed, outside tree intact, eviction policy verified') + finally: + assert sandbox.parent == temporary and sandbox.name.startswith('aps5-housekeeping-') + shutil.rmtree(native(sandbox)) + + +if __name__ == '__main__': + main() diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index f830045b522..420ba530bce 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -1615,6 +1615,11 @@ class DiskStore { const auto stamp = directory / "last-used"; std::ofstream(stamp, std::ios::binary | std::ios::trunc).close(); std::filesystem::last_write_time(stamp, std::filesystem::file_time_type::clock::now(), error); +#ifdef _WIN32 + auto nativeRoot = root; + nativeRoot.make_preferred(); + if (nativeRoot.native().starts_with(L"\\\\?\\") || nativeRoot.native().starts_with(L"\\\\.\\")) return; +#endif const auto now = std::filesystem::file_time_type::clock::now(); for (std::filesystem::directory_iterator it(root, error), end; !error && it != end; it.increment(error)) { const auto name = it->path().filename().string(); diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index e5ad6cd186b..1e8e9910f00 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -23,6 +23,8 @@ ### Functional +- [Windows cache housekeeping](../../../core/shader/recompiler/ShaderDiskCache.cpp): eviction of old versions and temporary files is skipped for `\\?\` and `\\.\` cache roots because the supported WinLibs directory iterator can enumerate the process working directory for long prefixed paths. The active version's `last-used` stamp update still runs; it fails for long `\\.\` aliases with the supported toolchain. Ordinary Windows directory enumeration remains subject to MAX_PATH. Prebuilt directory discovery is not guarded for prefixed roots. + - [f64 compares](../../../core/shader/recompiler/Translation/src/CompareInstructions.cpp) flush denormal inputs to signed zero when bit 0 of the `FLOAT_MODE` f16/f64 denormal field is clear (modes 0 and 2); modes 1 and 3 and an unknown mode keep them. This follows the RDNA2 ISA section 6.4 and LLVM's sign-preserving AMDGPU denormal modes. The gfx1036 audit in [#981](https://github.com/boykopovar/AnyPS5/issues/981#issuecomment-6070046381) reports mode-sensitive `VopcCompareF64` results. Class compares still inspect the original bits. The full four-mode matrix has not been newly measured on AMD hardware or the PS5's gfx1013. - [Unlinked calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) - the `s_swappc_b64` a table call's compare chain falls back to, when its target matches no linked copy, records the `UnlinkedCall` fault with the target and the program counter and stops the invocation, or continues after the call in programs whose faults do not stop invocations (see `bdaStopsInvocations`). The driver lists those calls in the request (`linkedCallFallbacks`), and it reports the fault like a BDA fault, naming the target: a dispatch that checks its faults when it completes throws, one whose check runs in a deferred write-back logs it and continues. The hardware would jump to the target, which is not modelled. Any other `s_swappc_b64` whose target is not resolved still throws when the shader is compiled.