diff --git a/docs/dev/PROGRESS.md b/docs/dev/PROGRESS.md index fe31e07c2f1..752cc224ca4 100644 --- a/docs/dev/PROGRESS.md +++ b/docs/dev/PROGRESS.md @@ -24,11 +24,17 @@ Real removals and transitions back to stubs remain visible. Percentages retain t Implemented definitions are also classified as behaviourally empty when their body holds only `(void)name;` discards and at most one constant return, bare `return;` or `throw`, including a return of a local `static` wrapper that is itself behaviourally empty; wrapper calls whose arguments contain calls do not qualify. Comparisons list newly implemented behaviourally empty functions without changing any count. Older JSON snapshots without `inert_names` are accepted. +## One library + +`--module NAME` limits the library totals to those directories under `core/libs/prx`. Repeat the flag to include several. A name that is not a directory is an error. Shader counts stay complete. A shared-source wrapper is included only when its own directory is named. `--json` prints the object written to `progress.json` on stdout, instead of the two summary lines. Either flag makes the output directory optional. Without them the directory is still required, and a run that names only the directory still writes the whole tree. + ## Verification ```sh python3 tools/tests/test_progress.py python3 tools/progress.py /tmp/anyps5-progress +python3 tools/progress.py --module libScePad +python3 tools/progress.py --json python3 tools/tests/test_progress_history.py ``` diff --git a/tools/progress.py b/tools/progress.py index 8a4f5995f34..89e2bf36566 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -204,6 +204,21 @@ def collect_libraries(): return summarize([scan_library(p) for p in sorted(PRX.iterdir()) if p.is_dir()]) +def select_libraries(libraries, modules): + if not modules: + return libraries + known = {group["name"] for group in libraries["groups"]} + missing = [] + for name in modules: + if name not in known and name not in missing: + missing.append(name) + if missing: + label = "library" if len(missing) == 1 else "libraries" + raise ValueError(f"unknown {label}: {', '.join(missing)}") + chosen = set(modules) + return summarize([group for group in libraries["groups"] if group["name"] in chosen]) + + def camel(name): return "".join(part.capitalize() for part in name.split("_")) @@ -431,31 +446,58 @@ def report(base, head): return "\n".join(lines) -if __name__ == "__main__": - if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8") +def write_site(output, libraries, shaders): + output.mkdir(parents=True, exist_ok=True) + (output / "progress.json").write_text(json.dumps({"libraries": libraries, "shaders": shaders}, indent=2), encoding="utf-8") + (output / "badge-libraries.svg").write_text(badge("libraries*", libraries), encoding="utf-8") + (output / "badge-shaders.svg").write_text(badge("shaders", shaders), encoding="utf-8") + (output / "progress.svg").write_text(render(libraries, shaders), encoding="utf-8") + (output / "progress.html").write_text(summary(libraries, shaders), encoding="utf-8") + + +def main(argv=None): + global PRX, OPCODES + previous = PRX, OPCODES parser = argparse.ArgumentParser() parser.add_argument("output", type=Path, nargs="?") parser.add_argument("--root", type=Path, help="source tree to measure instead of the one containing this script") parser.add_argument("--compare", type=Path, nargs=2, metavar=("BASE", "HEAD"), help="print a markdown report of the changes between two progress.json files") - args = parser.parse_args() - if args.compare: - base, head = (json.loads(path.read_text(encoding="utf-8")) for path in args.compare) - print(report(base, head), end="") - raise SystemExit - if not args.output: - parser.error("the output directory is required") - if args.root: - PRX = args.root / "core" / "libs" / "prx" - OPCODES = args.root / OPCODES.relative_to(ROOT) - output = args.output - output.mkdir(parents=True, exist_ok=True) - libraries, shaders = collect_libraries(), collect_shaders() - (output / "progress.json").write_text(json.dumps({"libraries": libraries, "shaders": shaders}, indent=2)) - (output / "badge-libraries.svg").write_text(badge("libraries*", libraries)) - (output / "badge-shaders.svg").write_text(badge("shaders", shaders)) - (output / "progress.svg").write_text(render(libraries, shaders)) - (output / "progress.html").write_text(summary(libraries, shaders)) - print(f'libraries {libraries["done"]}/{libraries["total"]} ({libraries["percent"]}%)') - print(f'shaders {shaders["done"]}/{shaders["total"]} ({shaders["percent"]}%)') + parser.add_argument("--module", action="append", metavar="NAME", + help="limit library counts to these core/libs/prx directories; repeat to include several. Shader counts stay complete") + parser.add_argument("--json", action="store_true", + help="print the progress.json object to stdout instead of the summary lines") + try: + args = parser.parse_args(argv) + if args.compare: + if args.module or args.json: + parser.error("--compare does not combine with --module or --json") + base, head = (json.loads(path.read_text(encoding="utf-8")) for path in args.compare) + print(report(base, head), end="") + return 0 + if not args.output and not (args.module or args.json): + parser.error("the output directory is required") + if args.root: + PRX = args.root / "core" / "libs" / "prx" + OPCODES = args.root / previous[1].relative_to(ROOT) + libraries, shaders = collect_libraries(), collect_shaders() + try: + libraries = select_libraries(libraries, args.module or []) + except ValueError as error: + parser.error(str(error)) + if args.output: + write_site(args.output, libraries, shaders) + if args.json: + print(json.dumps({"libraries": libraries, "shaders": shaders}, indent=2)) + else: + print(f'libraries {libraries["done"]}/{libraries["total"]} ({libraries["percent"]}%)') + print(f'shaders {shaders["done"]}/{shaders["total"]} ({shaders["percent"]}%)') + return 0 + finally: + PRX, OPCODES = previous + + +if __name__ == "__main__": + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + raise SystemExit(main()) diff --git a/tools/tests/test_progress.py b/tools/tests/test_progress.py index e2ac037b3f9..84c34796969 100644 --- a/tools/tests/test_progress.py +++ b/tools/tests/test_progress.py @@ -1,3 +1,5 @@ +import io +import json import sys import tempfile import unittest @@ -293,6 +295,97 @@ def test_qualified_calls_match_only_the_global_wrapper(self): group = progress.scan_library(library) self.assertEqual(group[expected + "_names"], ["Run"]) + def library_snapshot(self): + libraries = progress.summarize([ + {"name": "libScePad", "label": "Pad", "done": 1, "todo": 1, + "done_names": ["Ready"], "todo_names": ["Pending"], "inert_names": []}, + {"name": "libSceVideoOut", "label": "VideoOut", "done": 3, "todo": 1, + "done_names": ["A", "B", "C"], "todo_names": ["D"], "inert_names": []}, + {"name": "libScePad.native", "label": "Pad.native", "done": 0, "todo": 0, + "done_names": [], "todo_names": [], "inert_names": [], "shared_sources": "libScePad"}, + ]) + shaders = {"done": 2, "total": 4, "percent": 50.0, "groups": [], "extra": []} + return libraries, shaders + + def run_main(self, argv, libraries, shaders): + stdout = io.StringIO() + with patch.object(progress, "collect_libraries", return_value=libraries), \ + patch.object(progress, "collect_shaders", return_value=shaders), \ + patch("sys.stdout", stdout): + code = progress.main(argv) + return code, stdout.getvalue() + + def test_module_filter_recomputes_library_totals(self): + libraries, shaders = self.library_snapshot() + selected = progress.select_libraries(libraries, ["libSceVideoOut", "libScePad", "libScePad"]) + self.assertEqual([group["name"] for group in selected["groups"]], ["libScePad", "libSceVideoOut"]) + self.assertEqual((selected["done"], selected["total"], selected["percent"]), (4, 6, 66.67)) + self.assertEqual(shaders["total"], 4) + wrapper = progress.select_libraries(libraries, ["libScePad.native"]) + self.assertEqual(wrapper["groups"][0]["shared_sources"], "libScePad") + self.assertEqual((wrapper["done"], wrapper["total"]), (0, 0)) + with self.assertRaisesRegex(ValueError, "unknown libraries: libSceNope, libSceMissing"): + progress.select_libraries(libraries, ["libScePad", "libSceNope", "libSceMissing", "libSceNope"]) + + def test_json_and_module_do_not_require_an_output_directory(self): + libraries, shaders = self.library_snapshot() + code, output = self.run_main(["--json", "--module", "libScePad"], libraries, shaders) + self.assertEqual(code, 0) + payload = json.loads(output) + self.assertEqual(payload["libraries"]["groups"][0]["name"], "libScePad") + self.assertEqual((payload["libraries"]["done"], payload["libraries"]["total"]), (1, 2)) + self.assertEqual(payload["shaders"], shaders) + self.assertNotIn("libraries 1/2", output) + code, output = self.run_main(["--module", "libScePad"], libraries, shaders) + self.assertEqual(output, "libraries 1/2 (50.0%)\nshaders 2/4 (50.0%)\n") + stderr = io.StringIO() + with patch("sys.stderr", stderr), self.assertRaises(SystemExit) as raised: + progress.main([]) + self.assertEqual(raised.exception.code, 2) + self.assertIn("the output directory is required", stderr.getvalue()) + + def test_module_filter_is_what_the_site_writes(self): + libraries, shaders = self.library_snapshot() + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + code, text = self.run_main(["--json", "--module", "libScePad", str(output)], libraries, shaders) + self.assertEqual(code, 0) + written = json.loads((output / "progress.json").read_text(encoding="utf-8")) + self.assertEqual(written, json.loads(text)) + self.assertEqual(written["libraries"]["groups"][0]["name"], "libScePad") + self.assertEqual(written["shaders"]["done"], 2) + page = (output / "progress.html").read_text(encoding="utf-8") + self.assertIn("libScePad", page) + self.assertNotIn("libSceVideoOut", page) + for name in ("badge-libraries.svg", "badge-shaders.svg", "progress.svg", "progress.html"): + self.assertTrue((output / name).is_file()) + original = progress.PRX + with tempfile.TemporaryDirectory() as directory: + self.run_main(["--root", directory, "--json"], libraries, shaders) + self.assertEqual(progress.PRX, original) + + def test_compare_stays_a_report_of_two_snapshots(self): + def snapshot(names): + return {"libraries": progress.summarize([ + {"name": "libScePad", "label": "Pad", "done": len(names), "todo": 0, + "done_names": names, "todo_names": [], "inert_names": []}]), + "shaders": {"done": 0, "total": 0, "percent": 0, "groups": [], "extra": []}} + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) / "base.json" + head = Path(directory) / "head.json" + base.write_text(json.dumps(snapshot([])), encoding="utf-8") + head.write_text(json.dumps(snapshot(["Ready"])), encoding="utf-8") + stdout = io.StringIO() + with patch("sys.stdout", stdout): + code = progress.main(["--compare", str(base), str(head)]) + self.assertEqual(code, 0) + self.assertIn("implemented", stdout.getvalue()) + stderr = io.StringIO() + with patch("sys.stderr", stderr), self.assertRaises(SystemExit) as raised: + progress.main(["--compare", str(base), str(head), "--json"]) + self.assertEqual(raised.exception.code, 2) + self.assertIn("--compare does not combine with --module or --json", stderr.getvalue()) + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_progress_encoding.py b/tools/tests/test_progress_encoding.py index a0e1b46852b..1b6628f2853 100644 --- a/tools/tests/test_progress_encoding.py +++ b/tools/tests/test_progress_encoding.py @@ -57,6 +57,31 @@ def test_utf8_stdout_report_is_unchanged(self): base, head = snapshot(False), snapshot(True) self.assertEqual(self.compare(base, head, "utf-8"), self.compare(base, head)) + def test_site_generation_preserves_unicode_with_cp1252_stdout(self): + library = "libSce\u6e2c\u8a66\U0001f3ae" + with tempfile.TemporaryDirectory(prefix="progress site encoding ") as directory: + root = Path(directory) / "source" + module = root / "core" / "libs" / "prx" / library + module.mkdir(parents=True) + (module / "Export.cpp").write_text("int APS5_VABI Ready() { return 0; }\n", encoding="utf-8") + opcodes = root / "core" / "shader" / "recompiler" / "RdnaDecoder" / "include" / "RdnaDecoder" / "RdnaOpcode.hpp" + opcodes.parent.mkdir(parents=True) + opcodes.write_text("enum class RdnaOpcode {\n SEndpgm,\n};\n", encoding="utf-8") + output = Path(directory) / "site" + env = {**os.environ, "PYTHONIOENCODING": "cp1252", "PYTHONUTF8": "0"} + result = subprocess.run( + [sys.executable, "-X", "utf8=0", str(PROGRESS), str(output), "--root", str(root), + "--module", library, "--json"], env=env, capture_output=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", errors="replace")) + generated = {name: (output / name).read_text(encoding="utf-8") for name in ( + "progress.json", "badge-libraries.svg", "badge-shaders.svg", "progress.svg", "progress.html")} + payload = json.loads(generated["progress.json"]) + self.assertEqual(payload, json.loads(result.stdout.decode("utf-8"))) + self.assertEqual((payload["libraries"]["done"], payload["libraries"]["total"]), (1, 1)) + self.assertEqual(payload["libraries"]["groups"][0]["name"], library) + for name in ("progress.svg", "progress.html"): + self.assertIn(library.removeprefix("libSce"), generated[name]) + if __name__ == "__main__": unittest.main()