feat(tools): add a module filter and JSON output to progress.py

docs/dev/PROGRESS.md

@@ -24,11 +24,17 @@ Real removals and transitions back to stubs remain visible. Percentages retain t
 
 Implemented definitions are also classified as behaviourally empty when their body holds only `(void)name;` discards and at most one constant return, bare `return;` or `throw`, including a return of a local `static` wrapper that is itself behaviourally empty; wrapper calls whose arguments contain calls do not qualify. Comparisons list newly implemented behaviourally empty functions without changing any count. Older JSON snapshots without `inert_names` are accepted.
 
+## One library
+
+`--module NAME` limits the library totals to those directories under `core/libs/prx`. Repeat the flag to include several. A name that is not a directory is an error. Shader counts stay complete. A shared-source wrapper is included only when its own directory is named. `--json` prints the object written to `progress.json` on stdout, instead of the two summary lines. Either flag makes the output directory optional. Without them the directory is still required, and a run that names only the directory still writes the whole tree.
+
 ## Verification
 
 ```sh
 python3 tools/tests/test_progress.py
 python3 tools/progress.py /tmp/anyps5-progress
+python3 tools/progress.py --module libScePad
+python3 tools/progress.py --json
 python3 tools/tests/test_progress_history.py
 ```
 

tools/progress.py

@@ -204,6 +204,21 @@ def collect_libraries():
     return summarize([scan_library(p) for p in sorted(PRX.iterdir()) if p.is_dir()])
 
 
+def select_libraries(libraries, modules):
+    if not modules:
+        return libraries
+    known = {group["name"] for group in libraries["groups"]}
+    missing = []
+    for name in modules:
+        if name not in known and name not in missing:
+            missing.append(name)
+    if missing:
+        label = "library" if len(missing) == 1 else "libraries"
+        raise ValueError(f"unknown {label}: {', '.join(missing)}")
+    chosen = set(modules)
+    return summarize([group for group in libraries["groups"] if group["name"] in chosen])
+
+
 def camel(name):
     return "".join(part.capitalize() for part in name.split("_"))
 
@@ -431,31 +446,58 @@ def report(base, head):
     return "\n".join(lines)
 
 
-if __name__ == "__main__":
-    if hasattr(sys.stdout, "reconfigure"):
-        sys.stdout.reconfigure(encoding="utf-8")
+def write_site(output, libraries, shaders):
+    output.mkdir(parents=True, exist_ok=True)
+    (output / "progress.json").write_text(json.dumps({"libraries": libraries, "shaders": shaders}, indent=2), encoding="utf-8")
+    (output / "badge-libraries.svg").write_text(badge("libraries*", libraries), encoding="utf-8")
+    (output / "badge-shaders.svg").write_text(badge("shaders", shaders), encoding="utf-8")
+    (output / "progress.svg").write_text(render(libraries, shaders), encoding="utf-8")
+    (output / "progress.html").write_text(summary(libraries, shaders), encoding="utf-8")
+
+
+def main(argv=None):
+    global PRX, OPCODES
+    previous = PRX, OPCODES
     parser = argparse.ArgumentParser()
     parser.add_argument("output", type=Path, nargs="?")
     parser.add_argument("--root", type=Path, help="source tree to measure instead of the one containing this script")
     parser.add_argument("--compare", type=Path, nargs=2, metavar=("BASE", "HEAD"),
                         help="print a markdown report of the changes between two progress.json files")
-    args = parser.parse_args()
-    if args.compare:
-        base, head = (json.loads(path.read_text(encoding="utf-8")) for path in args.compare)
-        print(report(base, head), end="")
-        raise SystemExit
-    if not args.output:
-        parser.error("the output directory is required")
-    if args.root:
-        PRX = args.root / "core" / "libs" / "prx"
-        OPCODES = args.root / OPCODES.relative_to(ROOT)
-    output = args.output
-    output.mkdir(parents=True, exist_ok=True)
-    libraries, shaders = collect_libraries(), collect_shaders()
-    (output / "progress.json").write_text(json.dumps({"libraries": libraries, "shaders": shaders}, indent=2))
-    (output / "badge-libraries.svg").write_text(badge("libraries*", libraries))
-    (output / "badge-shaders.svg").write_text(badge("shaders", shaders))
-    (output / "progress.svg").write_text(render(libraries, shaders))
-    (output / "progress.html").write_text(summary(libraries, shaders))
-    print(f'libraries {libraries["done"]}/{libraries["total"]} ({libraries["percent"]}%)')
-    print(f'shaders {shaders["done"]}/{shaders["total"]} ({shaders["percent"]}%)')
+    parser.add_argument("--module", action="append", metavar="NAME",
+                        help="limit library counts to these core/libs/prx directories; repeat to include several. Shader counts stay complete")
+    parser.add_argument("--json", action="store_true",
+                        help="print the progress.json object to stdout instead of the summary lines")
+    try:
+        args = parser.parse_args(argv)
+        if args.compare:
+            if args.module or args.json:
+                parser.error("--compare does not combine with --module or --json")
+            base, head = (json.loads(path.read_text(encoding="utf-8")) for path in args.compare)
+            print(report(base, head), end="")
+            return 0
+        if not args.output and not (args.module or args.json):
+            parser.error("the output directory is required")
+        if args.root:
+            PRX = args.root / "core" / "libs" / "prx"
+            OPCODES = args.root / previous[1].relative_to(ROOT)
+        libraries, shaders = collect_libraries(), collect_shaders()
+        try:
+            libraries = select_libraries(libraries, args.module or [])
+        except ValueError as error:
+            parser.error(str(error))
+        if args.output:
+            write_site(args.output, libraries, shaders)
+        if args.json:
+            print(json.dumps({"libraries": libraries, "shaders": shaders}, indent=2))
+        else:
+            print(f'libraries {libraries["done"]}/{libraries["total"]} ({libraries["percent"]}%)')
+            print(f'shaders {shaders["done"]}/{shaders["total"]} ({shaders["percent"]}%)')
+        return 0
+    finally:
+        PRX, OPCODES = previous
+
+
+if __name__ == "__main__":
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8")
+    raise SystemExit(main())

tools/tests/test_progress.py

@@ -1,3 +1,5 @@
+import io
+import json
 import sys
 import tempfile
 import unittest
@@ -293,6 +295,97 @@ def test_qualified_calls_match_only_the_global_wrapper(self):
                 group = progress.scan_library(library)
                 self.assertEqual(group[expected + "_names"], ["Run"])
 
+    def library_snapshot(self):
+        libraries = progress.summarize([
+            {"name": "libScePad", "label": "Pad", "done": 1, "todo": 1,
+             "done_names": ["Ready"], "todo_names": ["Pending"], "inert_names": []},
+            {"name": "libSceVideoOut", "label": "VideoOut", "done": 3, "todo": 1,
+             "done_names": ["A", "B", "C"], "todo_names": ["D"], "inert_names": []},
+            {"name": "libScePad.native", "label": "Pad.native", "done": 0, "todo": 0,
+             "done_names": [], "todo_names": [], "inert_names": [], "shared_sources": "libScePad"},
+        ])
+        shaders = {"done": 2, "total": 4, "percent": 50.0, "groups": [], "extra": []}
+        return libraries, shaders
+
+    def run_main(self, argv, libraries, shaders):
+        stdout = io.StringIO()
+        with patch.object(progress, "collect_libraries", return_value=libraries), \
+             patch.object(progress, "collect_shaders", return_value=shaders), \
+             patch("sys.stdout", stdout):
+            code = progress.main(argv)
+        return code, stdout.getvalue()
+
+    def test_module_filter_recomputes_library_totals(self):
+        libraries, shaders = self.library_snapshot()
+        selected = progress.select_libraries(libraries, ["libSceVideoOut", "libScePad", "libScePad"])
+        self.assertEqual([group["name"] for group in selected["groups"]], ["libScePad", "libSceVideoOut"])
+        self.assertEqual((selected["done"], selected["total"], selected["percent"]), (4, 6, 66.67))
+        self.assertEqual(shaders["total"], 4)
+        wrapper = progress.select_libraries(libraries, ["libScePad.native"])
+        self.assertEqual(wrapper["groups"][0]["shared_sources"], "libScePad")
+        self.assertEqual((wrapper["done"], wrapper["total"]), (0, 0))
+        with self.assertRaisesRegex(ValueError, "unknown libraries: libSceNope, libSceMissing"):
+            progress.select_libraries(libraries, ["libScePad", "libSceNope", "libSceMissing", "libSceNope"])
+
+    def test_json_and_module_do_not_require_an_output_directory(self):
+        libraries, shaders = self.library_snapshot()
+        code, output = self.run_main(["--json", "--module", "libScePad"], libraries, shaders)
+        self.assertEqual(code, 0)
+        payload = json.loads(output)
+        self.assertEqual(payload["libraries"]["groups"][0]["name"], "libScePad")
+        self.assertEqual((payload["libraries"]["done"], payload["libraries"]["total"]), (1, 2))
+        self.assertEqual(payload["shaders"], shaders)
+        self.assertNotIn("libraries 1/2", output)
+        code, output = self.run_main(["--module", "libScePad"], libraries, shaders)
+        self.assertEqual(output, "libraries 1/2 (50.0%)\nshaders 2/4 (50.0%)\n")
+        stderr = io.StringIO()
+        with patch("sys.stderr", stderr), self.assertRaises(SystemExit) as raised:
+            progress.main([])
+        self.assertEqual(raised.exception.code, 2)
+        self.assertIn("the output directory is required", stderr.getvalue())
+
+    def test_module_filter_is_what_the_site_writes(self):
+        libraries, shaders = self.library_snapshot()
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory)
+            code, text = self.run_main(["--json", "--module", "libScePad", str(output)], libraries, shaders)
+            self.assertEqual(code, 0)
+            written = json.loads((output / "progress.json").read_text(encoding="utf-8"))
+            self.assertEqual(written, json.loads(text))
+            self.assertEqual(written["libraries"]["groups"][0]["name"], "libScePad")
+            self.assertEqual(written["shaders"]["done"], 2)
+            page = (output / "progress.html").read_text(encoding="utf-8")
+            self.assertIn("libScePad", page)
+            self.assertNotIn("libSceVideoOut", page)
+            for name in ("badge-libraries.svg", "badge-shaders.svg", "progress.svg", "progress.html"):
+                self.assertTrue((output / name).is_file())
+        original = progress.PRX
+        with tempfile.TemporaryDirectory() as directory:
+            self.run_main(["--root", directory, "--json"], libraries, shaders)
+        self.assertEqual(progress.PRX, original)
+
+    def test_compare_stays_a_report_of_two_snapshots(self):
+        def snapshot(names):
+            return {"libraries": progress.summarize([
+                        {"name": "libScePad", "label": "Pad", "done": len(names), "todo": 0,
+                         "done_names": names, "todo_names": [], "inert_names": []}]),
+                    "shaders": {"done": 0, "total": 0, "percent": 0, "groups": [], "extra": []}}
+        with tempfile.TemporaryDirectory() as directory:
+            base = Path(directory) / "base.json"
+            head = Path(directory) / "head.json"
+            base.write_text(json.dumps(snapshot([])), encoding="utf-8")
+            head.write_text(json.dumps(snapshot(["Ready"])), encoding="utf-8")
+            stdout = io.StringIO()
+            with patch("sys.stdout", stdout):
+                code = progress.main(["--compare", str(base), str(head)])
+            self.assertEqual(code, 0)
+            self.assertIn("implemented", stdout.getvalue())
+            stderr = io.StringIO()
+            with patch("sys.stderr", stderr), self.assertRaises(SystemExit) as raised:
+                progress.main(["--compare", str(base), str(head), "--json"])
+            self.assertEqual(raised.exception.code, 2)
+            self.assertIn("--compare does not combine with --module or --json", stderr.getvalue())
+
 
 if __name__ == "__main__":
     unittest.main()

tools/tests/test_progress_encoding.py

@@ -57,6 +57,31 @@ def test_utf8_stdout_report_is_unchanged(self):
         base, head = snapshot(False), snapshot(True)
         self.assertEqual(self.compare(base, head, "utf-8"), self.compare(base, head))
 
+    def test_site_generation_preserves_unicode_with_cp1252_stdout(self):
+        library = "libSce\u6e2c\u8a66\U0001f3ae"
+        with tempfile.TemporaryDirectory(prefix="progress site encoding ") as directory:
+            root = Path(directory) / "source"
+            module = root / "core" / "libs" / "prx" / library
+            module.mkdir(parents=True)
+            (module / "Export.cpp").write_text("int APS5_VABI Ready() { return 0; }\n", encoding="utf-8")
+            opcodes = root / "core" / "shader" / "recompiler" / "RdnaDecoder" / "include" / "RdnaDecoder" / "RdnaOpcode.hpp"
+            opcodes.parent.mkdir(parents=True)
+            opcodes.write_text("enum class RdnaOpcode {\n    SEndpgm,\n};\n", encoding="utf-8")
+            output = Path(directory) / "site"
+            env = {**os.environ, "PYTHONIOENCODING": "cp1252", "PYTHONUTF8": "0"}
+            result = subprocess.run(
+                [sys.executable, "-X", "utf8=0", str(PROGRESS), str(output), "--root", str(root),
+                 "--module", library, "--json"], env=env, capture_output=True, timeout=30)
+            self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", errors="replace"))
+            generated = {name: (output / name).read_text(encoding="utf-8") for name in (
+                "progress.json", "badge-libraries.svg", "badge-shaders.svg", "progress.svg", "progress.html")}
+            payload = json.loads(generated["progress.json"])
+            self.assertEqual(payload, json.loads(result.stdout.decode("utf-8")))
+            self.assertEqual((payload["libraries"]["done"], payload["libraries"]["total"]), (1, 1))
+            self.assertEqual(payload["libraries"]["groups"][0]["name"], library)
+            for name in ("progress.svg", "progress.html"):
+                self.assertIn(library.removeprefix("libSce"), generated[name])
+
 
 if __name__ == "__main__":
     unittest.main()