diff --git a/fusil/python/__init__.py b/fusil/python/__init__.py index 09d4407..6f341e6 100644 --- a/fusil/python/__init__.py +++ b/fusil/python/__init__.py @@ -988,6 +988,27 @@ def setupProject(self) -> None: # pushes a boring session over the threshold: 6 kept dirs in one PyPy fleet. It # is the target's benign diagnostic, not a target defect. r"http\.cookiejar bug!", + # A traceback is the target quoting ITSELF, not the target reporting a crash, and + # it prints two kinds of line that routinely collide with a scored word: + # + # 1. the frame line ` File ".../logging/__init__.py", line 1536, in critical` + # -- the crashing FUNCTION's name, here matching the 1.0 word `critical`; + # 2. the source line ` LOGGER.critical('Future %s in unexpected state: %s',` + # -- the offending SOURCE, echoed verbatim below the frame line. + # + # Each shape kept a crash dir in a PyPy fleet on its own (concurrent.futures._base, + # twice over, via different halves of the same traceback). Skip both: neither is + # ever a target-emitted diagnostic. A real critical-level MESSAGE is the formatted + # text ("CRITICAL:root:..."), which matches neither pattern and still scores. + r'^\s*File "[^"]*", line \d+, in ', + # bdb's tracer echoes every traced event and the VALUE involved: + # `+++ return ` + # The repr of an arbitrary fuzz value routinely contains a scored word -- here + # the builtin SystemError class, a 1.0 word -- so tracing any module under + # bdb/pdb manufactures crashes. Two kept dirs in one PyPy fleet. Only bdb emits + # these prefixes; no diagnostic starts with them. + r"^(\+\+\+|---|!!!) ", + r"\.critical\(", # The --new-uninit region prints a progress marker per poked type, # e.g. "[NEW-UNINIT] poking SystemError". The type name is arbitrary and # routinely collides with a crash word ("SystemError" -> a 1.0 hit) or, worse, diff --git a/fusil/python/blacklists.py b/fusil/python/blacklists.py index 42cc387..efdb978 100644 --- a/fusil/python/blacklists.py +++ b/fusil/python/blacklists.py @@ -197,6 +197,11 @@ # script's `except Exception` handlers and kills the session (the #192 class). It was # 29 of 53 kept dirs -- 55% -- in one PyPy fleet, reached because --test-private exposes # the underscore-prefixed method. + # Keyed on the module the class is DEFINED in. That is not enough on its own -- the same + # class is re-exported as `asyncio.Runner`, and a session whose target module is `asyncio` + # reaches `_on_sigint` through that path with this key never matching. `_on_sigint` is + # therefore also in METHOD_BLACKLIST, which is name-based and module-agnostic; this entry + # stays because it also filters the static generation path for the defining module. "asyncio.runners:Runner": {"_on_sigint"}, "_socket": SOCKET, "socket": SOCKET, @@ -296,6 +301,15 @@ "_acquire_lock", "_acquire_restore", "_handle_request_noblock", + # Two SIGINT handlers that unconditionally `raise KeyboardInterrupt()`. Called directly as + # fuzz targets they raise a BaseException, which escapes the generated script's + # `except Exception` handlers and kills the session; an uncaught KeyboardInterrupt then + # makes the interpreter re-raise SIGINT, so the process looks "killed by signal 2" and + # WatchProcess scores it 1.0. Name-based so every path to them is covered, including the + # runtime generic-method loop and re-export modules (`asyncio.Runner`, not just + # `asyncio.runners.Runner`) -- three kept dirs in one PyPy fleet came in that way. + "_on_sigint", + "sigint_handler", "_randbelow", "_randbelow_with_getrandbits", "_read", diff --git a/fusil/python/samples/tricky_objects.py b/fusil/python/samples/tricky_objects.py index 35b1af9..0548253 100644 --- a/fusil/python/samples/tricky_objects.py +++ b/fusil/python/samples/tricky_objects.py @@ -122,7 +122,7 @@ def __getattr__(self, name): # Each object re-enters a protocol on a PARTNER object, so the recursion crosses object boundaries # (mutual recursion, harder to short-circuit than plain self-recursion). CPython raises # RecursionError on the protocol call; an interpreter without a recursion guard on that native -# path overflows its C/Rust stack -> segfault. Construction is cheap -- the recursion fires only +# path overflows its C/Rust stack instead. Construction is cheap -- the recursion fires only # when the fuzzer exercises the named protocol (hash/eq/getitem/iter/repr/call) on the object. class _TrickyRecur: def __init__(self, name): @@ -155,7 +155,7 @@ def __call__(self, *args, **kwargs): # Deep generic-alias nesting list[list[...list[T]...]] bottomed on a TypeVar so the parameter walk # actually recurses to collect it -- exercises the genericalias parameter-walk native path -# (RustPython segfaulted in genericalias::make_parameters_from_slice). Bounded depth so construction +# (RustPython crashed in genericalias::make_parameters_from_slice). Bounded depth so construction # + a CPython repr stay well under the recursion limit; the native walk (and __getitem__ # substitution) is the target. Falls back to a plain nested alias if TypeVar is unavailable. try: diff --git a/tests/python/golden/fakemod_seed1234.py b/tests/python/golden/fakemod_seed1234.py index 057e876..fbd6df9 100644 --- a/tests/python/golden/fakemod_seed1234.py +++ b/tests/python/golden/fakemod_seed1234.py @@ -45,7 +45,7 @@ def skip_trivial_type(obj_instance_or_class): return False -_FUSIL_METHOD_BLACKLIST = frozenset({'__class__', '__enter__', '__imul__', '__ipow__', '__mul__', '__pow__', '__rmul__', '_acquire_lock', '_acquire_restore', '_handle_request_noblock', '_randbelow', '_randbelow_with_getrandbits', '_read', '_rehash', '_run_once', '_serve', '_shutdown', 'accept', 'acquire', 'acquire_lock', 'cmdloop', 'copyfileobj', 'get', 'get_request', 'handle_request', 'handle_request_noblock', 'prefix', 'raise_signal', 'repeat', 'run_forever', 'select', 'serve_forever', 'shutdown', 'sleep', 'test', 'tri', 'tril_indices', 'wait', 'zfill'}) +_FUSIL_METHOD_BLACKLIST = frozenset({'__class__', '__enter__', '__imul__', '__ipow__', '__mul__', '__pow__', '__rmul__', '_acquire_lock', '_acquire_restore', '_handle_request_noblock', '_on_sigint', '_randbelow', '_randbelow_with_getrandbits', '_read', '_rehash', '_run_once', '_serve', '_shutdown', 'accept', 'acquire', 'acquire_lock', 'cmdloop', 'copyfileobj', 'get', 'get_request', 'handle_request', 'handle_request_noblock', 'prefix', 'raise_signal', 'repeat', 'run_forever', 'select', 'serve_forever', 'shutdown', 'sigint_handler', 'sleep', 'test', 'tri', 'tril_indices', 'wait', 'zfill'}) import sys from abc import ABCMeta @@ -403,7 +403,7 @@ def __getattr__(self, name): # Each object re-enters a protocol on a PARTNER object, so the recursion crosses object boundaries # (mutual recursion, harder to short-circuit than plain self-recursion). CPython raises # RecursionError on the protocol call; an interpreter without a recursion guard on that native -# path overflows its C/Rust stack -> segfault. Construction is cheap -- the recursion fires only +# path overflows its C/Rust stack instead. Construction is cheap -- the recursion fires only # when the fuzzer exercises the named protocol (hash/eq/getitem/iter/repr/call) on the object. class _TrickyRecur: def __init__(self, name): @@ -436,7 +436,7 @@ def __call__(self, *args, **kwargs): # Deep generic-alias nesting list[list[...list[T]...]] bottomed on a TypeVar so the parameter walk # actually recurses to collect it -- exercises the genericalias parameter-walk native path -# (RustPython segfaulted in genericalias::make_parameters_from_slice). Bounded depth so construction +# (RustPython crashed in genericalias::make_parameters_from_slice). Bounded depth so construction # + a CPython repr stay well under the recursion limit; the native walk (and __getitem__ # substitution) is the target. Falls back to a plain nested alias if TypeVar is unavailable. try: diff --git a/tests/python/test_blacklists.py b/tests/python/test_blacklists.py index 7c66835..47f1dd5 100644 --- a/tests/python/test_blacklists.py +++ b/tests/python/test_blacklists.py @@ -50,6 +50,20 @@ def test_asyncio_runner_on_sigint_blacklisted(self): # because --test-private exposes the underscore-prefixed method. self.assertIn("_on_sigint", bl.BLACKLIST["asyncio.runners:Runner"]) + def test_sigint_handlers_blacklisted_by_NAME_not_only_by_module(self): + """The module-keyed entry alone is not enough, and a fleet proved it. + + `Runner` is defined in `asyncio.runners` but re-exported as `asyncio.Runner`, so a + session whose target module is `asyncio` never matches the + `"asyncio.runners:Runner"` key -- and reached `_on_sigint` anyway, through the + runtime generic-method loop. `pdb.sigint_handler` is the same shape and was never + keyed at all. Both raise KeyboardInterrupt unconditionally; uncaught, the interpreter + re-raises SIGINT, the process looks "killed by signal 2", and WatchProcess scores it + 1.0. METHOD_BLACKLIST is name-based, so it covers every path. + """ + for name in ("_on_sigint", "sigint_handler"): + self.assertIn(name, bl.METHOD_BLACKLIST, name) + def test_default_int_handler_blacklisted(self): # It raises KeyboardInterrupt, a BaseException, which escapes the generated script's # `except Exception` handlers and kills the session outright (the #192 class). diff --git a/tests/python/test_golden_output.py b/tests/python/test_golden_output.py index 9d61d15..d3d713d 100644 --- a/tests/python/test_golden_output.py +++ b/tests/python/test_golden_output.py @@ -173,6 +173,56 @@ def test_full_script_matches_snapshot(self): ) +class SelfNoiseVocabularyTests(unittest.TestCase): + """Emitted COMMENTS must not contain fusil's own crash vocabulary. + + ``pydoc.getdoc`` falls back to ``inspect.getcomments()`` for an object with no + docstring, so the comment block immediately above an undocumented class in a spliced + sample file is printed verbatim by ``help(obj)``. A fuzz session that calls + ``help()`` -- ``_sitebuiltins._Helper.__call__`` is a normal fuzz target -- therefore + echoes those comments into stdout, where ``WatchStdout`` matches them and manufactures + a crash. One PyPy fleet kept a ``_sitebuiltins-segfault`` dir this way, scored 100% on + fusil's own word ``segfault`` inside a ``tricky_objects.py`` comment. + + Only comment lines are checked: the scored words also occur in real emitted CODE + (``SystemError`` in the bomb exception list, ``AssertionError`` in a class statement), + which ``inspect.getcomments`` never reaches. + """ + + # The 1.0-scoring words from Fuzzer.setupProject's WatchStdout configuration. The + # sub-1.0 words ("bug", "fatal", "oops") are not listed: they cannot score a session + # on their own and appear unavoidably in explanatory prose. + CRASH_WORDS = ( + "assertion", + "critical", + "panic", + "panicked", + "glibc detected", + "segfault", + "segmentation fault", + "addresssanitizer", + ) + + def test_emitted_comments_are_free_of_crash_words(self): + offenders = [] + for lineno, line in enumerate(generate().splitlines(), 1): + stripped = line.strip() + if not stripped.startswith("#"): + continue + lowered = stripped.lower() + for word in self.CRASH_WORDS: + if word in lowered: + offenders.append((lineno, word, stripped[:100])) + break + self.assertEqual( + offenders, + [], + "generated comments contain crash vocabulary that help() would echo into " + "stdout, scoring the session as a false crash; reword them:\n" + + "\n".join(" line %d [%s] %s" % o for o in offenders), + ) + + def _update_snapshot(): GOLDEN_DIR.mkdir(parents=True, exist_ok=True) GOLDEN_FILE.write_text(generate(), encoding="utf-8") diff --git a/tests/test_file_watch.py b/tests/test_file_watch.py index d2fe223..c4c7401 100644 --- a/tests/test_file_watch.py +++ b/tests/test_file_watch.py @@ -289,6 +289,73 @@ def test_ignore_regex_covers_every_raised_bomb_signature(self): ) +class TestTracebackEchoIgnored(unittest.TestCase): + """A traceback is the target quoting ITSELF; neither half of it may score as a crash. + + A routine fuzz traceback through ``concurrent/futures/_base.py`` kept a crash dir in a + PyPy fleet twice, via two different lines of the same traceback: + + * the frame line ``File ".../logging/__init__.py", line 1536, in critical`` + * the source line ``LOGGER.critical('Future %s in unexpected state: %s',`` + + ``critical`` is a 1.0 word, so either alone scores the session 100%. Both are ignored; a + real critical-level MESSAGE (``CRITICAL:root:...``) matches neither and still scores. + + The frame-line rule requires the comma that Python tracebacks put before ``in``. + faulthandler writes ``line N in func`` WITHOUT one, so a genuine fatal-signal report is + untouched -- covered by the last test here. + """ + + FRAME_LINE = r'^\s*File "[^"]*", line \d+, in ' + SOURCE_LINE = r"\.critical\(" + + def _watch_with_rules(self): + w = _watch(words={"critical": 1.0}) + w.ignoreRegex(self.FRAME_LINE) + w.ignoreRegex(self.SOURCE_LINE) + return w + + def test_echoed_source_line_is_ignored(self): + """Shape 2: the offending source, echoed below the frame line.""" + w = self._watch_with_rules() + self.assertIsNone( + w.processLine(b" LOGGER.critical('Future %s in unexpected state: %s',") + ) + self.assertEqual(w.score, 0.0) + + def test_traceback_frame_line_is_ignored(self): + """Shape 1: the frame line, whose FUNCTION name is the scored word.""" + w = self._watch_with_rules() + self.assertIsNone( + w.processLine(b' File "/usr/lib/pypy3.11/logging/__init__.py", line 1536, in critical') + ) + self.assertEqual(w.score, 0.0) + + def test_a_real_critical_message_still_scores(self): + w = self._watch_with_rules() + w.processLine(b"CRITICAL:root:the target said something critical") + self.assertEqual(w.score, 1.0) + + def test_bdb_tracer_lines_are_ignored(self): + """bdb echoes the VALUE of every traced event; its repr may hold a scored word.""" + w = _watch(words={"systemerror": 1.0}) + w.ignoreRegex(r"^(\+\+\+|---|!!!) ") + self.assertIsNone(w.processLine(b"+++ return ")) + self.assertEqual(w.score, 0.0) + w.processLine(b"SystemError: the target actually raised one") + self.assertEqual(w.score, 1.0) + + def test_faulthandler_stack_lines_are_not_swallowed(self): + """faulthandler writes `line N in func` with NO comma; only real tracebacks have one. + + The frame-line rule must not reach faulthandler's stack, which is how a genuine + fatal-signal crash is reported. + """ + w = self._watch_with_rules() + w.processLine(b' File "/tmp/session/source.py", line 1360 in critical') + self.assertEqual(w.score, 1.0) + + class TestCookiejarWarningIgnored(unittest.TestCase): """http.cookiejar's own "bug!" warning must not push a boring session over the threshold.