From 76f545cd69f834a7c7acd8bc9e89be98d92beac8 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:42:31 -0300 Subject: [PATCH] drop four fuzzer-self-noise sources found across three PyPy fleets Every one of these is fusil scoring text the TARGET echoed rather than a crash the target reported. Together they accounted for 8 kept crash dirs across fleets 05-07. 1. help() prints fusil's own comments. pydoc.getdoc falls back to inspect.getcomments() for an object with no docstring, so the comment block above an undocumented class in a spliced sample file is printed verbatim by help(obj) -- and _sitebuiltins._Helper.__call__ is a normal fuzz target. Two comments in samples/tricky_objects.py contained "segfault" (a 1.0 word) and scored a session 100%. Reworded, plus SelfNoiseVocabularyTests scans the emitted script's COMMENT lines for the 1.0 vocabulary so it cannot creep back. Only comments are checked: the same words legitimately occur in real emitted code (SystemError in the bomb list, AssertionError in a class statement), which inspect.getcomments never reaches. 2. A traceback is the target quoting itself, in two shapes. The source line (`LOGGER.critical('Future %s in unexpected state: %s',`) and the frame line (`File ".../logging/__init__.py", line 1536, in critical`), where the scored word is the crashing function's NAME. concurrent.futures._base kept a dir via each half in two different fleets. The frame-line rule requires the comma Python tracebacks put before `in`; faulthandler writes `line N in func` WITHOUT one, so genuine fatal-signal reports still score -- pinned by a test. 3. bdb's tracer echoes every traced event and the value involved (`+++ return `). The repr of an arbitrary fuzz value routinely holds a scored word, so tracing any module under bdb/pdb manufactures crashes. Ignore the tracer prefixes; no diagnostic uses them. 4. SIGINT handlers called directly as fuzz targets. asyncio's Runner._on_sigint and pdb.sigint_handler both `raise KeyboardInterrupt()` unconditionally; that BaseException escapes the generated script's `except Exception` handlers, and an uncaught KeyboardInterrupt makes the interpreter re-raise SIGINT, so the process looks "killed by signal 2" and WatchProcess scores it 1.0. _on_sigint was already blacklisted as "asyncio.runners:Runner" (#263), but Runner is re-exported as asyncio.Runner, so a session whose target module is asyncio never matches that key -- and reached it anyway through the runtime generic-method loop. pdb.sigint_handler was never keyed at all. Both now live in METHOD_BLACKLIST, which is name-based and module-agnostic, the same mechanism that already covers raise_signal. The module-keyed entry stays: it still filters the static generation path for the defining module. WatchProcess is deliberately left alone. Fixing the cause beats masking the symptom, and changing how signals are scored is far riskier than a denylist entry. The golden snapshot moves because METHOD_BLACKLIST is emitted into every generated script as _FUSIL_METHOD_BLACKLIST -- which is exactly why the name-based fix reaches the paths the module-keyed one missed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WPBSmN87d2BqUnrDbojUT1 --- fusil/python/__init__.py | 21 ++++++++ fusil/python/blacklists.py | 14 ++++++ fusil/python/samples/tricky_objects.py | 4 +- tests/python/golden/fakemod_seed1234.py | 6 +-- tests/python/test_blacklists.py | 14 ++++++ tests/python/test_golden_output.py | 50 ++++++++++++++++++ tests/test_file_watch.py | 67 +++++++++++++++++++++++++ 7 files changed, 171 insertions(+), 5 deletions(-) diff --git a/fusil/python/__init__.py b/fusil/python/__init__.py index 09d4407..6f341e6 100644 --- a/fusil/python/__init__.py +++ b/fusil/python/__init__.py @@ -988,6 +988,27 @@ def setupProject(self) -> None: # pushes a boring session over the threshold: 6 kept dirs in one PyPy fleet. It # is the target's benign diagnostic, not a target defect. r"http\.cookiejar bug!", + # A traceback is the target quoting ITSELF, not the target reporting a crash, and + # it prints two kinds of line that routinely collide with a scored word: + # + # 1. the frame line ` File ".../logging/__init__.py", line 1536, in critical` + # -- the crashing FUNCTION's name, here matching the 1.0 word `critical`; + # 2. the source line ` LOGGER.critical('Future %s in unexpected state: %s',` + # -- the offending SOURCE, echoed verbatim below the frame line. + # + # Each shape kept a crash dir in a PyPy fleet on its own (concurrent.futures._base, + # twice over, via different halves of the same traceback). Skip both: neither is + # ever a target-emitted diagnostic. A real critical-level MESSAGE is the formatted + # text ("CRITICAL:root:..."), which matches neither pattern and still scores. + r'^\s*File "[^"]*", line \d+, in ', + # bdb's tracer echoes every traced event and the VALUE involved: + # `+++ return ` + # The repr of an arbitrary fuzz value routinely contains a scored word -- here + # the builtin SystemError class, a 1.0 word -- so tracing any module under + # bdb/pdb manufactures crashes. Two kept dirs in one PyPy fleet. Only bdb emits + # these prefixes; no diagnostic starts with them. + r"^(\+\+\+|---|!!!) ", + r"\.critical\(", # The --new-uninit region prints a progress marker per poked type, # e.g. "[NEW-UNINIT] poking SystemError". The type name is arbitrary and # routinely collides with a crash word ("SystemError" -> a 1.0 hit) or, worse, diff --git a/fusil/python/blacklists.py b/fusil/python/blacklists.py index 42cc387..efdb978 100644 --- a/fusil/python/blacklists.py +++ b/fusil/python/blacklists.py @@ -197,6 +197,11 @@ # script's `except Exception` handlers and kills the session (the #192 class). It was # 29 of 53 kept dirs -- 55% -- in one PyPy fleet, reached because --test-private exposes # the underscore-prefixed method. + # Keyed on the module the class is DEFINED in. That is not enough on its own -- the same + # class is re-exported as `asyncio.Runner`, and a session whose target module is `asyncio` + # reaches `_on_sigint` through that path with this key never matching. `_on_sigint` is + # therefore also in METHOD_BLACKLIST, which is name-based and module-agnostic; this entry + # stays because it also filters the static generation path for the defining module. "asyncio.runners:Runner": {"_on_sigint"}, "_socket": SOCKET, "socket": SOCKET, @@ -296,6 +301,15 @@ "_acquire_lock", "_acquire_restore", "_handle_request_noblock", + # Two SIGINT handlers that unconditionally `raise KeyboardInterrupt()`. Called directly as + # fuzz targets they raise a BaseException, which escapes the generated script's + # `except Exception` handlers and kills the session; an uncaught KeyboardInterrupt then + # makes the interpreter re-raise SIGINT, so the process looks "killed by signal 2" and + # WatchProcess scores it 1.0. Name-based so every path to them is covered, including the + # runtime generic-method loop and re-export modules (`asyncio.Runner`, not just + # `asyncio.runners.Runner`) -- three kept dirs in one PyPy fleet came in that way. + "_on_sigint", + "sigint_handler", "_randbelow", "_randbelow_with_getrandbits", "_read", diff --git a/fusil/python/samples/tricky_objects.py b/fusil/python/samples/tricky_objects.py index 35b1af9..0548253 100644 --- a/fusil/python/samples/tricky_objects.py +++ b/fusil/python/samples/tricky_objects.py @@ -122,7 +122,7 @@ def __getattr__(self, name): # Each object re-enters a protocol on a PARTNER object, so the recursion crosses object boundaries # (mutual recursion, harder to short-circuit than plain self-recursion). CPython raises # RecursionError on the protocol call; an interpreter without a recursion guard on that native -# path overflows its C/Rust stack -> segfault. Construction is cheap -- the recursion fires only +# path overflows its C/Rust stack instead. Construction is cheap -- the recursion fires only # when the fuzzer exercises the named protocol (hash/eq/getitem/iter/repr/call) on the object. class _TrickyRecur: def __init__(self, name): @@ -155,7 +155,7 @@ def __call__(self, *args, **kwargs): # Deep generic-alias nesting list[list[...list[T]...]] bottomed on a TypeVar so the parameter walk # actually recurses to collect it -- exercises the genericalias parameter-walk native path -# (RustPython segfaulted in genericalias::make_parameters_from_slice). Bounded depth so construction +# (RustPython crashed in genericalias::make_parameters_from_slice). Bounded depth so construction # + a CPython repr stay well under the recursion limit; the native walk (and __getitem__ # substitution) is the target. Falls back to a plain nested alias if TypeVar is unavailable. try: diff --git a/tests/python/golden/fakemod_seed1234.py b/tests/python/golden/fakemod_seed1234.py index 057e876..fbd6df9 100644 --- a/tests/python/golden/fakemod_seed1234.py +++ b/tests/python/golden/fakemod_seed1234.py @@ -45,7 +45,7 @@ def skip_trivial_type(obj_instance_or_class): return False -_FUSIL_METHOD_BLACKLIST = frozenset({'__class__', '__enter__', '__imul__', '__ipow__', '__mul__', '__pow__', '__rmul__', '_acquire_lock', '_acquire_restore', '_handle_request_noblock', '_randbelow', '_randbelow_with_getrandbits', '_read', '_rehash', '_run_once', '_serve', '_shutdown', 'accept', 'acquire', 'acquire_lock', 'cmdloop', 'copyfileobj', 'get', 'get_request', 'handle_request', 'handle_request_noblock', 'prefix', 'raise_signal', 'repeat', 'run_forever', 'select', 'serve_forever', 'shutdown', 'sleep', 'test', 'tri', 'tril_indices', 'wait', 'zfill'}) +_FUSIL_METHOD_BLACKLIST = frozenset({'__class__', '__enter__', '__imul__', '__ipow__', '__mul__', '__pow__', '__rmul__', '_acquire_lock', '_acquire_restore', '_handle_request_noblock', '_on_sigint', '_randbelow', '_randbelow_with_getrandbits', '_read', '_rehash', '_run_once', '_serve', '_shutdown', 'accept', 'acquire', 'acquire_lock', 'cmdloop', 'copyfileobj', 'get', 'get_request', 'handle_request', 'handle_request_noblock', 'prefix', 'raise_signal', 'repeat', 'run_forever', 'select', 'serve_forever', 'shutdown', 'sigint_handler', 'sleep', 'test', 'tri', 'tril_indices', 'wait', 'zfill'}) import sys from abc import ABCMeta @@ -403,7 +403,7 @@ def __getattr__(self, name): # Each object re-enters a protocol on a PARTNER object, so the recursion crosses object boundaries # (mutual recursion, harder to short-circuit than plain self-recursion). CPython raises # RecursionError on the protocol call; an interpreter without a recursion guard on that native -# path overflows its C/Rust stack -> segfault. Construction is cheap -- the recursion fires only +# path overflows its C/Rust stack instead. Construction is cheap -- the recursion fires only # when the fuzzer exercises the named protocol (hash/eq/getitem/iter/repr/call) on the object. class _TrickyRecur: def __init__(self, name): @@ -436,7 +436,7 @@ def __call__(self, *args, **kwargs): # Deep generic-alias nesting list[list[...list[T]...]] bottomed on a TypeVar so the parameter walk # actually recurses to collect it -- exercises the genericalias parameter-walk native path -# (RustPython segfaulted in genericalias::make_parameters_from_slice). Bounded depth so construction +# (RustPython crashed in genericalias::make_parameters_from_slice). Bounded depth so construction # + a CPython repr stay well under the recursion limit; the native walk (and __getitem__ # substitution) is the target. Falls back to a plain nested alias if TypeVar is unavailable. try: diff --git a/tests/python/test_blacklists.py b/tests/python/test_blacklists.py index 7c66835..47f1dd5 100644 --- a/tests/python/test_blacklists.py +++ b/tests/python/test_blacklists.py @@ -50,6 +50,20 @@ def test_asyncio_runner_on_sigint_blacklisted(self): # because --test-private exposes the underscore-prefixed method. self.assertIn("_on_sigint", bl.BLACKLIST["asyncio.runners:Runner"]) + def test_sigint_handlers_blacklisted_by_NAME_not_only_by_module(self): + """The module-keyed entry alone is not enough, and a fleet proved it. + + `Runner` is defined in `asyncio.runners` but re-exported as `asyncio.Runner`, so a + session whose target module is `asyncio` never matches the + `"asyncio.runners:Runner"` key -- and reached `_on_sigint` anyway, through the + runtime generic-method loop. `pdb.sigint_handler` is the same shape and was never + keyed at all. Both raise KeyboardInterrupt unconditionally; uncaught, the interpreter + re-raises SIGINT, the process looks "killed by signal 2", and WatchProcess scores it + 1.0. METHOD_BLACKLIST is name-based, so it covers every path. + """ + for name in ("_on_sigint", "sigint_handler"): + self.assertIn(name, bl.METHOD_BLACKLIST, name) + def test_default_int_handler_blacklisted(self): # It raises KeyboardInterrupt, a BaseException, which escapes the generated script's # `except Exception` handlers and kills the session outright (the #192 class). diff --git a/tests/python/test_golden_output.py b/tests/python/test_golden_output.py index 9d61d15..d3d713d 100644 --- a/tests/python/test_golden_output.py +++ b/tests/python/test_golden_output.py @@ -173,6 +173,56 @@ def test_full_script_matches_snapshot(self): ) +class SelfNoiseVocabularyTests(unittest.TestCase): + """Emitted COMMENTS must not contain fusil's own crash vocabulary. + + ``pydoc.getdoc`` falls back to ``inspect.getcomments()`` for an object with no + docstring, so the comment block immediately above an undocumented class in a spliced + sample file is printed verbatim by ``help(obj)``. A fuzz session that calls + ``help()`` -- ``_sitebuiltins._Helper.__call__`` is a normal fuzz target -- therefore + echoes those comments into stdout, where ``WatchStdout`` matches them and manufactures + a crash. One PyPy fleet kept a ``_sitebuiltins-segfault`` dir this way, scored 100% on + fusil's own word ``segfault`` inside a ``tricky_objects.py`` comment. + + Only comment lines are checked: the scored words also occur in real emitted CODE + (``SystemError`` in the bomb exception list, ``AssertionError`` in a class statement), + which ``inspect.getcomments`` never reaches. + """ + + # The 1.0-scoring words from Fuzzer.setupProject's WatchStdout configuration. The + # sub-1.0 words ("bug", "fatal", "oops") are not listed: they cannot score a session + # on their own and appear unavoidably in explanatory prose. + CRASH_WORDS = ( + "assertion", + "critical", + "panic", + "panicked", + "glibc detected", + "segfault", + "segmentation fault", + "addresssanitizer", + ) + + def test_emitted_comments_are_free_of_crash_words(self): + offenders = [] + for lineno, line in enumerate(generate().splitlines(), 1): + stripped = line.strip() + if not stripped.startswith("#"): + continue + lowered = stripped.lower() + for word in self.CRASH_WORDS: + if word in lowered: + offenders.append((lineno, word, stripped[:100])) + break + self.assertEqual( + offenders, + [], + "generated comments contain crash vocabulary that help() would echo into " + "stdout, scoring the session as a false crash; reword them:\n" + + "\n".join(" line %d [%s] %s" % o for o in offenders), + ) + + def _update_snapshot(): GOLDEN_DIR.mkdir(parents=True, exist_ok=True) GOLDEN_FILE.write_text(generate(), encoding="utf-8") diff --git a/tests/test_file_watch.py b/tests/test_file_watch.py index d2fe223..c4c7401 100644 --- a/tests/test_file_watch.py +++ b/tests/test_file_watch.py @@ -289,6 +289,73 @@ def test_ignore_regex_covers_every_raised_bomb_signature(self): ) +class TestTracebackEchoIgnored(unittest.TestCase): + """A traceback is the target quoting ITSELF; neither half of it may score as a crash. + + A routine fuzz traceback through ``concurrent/futures/_base.py`` kept a crash dir in a + PyPy fleet twice, via two different lines of the same traceback: + + * the frame line ``File ".../logging/__init__.py", line 1536, in critical`` + * the source line ``LOGGER.critical('Future %s in unexpected state: %s',`` + + ``critical`` is a 1.0 word, so either alone scores the session 100%. Both are ignored; a + real critical-level MESSAGE (``CRITICAL:root:...``) matches neither and still scores. + + The frame-line rule requires the comma that Python tracebacks put before ``in``. + faulthandler writes ``line N in func`` WITHOUT one, so a genuine fatal-signal report is + untouched -- covered by the last test here. + """ + + FRAME_LINE = r'^\s*File "[^"]*", line \d+, in ' + SOURCE_LINE = r"\.critical\(" + + def _watch_with_rules(self): + w = _watch(words={"critical": 1.0}) + w.ignoreRegex(self.FRAME_LINE) + w.ignoreRegex(self.SOURCE_LINE) + return w + + def test_echoed_source_line_is_ignored(self): + """Shape 2: the offending source, echoed below the frame line.""" + w = self._watch_with_rules() + self.assertIsNone( + w.processLine(b" LOGGER.critical('Future %s in unexpected state: %s',") + ) + self.assertEqual(w.score, 0.0) + + def test_traceback_frame_line_is_ignored(self): + """Shape 1: the frame line, whose FUNCTION name is the scored word.""" + w = self._watch_with_rules() + self.assertIsNone( + w.processLine(b' File "/usr/lib/pypy3.11/logging/__init__.py", line 1536, in critical') + ) + self.assertEqual(w.score, 0.0) + + def test_a_real_critical_message_still_scores(self): + w = self._watch_with_rules() + w.processLine(b"CRITICAL:root:the target said something critical") + self.assertEqual(w.score, 1.0) + + def test_bdb_tracer_lines_are_ignored(self): + """bdb echoes the VALUE of every traced event; its repr may hold a scored word.""" + w = _watch(words={"systemerror": 1.0}) + w.ignoreRegex(r"^(\+\+\+|---|!!!) ") + self.assertIsNone(w.processLine(b"+++ return ")) + self.assertEqual(w.score, 0.0) + w.processLine(b"SystemError: the target actually raised one") + self.assertEqual(w.score, 1.0) + + def test_faulthandler_stack_lines_are_not_swallowed(self): + """faulthandler writes `line N in func` with NO comma; only real tracebacks have one. + + The frame-line rule must not reach faulthandler's stack, which is how a genuine + fatal-signal crash is reported. + """ + w = self._watch_with_rules() + w.processLine(b' File "/tmp/session/source.py", line 1360 in critical') + self.assertEqual(w.score, 1.0) + + class TestCookiejarWarningIgnored(unittest.TestCase): """http.cookiejar's own "bug!" warning must not push a boring session over the threshold.