From 71336ac9ccbdfd851987f6235347e6b0967ed391 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Nov 2026 09:16:07 -0400 Subject: [PATCH 01/13] reactors: dispatch management completion on shared I/O loops Management completions must run on the application's I/O reactor without blocking lifecycle notification delivery or starting a second I/O loop. Share each reactor's existing runner, serialize startup, and keep libev alive while management channels borrow it. Reset child reactor ownership after fork so inherited parent locks and descriptors cannot strand child submissions. --- cassandra/io/asyncioreactor.py | 46 +++- cassandra/io/asyncorereactor.py | 112 +++++++--- cassandra/io/libevreactor.py | 142 ++++++++++-- tests/unit/io/test_asyncioreactor.py | 8 +- tests/unit/io/test_libevreactor_shutdown.py | 12 ++ tests/unit/test_management_reactors.py | 225 ++++++++++++++++++++ 6 files changed, 490 insertions(+), 55 deletions(-) create mode 100644 tests/unit/test_management_reactors.py diff --git a/cassandra/io/asyncioreactor.py b/cassandra/io/asyncioreactor.py index 20fe79b851..1688a160f2 100644 --- a/cassandra/io/asyncioreactor.py +++ b/cassandra/io/asyncioreactor.py @@ -7,7 +7,7 @@ import os import socket import ssl -from threading import Lock, Thread +from threading import Event, Lock, Thread, current_thread log = logging.getLogger(__name__) @@ -258,13 +258,34 @@ def initialize_reactor(cls): cls._pid = os.getpid() if cls._loop is None: assert cls._loop_thread is None - cls._loop = asyncio.new_event_loop() - # daemonize so the loop will be shut down on interpreter - # shutdown - cls._loop_thread = Thread( - target=cls._loop.run_forever, daemon=True, name="asyncio_thread" - ) - cls._loop_thread.start() + loop = asyncio.new_event_loop() + thread = Thread(target=loop.run_forever, daemon=True, + name="asyncio_thread") + ready = Event() + try: + loop.call_soon(ready.set) + thread.start() + if not ready.wait(5): + raise RuntimeError("Asyncio reactor startup did not complete") + except BaseException: + if thread.is_alive(): + loop.call_soon_threadsafe(loop.stop) + thread.join(5) + if not thread.is_alive(): + loop.close() + raise + cls._loop, cls._loop_thread = loop, thread + + @classmethod + def _management_dispatch(cls, callback): + loop = cls._loop + if loop is None or loop.is_closed() or not cls._loop_thread.is_alive(): + raise RuntimeError("Asyncio reactor is unavailable") + loop.call_soon_threadsafe(callback) + + @classmethod + def _management_in_context(cls): + return current_thread() is cls._loop_thread @classmethod def create_timer(cls, timeout, callback): @@ -381,3 +402,12 @@ async def handle_read(self): log.debug("Connection %s closed by server", self) self.close() return + + +def _reset_management_startup_lock_after_fork(): + # A parent thread may own this lock when another thread forks. + AsyncioConnection._lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 4d19bb9849..c91e1a2ebf 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -18,7 +18,7 @@ import os import socket import sys -from threading import Lock, Thread, Event +from threading import Lock, Thread, Event, current_thread import time import sys import ssl @@ -210,6 +210,9 @@ class AsyncoreLoop(object): def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() + self._state_lock = Lock() + self._management_lock = Lock() + self._management_callbacks = deque() self._started = False self._shutdown = False @@ -228,21 +231,38 @@ def __init__(self): self._loop_dispatcher = dispatcher def maybe_start(self): - should_start = False - did_acquire = False - try: - did_acquire = self._loop_lock.acquire(False) - if did_acquire and not self._started: - self._started = True - should_start = True - finally: - if did_acquire: - self._loop_lock.release() - - if should_start: - self._thread = Thread(target=self._run_loop, name="asyncore_cassandra_driver_event_loop") - self._thread.daemon = True - self._thread.start() + with self._state_lock: + if self._shutdown or self._started: + return + self._started = True + self._thread = Thread(target=self._run_loop, + name="asyncore_cassandra_driver_event_loop", daemon=True) + try: + self._thread.start() + except BaseException: + self._started = False + self._thread = None + raise + + def call_soon(self, callback): + with self._management_lock: + if self._shutdown: + raise RuntimeError("Asyncore reactor is shutting down") + self._management_callbacks.append(callback) + self.maybe_start() + self.wake_loop() + + def _dispatch_management(self): + for _ in range(64): + with self._management_lock: + if not self._management_callbacks: + return + callback = self._management_callbacks.popleft() + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + self.wake_loop() def wake_loop(self): self._loop_dispatcher.notify_loop() @@ -254,10 +274,12 @@ def _run_loop(self): try: self._loop_dispatcher.loop(self.timer_resolution) self._timers.service_timeouts() + self._dispatch_management() except Exception as exc: self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) break - self._started = False + with self._state_lock: + self._started = False self._maybe_log_debug("Asyncore event loop ended") @@ -281,13 +303,14 @@ def add_timer(self, timer): def _cleanup(self): global _dispatcher_map - self._shutdown = True - if not self._thread: - return + with self._state_lock: + self._shutdown = True + self.wake_loop() log.debug("Waiting for event loop thread to join...") - self._thread.join(timeout=1.0) - if self._thread.is_alive(): + if self._thread is not None and self._thread is not current_thread(): + self._thread.join(timeout=1.0) + if self._thread is not None and self._thread.is_alive(): log.warning( "Event loop thread could not be joined, so shutdown may not be clean. " "Please call Cluster.shutdown() to avoid this.") @@ -306,6 +329,7 @@ def _cleanup(self): _global_loop = None +_initialization_lock = Lock() atexit.register(partial(_cleanup, _global_loop)) @@ -320,15 +344,32 @@ class AsyncoreConnection(Connection, asyncore.dispatcher): @classmethod def initialize_reactor(cls): - global _global_loop - if not _global_loop: - _global_loop = AsyncoreLoop() - else: - current_pid = os.getpid() - if _global_loop._pid != current_pid: - log.debug("Detected fork, clearing and reinitializing reactor state") - cls.handle_fork() - _global_loop = AsyncoreLoop() + global _global_loop, _dispatcher_map + with _initialization_lock: + if _global_loop is not None and _global_loop._pid != os.getpid(): + _dispatcher_map = {} + _global_loop = None + if _global_loop is None: + candidate = AsyncoreLoop() + ready = Event() + try: + candidate.call_soon(ready.set) + if not ready.wait(5): + raise RuntimeError("Asyncore reactor startup did not complete") + except BaseException: + candidate._cleanup() + raise + _global_loop = candidate + + @classmethod + def _management_dispatch(cls, callback): + if _global_loop is None: + raise RuntimeError("Asyncore reactor is unavailable") + _global_loop.call_soon(callback) + + @classmethod + def _management_in_context(cls): + return _global_loop is not None and current_thread() is _global_loop._thread @classmethod def handle_fork(cls): @@ -476,3 +517,12 @@ def writable(self): def readable(self): return self._readable or ((self.is_control_connection or self._continuous_paging_sessions) and not (self.is_defunct or self.is_closed)) + + +def _reset_management_startup_lock_after_fork(): + global _initialization_lock + _initialization_lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index f963d283ac..3f5dc0c371 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -17,7 +17,7 @@ import os import socket import ssl -from threading import Lock, RLock, Thread +from threading import Event, Lock, RLock, Thread, current_thread import time from cassandra import DependencyException @@ -83,6 +83,8 @@ def __init__(self): self._closed_conns = set() # connections closed while their thread held _lock; drained by the loop self._deferred_destroyed = [] + self._management_callbacks = deque() + self._management_users = set() self._preparer = libev.Prepare(self._loop, self._loop_will_run) # prevent _preparer from keeping the loop from returning @@ -91,6 +93,7 @@ def __init__(self): self._timers = TimerManager() self._loop_timer = libev.Timer(self._loop, self._on_loop_timer) + self._management_keeper = libev.Timer(self._loop, self._keep_management_alive) def maybe_start(self): should_start = False @@ -105,7 +108,13 @@ def maybe_start(self): if not self._shutdown: self._thread = Thread(target=self._run_loop, name="event_loop") self._thread.daemon = True - self._thread.start() + try: + self._thread.start() + except BaseException: + with self._lock: + self._started = False + self._thread = None + raise self._notifier.send() @@ -124,7 +133,7 @@ def _run_loop(self): # connection_created(), then observes _started == False and # starts a fresh thread). There is no interleaving in which # the new connection is invisible to both. - if not self._shutdown and self._live_conns: + if not self._shutdown and (self._live_conns or self._management_callbacks): log.debug("Restarting event loop") continue else: @@ -148,9 +157,6 @@ def _run_loop(self): def _cleanup(self): # TODO: unguarded write; self._notify() below wakes _run_loop to re-check. self._shutdown = True - if not self._thread: - return - # Stop the prepare watcher first to prevent race conditions. # It was unref'd when started, so ref the loop back first: stopping it # unbalanced leaves libev's active count at -1, and a loop thread still @@ -161,6 +167,7 @@ def _cleanup(self): self._loop.ref() self._preparer.stop() self._loop_timer.stop() + self._management_keeper.stop() # TODO: unguarded read; sets are copy-on-write, so at most a stale # snapshot (redundant idempotent close), never a torn read. @@ -176,9 +183,10 @@ def _cleanup(self): # PYTHON-752 Thread might have just been created and not started # Use longer timeout to allow proper cleanup with self._lock_thread: - self._thread.join(timeout=5.0) + if self._thread is not None and self._thread is not current_thread(): + self._thread.join(timeout=5.0) - if self._thread.is_alive(): + if self._thread is not None and self._thread.is_alive(): log.warning( "Event loop thread could not be joined, so shutdown may not be clean. " "Please call Cluster.shutdown() to avoid this.") @@ -203,6 +211,77 @@ def _update_timer(self): def _on_loop_timer(self): self._timers.service_timeouts() + def call_soon(self, callback): + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + self._management_callbacks.append(callback) + try: + self.maybe_start() + except BaseException: + with self._lock: + try: + self._management_callbacks.remove(callback) + except ValueError: + return # Already claimed: the dispatch was accepted. + raise + + def _keep_management_alive(self): + if self._management_users and not self._shutdown: + self._management_keeper.start(3600) + + def borrow_management(self): + token = object() + ready, abandoned = Event(), Event() + + def acquire(): + if abandoned.is_set(): + return + self._management_users.add(token) + self._management_keeper.start(3600) + if abandoned.is_set(): + release(lambda: None) + else: + ready.set() + + def release(completed): + def detach(): + self._management_users.discard(token) + if not self._management_users: + self._management_keeper.stop() + completed() + if current_thread() is self._thread: + detach() + elif self._shutdown or self._thread is None or not self._thread.is_alive(): + # Catastrophic service loss: no native runner can touch watchers. + with self._lock: + detach() + else: + self.call_soon(detach) + + if current_thread() is self._thread: + acquire() + else: + self.call_soon(acquire) + if not ready.wait(5): + abandoned.set() + release(lambda: None) + raise RuntimeError('Libev management borrow did not complete') + return release + + def _dispatch_management(self): + # Bounded batches keep management completions fair to socket I/O. + for _ in range(64): + with self._lock: + if not self._management_callbacks: + return + callback = self._management_callbacks.popleft() + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + self._notifier.send() + def notify(self): self._notifier.send() @@ -288,6 +367,7 @@ def _close_conns(conns): def _loop_will_run(self, prepare): changed = False self._drain_deferred() + self._dispatch_management() # TODO: unguarded read; copy-on-write set, self-heals next tick. for conn in self._live_conns: @@ -341,6 +421,7 @@ def _atexit_cleanup(): _global_loop = None +_initialization_lock = Lock() atexit.register(_atexit_cleanup) @@ -356,13 +437,35 @@ class LibevConnection(Connection): @classmethod def initialize_reactor(cls): global _global_loop - if not _global_loop: - _global_loop = LibevLoop() - else: - if _global_loop._pid != os.getpid(): - log.debug("Detected fork, clearing and reinitializing reactor state") - cls.handle_fork() - _global_loop = LibevLoop() + with _initialization_lock: + if _global_loop is not None and _global_loop._pid != os.getpid(): + # Inherited threads cannot be joined in the child. + _global_loop = None + if _global_loop is None: + candidate = LibevLoop() + ready = Event() + try: + candidate.call_soon(ready.set) + if not ready.wait(5): + raise RuntimeError("Libev reactor startup did not complete") + except BaseException: + candidate._cleanup() + raise + _global_loop = candidate + + @classmethod + def _management_dispatch(cls, callback): + if _global_loop is None: + raise RuntimeError("Libev reactor is unavailable") + _global_loop.call_soon(callback) + + @classmethod + def _management_borrow(cls): + return _global_loop.borrow_management() + + @classmethod + def _management_in_context(cls): + return _global_loop is not None and current_thread() is _global_loop._thread @classmethod def handle_fork(cls): @@ -520,3 +623,12 @@ def push(self, data): with self._deque_lock: self.deque.extend(chunks) _global_loop.notify() + + +def _reset_management_startup_lock_after_fork(): + global _initialization_lock + _initialization_lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/tests/unit/io/test_asyncioreactor.py b/tests/unit/io/test_asyncioreactor.py index 76aca20ef3..31bef870da 100644 --- a/tests/unit/io/test_asyncioreactor.py +++ b/tests/unit/io/test_asyncioreactor.py @@ -41,7 +41,13 @@ def tearDownClass(cls): if skip_me: return if ASYNCIO_AVAILABLE and AsyncioConnection._loop: - AsyncioConnection._loop.stop() + loop = AsyncioConnection._loop + loop.call_soon_threadsafe(loop.stop) + AsyncioConnection._loop_thread.join(5) + assert not AsyncioConnection._loop_thread.is_alive() + loop.close() + AsyncioConnection._loop = None + AsyncioConnection._loop_thread = None @property def create_timer(self): diff --git a/tests/unit/io/test_libevreactor_shutdown.py b/tests/unit/io/test_libevreactor_shutdown.py index af5b420888..27bfe1e3e5 100644 --- a/tests/unit/io/test_libevreactor_shutdown.py +++ b/tests/unit/io/test_libevreactor_shutdown.py @@ -26,6 +26,7 @@ import tempfile import os from pathlib import Path +from unittest.mock import patch from cassandra import DependencyException @@ -216,6 +217,17 @@ def test_cleanup_with_fix_properly_shuts_down(self): from cassandra.io import libevreactor from cassandra.io.libevreactor import _cleanup, _atexit_cleanup + # Simulated process exit must not stop the shared reactor used by later + # tests. Restore its singleton even if an assertion below fails. + singleton = patch.object(libevreactor, '_global_loop', None) + singleton.start() + self.addCleanup(singleton.stop) + def close_private_loop(): + loop = libevreactor._global_loop + if loop is not None and not loop._shutdown: + _cleanup(loop) + self.addCleanup(close_private_loop) + LibevConnection.initialize_reactor() # Verify the loop exists diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py new file mode 100644 index 0000000000..9c3686a70c --- /dev/null +++ b/tests/unit/test_management_reactors.py @@ -0,0 +1,225 @@ +"""Socket-free tests of the actual supported reactor completion adapters.""" + +import asyncio +import importlib +from concurrent.futures import CancelledError, Future +from threading import Event, Thread, get_ident + +import pytest + +from cassandra import DependencyException +from cassandra._management import ( + ManagementRuntime, ReactorChannel, execution_context) + + +@pytest.fixture(autouse=True) +def close_borrowed_channels(monkeypatch): + channels = [] + initialize = ReactorChannel.__init__ + def tracked(self, *args, **kwargs): + initialize(self, *args, **kwargs) + channels.append(self) + monkeypatch.setattr(ReactorChannel, '__init__', tracked) + yield + for channel in channels: + detached = Event() + channel.close(detached.set) + assert detached.wait(5) + + +@pytest.fixture(params=['asyncio', 'libev', 'asyncore']) +def connection_class(request): + try: + module = importlib.import_module('cassandra.io.' + { + 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', + 'asyncore': 'asyncorereactor'}[request.param]) + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + return getattr(module, {'asyncio': 'AsyncioConnection', + 'libev': 'LibevConnection', + 'asyncore': 'AsyncoreConnection'}[request.param]) + + +def test_idle_reactor_completion_runs_on_io_thread(connection_class): + channel = ReactorChannel(connection_class) + for _ in range(3): + finished = Event() + observations = [] + def on_io(): + assert execution_context() is None # Ordinary ResponseFuture context. + assert connection_class._management_in_context() + observations.append(get_ident()) + finished.set() + channel.dispatch(on_io) + assert finished.wait(5) + assert observations[0] != get_ident() + + +def test_clusters_borrow_reactor_without_closing_other_channel(connection_class): + channel = ReactorChannel(connection_class) + first = ManagementRuntime(channel.dispatch, on_finalized=channel.close) + second_channel = ReactorChannel(connection_class) + second = ManagementRuntime(second_channel.dispatch, on_finalized=second_channel.close) + callback_entered, release, other_done = Event(), Event(), Event() + handler_entered, handler_release = Event(), Event() + try: + def handler(record): + handler_entered.set() + assert handler_release.wait(5) + return 3 + future = first.submit(handler) + assert handler_entered.wait(5) + def hold(future): + callback_entered.set() + assert release.wait(5) + future.add_done_callback(hold) + handler_release.set() + assert future.result(5) == 3 + assert callback_entered.wait(5) + first.close() + assert not first.finalized.is_set() + # Shared reactor may be blocked by application code; native state still + # progresses and the second channel works as soon as that code returns. + other = second.submit(lambda record: 8) + other.add_done_callback(lambda f: other_done.set()) + assert other.result(5) == 8 + release.set() + assert other_done.wait(5) + first.join(5) + assert second.submit(lambda record: 9).result(5) == 9 + finally: + handler_release.set() + release.set() + first.close() + second.close() + first.join(5) + second.join(5) + + +@pytest.mark.parametrize('terminal', ['result', 'error', 'cancel']) +@pytest.mark.parametrize('registered', ['before', 'after']) +def test_asyncio_chaining_progresses_while_listener_waits(connection_class, terminal, registered): + channel = ReactorChannel(connection_class) + manager = ManagementRuntime(channel.dispatch, on_finalized=channel.close) + application_loop = asyncio.new_event_loop() + app_thread = Thread(target=application_loop.run_forever) + app_thread.start() + entered, complete = Event(), Event() + release_handler = Event() + outcomes = [] + try: + def listener(): + def handler(record): + assert release_handler.wait(5) + if terminal == 'error': + raise ValueError('expected') + return 12 + source = manager.submit(handler) + if registered == 'after': + if terminal == 'cancel': + source.cancel() + release_handler.set() + try: + source.result(5) + except (ValueError, CancelledError): + pass + async def consume(): + wrapped = asyncio.wrap_future(source) + entered.set() + return await wrapped + chained = asyncio.run_coroutine_threadsafe(consume(), application_loop) + if registered == 'before': + assert entered.wait(5) + if terminal == 'cancel': + source.cancel() + release_handler.set() + try: + outcomes.append(chained.result(5)) + except (ValueError, CancelledError) as exc: + outcomes.append(type(exc)) + complete.set() + notification = manager.notify(listener) + assert complete.wait(5) + notification.result(5) + assert outcomes == [{'result': 12, 'error': ValueError, 'cancel': CancelledError}[terminal]] + finally: + release_handler.set() + manager.close() + manager.join(5) + application_loop.call_soon_threadsafe(application_loop.stop) + app_thread.join(5) + application_loop.close() + + +def test_libev_dispatch_enqueued_during_idle_exit_is_not_lost(): + try: + module = importlib.import_module('cassandra.io.libevreactor') + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + loop = module.LibevLoop() + native = loop._loop + returned, release, first, second = Event(), Event(), Event(), Event() + class PausedExit: + paused = False + def start(self): + native.start() + if not self.paused: + self.paused = True + returned.set() + assert release.wait(5) + def __getattr__(self, name): + return getattr(native, name) + loop._loop = PausedExit() + try: + loop.call_soon(first.set) + assert first.wait(5) + assert returned.wait(5) + loop.call_soon(second.set) + release.set() + assert second.wait(5), 'idle exit lost a completion queued before exit committed' + loop._thread.join(5) + assert not loop._thread.is_alive() + finally: + release.set() + loop._cleanup() + + +def test_reactor_startup_lock_is_reset_in_fork_child(connection_class): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + channel = ReactorChannel(connection_class) + module = importlib.import_module(connection_class.__module__) + lock = (connection_class._lock if module.__name__.endswith('asyncioreactor') + else module._initialization_lock) + entered, release = Event(), Event() + def holder(): + with lock: + entered.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert entered.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + child_channel = ReactorChannel(connection_class) + os.write(write_fd, b'started') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'new child reactor entered inherited startup lock' + assert os.read(read_fd, 8) == b'started' + finally: + release.set() + thread.join(5) + os.close(read_fd) + os.waitpid(pid, 0) From b159eca8aac9cadf2b2db1f7f064f9250f0023f2 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 01:09:39 -0400 Subject: [PATCH 02/13] reactors: drain admitted completions before runner exit Serialize callback admission with startup and shutdown, and drain accepted callbacks on the reactor thread when dispatch stops. Reject later submissions after asyncore service loss and remove callbacks whose startup failed. Close inherited socket and dispatcher descriptors before creating the child reactor without acquiring parent locks. Cover startup failures, shutdown backlogs, loop failure and descriptor ownership with socket-free tests. --- cassandra/io/asyncioreactor.py | 46 ++++- cassandra/io/asyncorereactor.py | 65 +++++-- cassandra/io/libevreactor.py | 90 ++++++---- tests/unit/test_management_reactors.py | 236 +++++++++++++++++++++++++ 4 files changed, 389 insertions(+), 48 deletions(-) diff --git a/cassandra/io/asyncioreactor.py b/cassandra/io/asyncioreactor.py index 1688a160f2..03745f5ab8 100644 --- a/cassandra/io/asyncioreactor.py +++ b/cassandra/io/asyncioreactor.py @@ -1,4 +1,5 @@ import threading +from collections import deque from cassandra.connection import Connection, ConnectionShutdown import sys @@ -133,6 +134,7 @@ class AsyncioConnection(Connection): _lock = Lock() _loop_thread = None + _management_state = None _write_queue = None _write_queue_lock = None @@ -259,7 +261,34 @@ def initialize_reactor(cls): if cls._loop is None: assert cls._loop_thread is None loop = asyncio.new_event_loop() - thread = Thread(target=loop.run_forever, daemon=True, + pending, management_lock = deque(), Lock() + accepting = [True] + + def drain(final=False): + count = 0 + while final or count < 64: + with management_lock: + if not pending: + return + callback = pending.popleft() + count += 1 + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + with management_lock: + if pending and accepting[0]: + loop.call_soon(drain) + + def run(): + try: + loop.run_forever() + finally: + with management_lock: + accepting[0] = False + drain(final=True) + + thread = Thread(target=run, daemon=True, name="asyncio_thread") ready = Event() try: @@ -275,13 +304,24 @@ def initialize_reactor(cls): loop.close() raise cls._loop, cls._loop_thread = loop, thread + cls._management_state = (pending, management_lock, accepting, drain) @classmethod def _management_dispatch(cls, callback): loop = cls._loop - if loop is None or loop.is_closed() or not cls._loop_thread.is_alive(): + if cls._management_state is None: raise RuntimeError("Asyncio reactor is unavailable") - loop.call_soon_threadsafe(callback) + pending, lock, accepting, drain = cls._management_state + with lock: + if (not accepting[0] or loop is None or loop.is_closed() or + not cls._loop_thread.is_alive()): + raise RuntimeError("Asyncio reactor is unavailable") + pending.append(callback) + try: + loop.call_soon_threadsafe(drain) + except BaseException: + pending.remove(callback) + raise @classmethod def _management_in_context(cls): diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index c91e1a2ebf..fdd600f0e9 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -18,7 +18,7 @@ import os import socket import sys -from threading import Lock, Thread, Event, current_thread +from threading import Lock, RLock, Thread, Event, current_thread import time import sys import ssl @@ -137,6 +137,14 @@ def notify_loop(self): self._notified = True os.write(self.write_fd, b'x') + def close(self): + try: + asyncore.dispatcher.close(self) + finally: + if self.write_fd is not None: + os.close(self.write_fd) + self.write_fd = None + class _AsyncoreUDPDispatcher(_AsyncoreDispatcher): """ @@ -210,11 +218,12 @@ class AsyncoreLoop(object): def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() - self._state_lock = Lock() - self._management_lock = Lock() + self._state_lock = RLock() + self._management_lock = self._state_lock self._management_callbacks = deque() self._started = False self._shutdown = False + self._cleanup_started = False self._thread = None @@ -240,8 +249,9 @@ def maybe_start(self): try: self._thread.start() except BaseException: - self._started = False - self._thread = None + if not self._thread.is_alive(): + self._started = False + self._thread = None raise def call_soon(self, callback): @@ -249,8 +259,15 @@ def call_soon(self, callback): if self._shutdown: raise RuntimeError("Asyncore reactor is shutting down") self._management_callbacks.append(callback) - self.maybe_start() - self.wake_loop() + try: + self.maybe_start() + self.wake_loop() + except BaseException: + try: + self._management_callbacks.remove(callback) + except ValueError: + return + raise def _dispatch_management(self): for _ in range(64): @@ -270,16 +287,23 @@ def wake_loop(self): def _run_loop(self): log.debug("Starting asyncore event loop") with self._loop_lock: - while not self._shutdown: - try: + try: + while not self._shutdown: self._loop_dispatcher.loop(self.timer_resolution) self._timers.service_timeouts() self._dispatch_management() - except Exception as exc: - self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) - break - with self._state_lock: - self._started = False + except BaseException as exc: + self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) + finally: + # Commit service loss atomically with admission. Accepted + # callbacks still run on this runner before it relinquishes + # ownership; later submissions receive an explicit rejection. + with self._state_lock: + self._shutdown = True + while self._management_callbacks: + self._dispatch_management() + with self._state_lock: + self._started = False self._maybe_log_debug("Asyncore event loop ended") @@ -304,6 +328,9 @@ def _cleanup(self): global _dispatcher_map with self._state_lock: + if self._cleanup_started: + return + self._cleanup_started = True self._shutdown = True self.wake_loop() @@ -347,6 +374,16 @@ def initialize_reactor(cls): global _global_loop, _dispatcher_map with _initialization_lock: if _global_loop is not None and _global_loop._pid != os.getpid(): + # Close descriptors directly: Connection.close() can acquire + # parent-owned locks or schedule timers on a vanished runner. + for dispatcher in tuple(_dispatcher_map.values()): + try: + if dispatcher is _global_loop._loop_dispatcher: + dispatcher.close() + else: + dispatcher.socket.close() + except OSError: + pass _dispatcher_map = {} _global_loop = None if _global_loop is None: diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 3f5dc0c371..b8c100328a 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -70,7 +70,7 @@ def __init__(self): # re-entered by a thread that already holds it, via _is_owned(): # private, but threading.Condition relies on it too. self._lock = RLock() - self._lock_thread = Lock() + self._lock_thread = RLock() self._thread = None @@ -85,6 +85,7 @@ def __init__(self): self._deferred_destroyed = [] self._management_callbacks = deque() self._management_users = set() + self._cleanup_started = False self._preparer = libev.Prepare(self._loop, self._loop_will_run) # prevent _preparer from keeping the loop from returning @@ -96,27 +97,24 @@ def __init__(self): self._management_keeper = libev.Timer(self._loop, self._keep_management_alive) def maybe_start(self): - should_start = False - with self._lock: - if not self._started: - log.debug("Starting libev event loop") - self._started = True - should_start = True - - if should_start: - with self._lock_thread: - if not self._shutdown: + # Serialize all admissions with startup, including its failure path. + with self._lock_thread: + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + if not self._started: + log.debug("Starting libev event loop") + self._started = True self._thread = Thread(target=self._run_loop, name="event_loop") self._thread.daemon = True try: self._thread.start() except BaseException: - with self._lock: + if not self._thread.is_alive(): self._started = False self._thread = None raise - - self._notifier.send() + self._notifier.send() def _run_loop(self): while True: @@ -153,10 +151,18 @@ def _run_loop(self): self._close_conns(to_stop) break self._drain_deferred() + # Shutdown rejects new admissions; finish the accepted queue on this + # reactor thread even when cleanup has stopped the prepare watcher. + while self._management_callbacks: + self._dispatch_management() def _cleanup(self): - # TODO: unguarded write; self._notify() below wakes _run_loop to re-check. - self._shutdown = True + with self._lock_thread: + with self._lock: + if self._cleanup_started: + return + self._cleanup_started = True + self._shutdown = True # Stop the prepare watcher first to prevent race conditions. # It was unref'd when started, so ref the loop back first: stopping it # unbalanced leaves libev's active count at -1, and a loop thread still @@ -183,8 +189,9 @@ def _cleanup(self): # PYTHON-752 Thread might have just been created and not started # Use longer timeout to allow proper cleanup with self._lock_thread: - if self._thread is not None and self._thread is not current_thread(): - self._thread.join(timeout=5.0) + thread = self._thread + if thread is not None and thread is not current_thread(): + thread.join(timeout=5.0) if self._thread is not None and self._thread.is_alive(): log.warning( @@ -193,6 +200,26 @@ def _cleanup(self): log.debug("Event loop thread was joined") + def _cleanup_after_fork(self): + # Only this thread survived. Do not enter inherited Python locks, + # invoke connection callbacks, or join vanished parent threads. + self._shutdown = True + for conn in self._live_conns | self._new_conns | self._closed_conns: + try: + conn._socket.close() + except OSError: + pass + conn._read_watcher = conn._write_watcher = None + self._live_conns = self._new_conns = self._closed_conns = set() + self._deferred_destroyed = [] + self._management_callbacks.clear() + self._management_users.clear() + # Break watcher/callback cycles and release the native loop, whose + # destructor closes its internal wakeup descriptors in the child. + self._preparer = self._loop_timer = self._management_keeper = None + self._notifier = None + self._loop = None + def add_timer(self, timer): self._timers.add_timer(timer) self._notifier.send() # wake up in case this timer is earlier @@ -212,19 +239,20 @@ def _on_loop_timer(self): self._timers.service_timeouts() def call_soon(self, callback): - with self._lock: - if self._shutdown: - raise RuntimeError("Libev reactor is shutting down") - self._management_callbacks.append(callback) - try: - self.maybe_start() - except BaseException: + with self._lock_thread: with self._lock: - try: - self._management_callbacks.remove(callback) - except ValueError: - return # Already claimed: the dispatch was accepted. - raise + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + self._management_callbacks.append(callback) + try: + self.maybe_start() + except BaseException: + with self._lock: + try: + self._management_callbacks.remove(callback) + except ValueError: + return # Already claimed: the dispatch was accepted. + raise def _keep_management_alive(self): if self._management_users and not self._shutdown: @@ -439,7 +467,7 @@ def initialize_reactor(cls): global _global_loop with _initialization_lock: if _global_loop is not None and _global_loop._pid != os.getpid(): - # Inherited threads cannot be joined in the child. + _global_loop._cleanup_after_fork() _global_loop = None if _global_loop is None: candidate = LibevLoop() diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 9c3686a70c..29b5deff32 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -223,3 +223,239 @@ def holder(): thread.join(5) os.close(read_fd) os.waitpid(pid, 0) + + +@pytest.mark.parametrize('reactor', ['libev', 'asyncore']) +def test_startup_failure_does_not_acknowledge_concurrent_dispatch(monkeypatch, reactor): + module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + if reactor == 'asyncore': + monkeypatch.setattr(module, '_dispatcher_map', {}) + class Dispatcher: + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + Event().wait(0.001) + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() + starting, fail, second_entered, accepted, delivered = [Event() for _ in range(5)] + original_thread = module.Thread + calls = [] + class FailedStart: + def start(self): + starting.set() + assert fail.wait(5) + raise RuntimeError('thread creation failed') + def is_alive(self): + return False + def factory(*args, **kwargs): + calls.append(True) + return FailedStart() if len(calls) == 1 else original_thread(*args, **kwargs) + monkeypatch.setattr(module, 'Thread', factory) + errors, rejected_callbacks = [], [] + def first(): + try: + loop.call_soon(lambda: rejected_callbacks.append(True)) + except RuntimeError as exc: + errors.append(str(exc)) + def second(): + second_entered.set() + loop.call_soon(delivered.set) + accepted.set() + first_thread, second_thread = Thread(target=first), Thread(target=second) + try: + first_thread.start() + assert starting.wait(5) + second_thread.start() + assert second_entered.wait(5) + assert not accepted.wait(0.05) + fail.set() + first_thread.join(5) + second_thread.join(5) + assert accepted.is_set() + assert delivered.wait(5) + assert errors == ['thread creation failed'] + assert rejected_callbacks == [] + finally: + fail.set() + first_thread.join(5) + if second_thread.ident is not None: + second_thread.join(5) + loop._cleanup() + + +def test_asyncore_dispatch_accepted_during_runner_failure_is_delivered(monkeypatch): + module = pytest.importorskip('cassandra.io.asyncorereactor') + monkeypatch.setattr(module, '_dispatcher_map', {}) + entered, fail, delivered = Event(), Event(), Event() + class Dispatcher: + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + entered.set() + assert fail.wait(5) + raise RuntimeError('dispatcher failed') + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = module.AsyncoreLoop() + observations = [] + try: + loop.maybe_start() + assert entered.wait(5) + def completion(): + observations.append(get_ident()) + delivered.set() + loop.call_soon(completion) + fail.set() + loop._thread.join(5) + assert delivered.is_set() + assert observations == [loop._thread.ident] + with pytest.raises(RuntimeError, match='shutting down'): + loop.call_soon(lambda: None) + assert not loop._management_callbacks + finally: + fail.set() + loop._cleanup() + + +@pytest.mark.parametrize('reactor', ['asyncio', 'libev', 'asyncore']) +@pytest.mark.parametrize('count', [1, 130]) +def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, reactor, count): + module = pytest.importorskip('cassandra.io.' + { + 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', + 'asyncore': 'asyncorereactor'}[reactor]) + entered, release, delivered, stopping = [Event() for _ in range(4)] + if reactor == 'asyncore': + monkeypatch.setattr(module, '_dispatcher_map', {}) + if reactor == 'asyncio': + connection = type('IsolatedConnection', (module.AsyncioConnection,), + {'_loop': None, '_loop_thread': None}) + connection.initialize_reactor() + dispatch, runner = connection._management_dispatch, connection._loop_thread + def stop(): + connection._loop.call_soon_threadsafe(stopping.set) + connection._loop.call_soon_threadsafe(connection._loop.stop) + def cleanup(): + runner.join(5) + connection._loop.close() + else: + loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() + dispatch = loop.call_soon + def stop(): + stopping.set() + loop._cleanup() + def cleanup(): + loop._cleanup() + runner = None + observations = [] + def hold(): + entered.set() + assert release.wait(5) + stopper = None + try: + dispatch(hold) + assert entered.wait(5) + def completion(): + observations.append(get_ident()) + if len(observations) == count: + delivered.set() + for _ in range(count): + dispatch(completion) + if reactor == 'asyncio': + stop() + else: + runner = loop._thread + stopper = Thread(target=stop) + stopper.start() + assert stopping.wait(5) + # Cleanup commits shutdown under the admission lock before join. + for _ in range(500): + with (loop._lock if reactor == 'libev' else loop._state_lock): + if loop._shutdown: + break + Event().wait(0.001) + assert loop._shutdown + release.set() + assert delivered.wait(5) + runner.join(5) + assert not runner.is_alive() + if stopper is not None: + stopper.join(5) + assert not stopper.is_alive() + assert observations == [runner.ident] * count + with pytest.raises(RuntimeError): + dispatch(lambda: None) + finally: + release.set() + if stopper is not None: + stopper.join(5) + cleanup() + + +@pytest.mark.parametrize('reactor', ['libev', 'asyncore']) +def test_fork_reinitialization_closes_inherited_descriptors(monkeypatch, reactor): + import os + import select + import signal + import socket + from types import SimpleNamespace + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + connection = getattr(module, {'libev': 'LibevConnection', + 'asyncore': 'AsyncoreConnection'}[reactor]) + connection.initialize_reactor() + inherited = module._global_loop + ours, peer = socket.socketpair() + if reactor == 'libev': + class Connection: + _socket = ours + _read_watcher = _write_watcher = None + conn = Connection() + inherited._live_conns.add(conn) + else: + module._dispatcher_map[ours.fileno()] = SimpleNamespace(socket=ours) + original_factory = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor]) + def checked_factory(): + assert ours.fileno() == -1 + if reactor == 'libev': + assert inherited._loop is None + assert inherited._notifier is None + else: + assert inherited._loop_dispatcher.write_fd is None + return original_factory() + monkeypatch.setattr(module, original_factory.__name__, checked_factory) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + connection.initialize_reactor() + os.write(write_fd, b'closed') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready + assert os.read(read_fd, 6) == b'closed' + # The child closed only its descriptor copies. + ours.send(b'x') + assert peer.recv(1) == b'x' + finally: + if reactor == 'libev': + inherited._live_conns.discard(conn) + else: + module._dispatcher_map.pop(ours.fileno(), None) + ours.close() + peer.close() + os.close(read_fd) + os.waitpid(pid, 0) From 984798cfa719cec3fe98297b582e7396fe993708 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 03:19:48 -0400 Subject: [PATCH 03/13] reactors: preserve runner ownership during teardown and recovery Drain libev completions after exit only for the shutdown owner, so a retired idle runner cannot claim a replacement runner's work. Stop a management keeper armed concurrently with cleanup or final borrower release. Separate permanent asyncore management failure from ordinary I/O shutdown. Drain accepted completions before retiring the failed runner, reject stale management channels, and allow ordinary connections to restart I/O. Gate tests at the actual admission, idle-exit and timer-arm boundaries. Exercise asyncio final draining before queued normal drains can run, and always stop the chaining test's application loop when management join fails. --- cassandra/io/asyncorereactor.py | 16 +- cassandra/io/libevreactor.py | 18 +- tests/unit/test_management_reactors.py | 266 ++++++++++++++++++++++++- 3 files changed, 277 insertions(+), 23 deletions(-) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index fdd600f0e9..79dbfeef2e 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -219,8 +219,8 @@ def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() self._state_lock = RLock() - self._management_lock = self._state_lock self._management_callbacks = deque() + self._management_failed = False self._started = False self._shutdown = False self._cleanup_started = False @@ -255,8 +255,8 @@ def maybe_start(self): raise def call_soon(self, callback): - with self._management_lock: - if self._shutdown: + with self._state_lock: + if self._shutdown or self._management_failed: raise RuntimeError("Asyncore reactor is shutting down") self._management_callbacks.append(callback) try: @@ -271,7 +271,7 @@ def call_soon(self, callback): def _dispatch_management(self): for _ in range(64): - with self._management_lock: + with self._state_lock: if not self._management_callbacks: return callback = self._management_callbacks.popleft() @@ -295,11 +295,11 @@ def _run_loop(self): except BaseException as exc: self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) finally: - # Commit service loss atomically with admission. Accepted - # callbacks still run on this runner before it relinquishes - # ownership; later submissions receive an explicit rejection. + # Management service loss is permanent for existing channels. + # Ordinary I/O can restart after this runner drains accepted + # completions and relinquishes ownership. with self._state_lock: - self._shutdown = True + self._management_failed = True while self._management_callbacks: self._dispatch_management() with self._state_lock: diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index b8c100328a..a4ee93bb63 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -137,6 +137,7 @@ def _run_loop(self): else: # all Connections have been closed, no active watchers log.debug("All Connections currently closed, event loop ended") + shutdown = self._shutdown self._started = False # A connection may have been closed after the last # _loop_will_run() pass, which is where closed @@ -151,10 +152,11 @@ def _run_loop(self): self._close_conns(to_stop) break self._drain_deferred() - # Shutdown rejects new admissions; finish the accepted queue on this - # reactor thread even when cleanup has stopped the prepare watcher. - while self._management_callbacks: - self._dispatch_management() + # Only the shutdown owner may drain after publishing stopped state. + # Ordinary idle exit allows a replacement runner to claim new work. + if shutdown: + while self._management_callbacks: + self._dispatch_management() def _cleanup(self): with self._lock_thread: @@ -257,6 +259,9 @@ def call_soon(self, callback): def _keep_management_alive(self): if self._management_users and not self._shutdown: self._management_keeper.start(3600) + # Cleanup or release can stop the timer while start() is in flight. + if self._shutdown or not self._management_users: + self._management_keeper.stop() def borrow_management(self): token = object() @@ -266,7 +271,7 @@ def acquire(): if abandoned.is_set(): return self._management_users.add(token) - self._management_keeper.start(3600) + self._keep_management_alive() if abandoned.is_set(): release(lambda: None) else: @@ -281,7 +286,8 @@ def detach(): if current_thread() is self._thread: detach() elif self._shutdown or self._thread is None or not self._thread.is_alive(): - # Catastrophic service loss: no native runner can touch watchers. + # Shutdown closes admission before the runner finishes draining. + # Release remains idempotent while that runner unwinds. with self._lock: detach() else: diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 29b5deff32..bcb8539d44 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -3,7 +3,7 @@ import asyncio import importlib from concurrent.futures import CancelledError, Future -from threading import Event, Thread, get_ident +from threading import Event, Thread, current_thread, get_ident import pytest @@ -144,11 +144,14 @@ async def consume(): assert outcomes == [{'result': 12, 'error': ValueError, 'cancel': CancelledError}[terminal]] finally: release_handler.set() - manager.close() - manager.join(5) - application_loop.call_soon_threadsafe(application_loop.stop) - app_thread.join(5) - application_loop.close() + try: + manager.close() + manager.join(5) + finally: + application_loop.call_soon_threadsafe(application_loop.stop) + app_thread.join(5) + assert not app_thread.is_alive() + application_loop.close() def test_libev_dispatch_enqueued_during_idle_exit_is_not_lost(): @@ -184,6 +187,155 @@ def __getattr__(self, name): loop._cleanup() +def test_libev_retired_runner_does_not_claim_replacement_work(monkeypatch): + module = pytest.importorskip('cassandra.io.libevreactor') + loop = module.LibevLoop() + native, drain = loop._loop, loop._drain_deferred + retired, release_old, replacement, release_new, delivered = [Event() for _ in range(5)] + old_threads, observations = [], [] + def pause_retired(): + drain() + if not loop._started and not retired.is_set(): + old_threads.append(current_thread()) + retired.set() + assert release_old.wait(5) + class PausedReplacement: + def start(self): + if retired.is_set() and current_thread() is not old_threads[0]: + replacement.set() + assert release_new.wait(5) + native.start() + def __getattr__(self, name): + return getattr(native, name) + monkeypatch.setattr(loop, '_drain_deferred', pause_retired) + loop._loop = PausedReplacement() + def completion(): + observations.append((get_ident(), current_thread() is loop._thread)) + delivered.set() + try: + loop.call_soon(lambda: None) + assert retired.wait(5), 'runner did not publish its idle exit' + loop.call_soon(completion) + assert replacement.wait(5) + new_thread = loop._thread + assert new_thread is not old_threads[0] + release_old.set() + old_threads[0].join(5) + assert not old_threads[0].is_alive() + assert observations == [], 'retired runner consumed new-generation work' + release_new.set() + assert delivered.wait(5) + new_thread.join(5) + assert observations == [(new_thread.ident, True)] + assert not new_thread.is_alive() + finally: + release_old.set() + release_new.set() + loop._cleanup() + for thread in old_threads: + thread.join(5) + + +@pytest.mark.parametrize('arming_path', ['acquire', 'keepalive']) +def test_libev_keeper_arming_cannot_outlive_cleanup(monkeypatch, arming_path): + module = pytest.importorskip('cassandra.io.libevreactor') + loop = module.LibevLoop() + native = loop._management_keeper + arming, resume, stopped, armed = [Event() for _ in range(4)] + releases, errors = [], [] + class PausedKeeper: + active = False + def start(self, timeout): + arming.set() + assert resume.wait(5) + native.start(timeout) + self.active = True + armed.set() + def stop(self): + native.stop() + self.active = False + stopped.set() + keeper = PausedKeeper() + monkeypatch.setattr(loop, '_management_keeper', keeper) + def borrow(): + try: + releases.append(loop.borrow_management()) + except BaseException as exc: + errors.append(exc) + borrower = Thread(target=borrow) if arming_path == 'acquire' else None + stopper = None + try: + if borrower is not None: + borrower.start() + else: + loop._management_users.add(object()) + loop.call_soon(loop._keep_management_alive) + assert arming.wait(5) + runner = loop._thread + stopper = Thread(target=loop._cleanup) + stopper.start() + assert stopped.wait(5), 'cleanup did not stop the in-flight keeper' + assert loop._shutdown + resume.set() + assert armed.wait(5) + runner.join(5) + assert not runner.is_alive(), 'keeper was resurrected after cleanup' + assert not keeper.active + stopper.join(5) + assert not stopper.is_alive() + if borrower is not None: + borrower.join(5) + assert not borrower.is_alive() + assert errors == [] + assert len(releases) == 1 + finally: + resume.set() + keeper.stop() + loop.notify() + if borrower is not None: + borrower.join(5) + for release in releases: + release(lambda: None) + if stopper is not None: + stopper.join(5) + loop._cleanup() + + +def test_chaining_application_loop_closes_when_management_join_raises(monkeypatch, connection_class): + # Exercise the actual chaining test's failure cleanup, then supply our own + # fallback cleanup so a negative control cannot leak a non-daemon thread. + ReactorChannel(connection_class) + app_threads, app_loops = [], [] + make_loop, thread_class, join = asyncio.new_event_loop, Thread, ManagementRuntime.join + def new_loop(): + loop = make_loop() + app_loops.append(loop) + return loop + def new_thread(*args, **kwargs): + thread = thread_class(*args, **kwargs) + app_threads.append(thread) + return thread + def failed_join(manager, timeout): + join(manager, timeout) + raise RuntimeError('forced finalization failure') + monkeypatch.setattr(asyncio, 'new_event_loop', new_loop) + monkeypatch.setattr(__name__ + '.Thread', new_thread) + monkeypatch.setattr(ManagementRuntime, 'join', failed_join) + try: + with pytest.raises(RuntimeError, match='forced finalization failure'): + test_asyncio_chaining_progresses_while_listener_waits(connection_class, 'result', 'before') + assert len(app_threads) == len(app_loops) == 1 + assert not app_threads[0].is_alive() + assert app_loops[0].is_closed() + finally: + for loop, thread in zip(app_loops, app_threads): + if thread.is_alive(): + loop.call_soon_threadsafe(loop.stop) + thread.join(5) + if not thread.is_alive() and not loop.is_closed(): + loop.close() + + def test_reactor_startup_lock_is_reset_in_fork_child(connection_class): import os import select @@ -242,6 +394,7 @@ def close(self): monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() starting, fail, second_entered, accepted, delivered = [Event() for _ in range(5)] + admission_attempted = Event() original_thread = module.Thread calls = [] class FailedStart: @@ -266,11 +419,29 @@ def second(): loop.call_soon(delivered.set) accepted.set() first_thread, second_thread = Thread(target=first), Thread(target=second) + lock_name = '_lock_thread' if reactor == 'libev' else '_state_lock' + admission_lock = getattr(loop, lock_name) + blocked = [] + class ObservedLock: + def __enter__(self): + if get_ident() == second_thread.ident: + acquired = admission_lock.acquire(False) + blocked.append(not acquired) + admission_attempted.set() + if acquired: + return self + admission_lock.acquire() + return self + def __exit__(self, *args): + admission_lock.release() + setattr(loop, lock_name, ObservedLock()) try: first_thread.start() assert starting.wait(5) second_thread.start() assert second_entered.wait(5) + assert admission_attempted.wait(5) + assert blocked == [True], 'second admission did not block on provisional startup' assert not accepted.wait(0.05) fail.set() first_thread.join(5) @@ -324,6 +495,74 @@ def completion(): loop._cleanup() +def test_asyncore_ordinary_connection_recovers_without_reviving_management(monkeypatch): + import socket + module = pytest.importorskip('cassandra.io.asyncorereactor') + monkeypatch.setattr(module, '_dispatcher_map', {}) + entered, fail, finish_failure, recovered = [Event() for _ in range(4)] + class Dispatcher: + failed = False + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + if fail.is_set() and not self.failed: + entered.set() + assert finish_failure.wait(5) + self.failed = True + raise OSError('transient poll failure') + if self.failed: + recovered.set() + Event().wait(0.001) + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = module.AsyncoreLoop() + monkeypatch.setattr(module, '_global_loop', loop) + # This channel predates service loss and must never become usable again. + channel = None + ours, peer = socket.socketpair() + connection = None + try: + channel = ReactorChannel(module.AsyncoreConnection) + old_runner = loop._thread + fail.set() + assert entered.wait(5) + delivered = Event() + channel.dispatch(delivered.set) + finish_failure.set() + old_runner.join(5) + assert not old_runner.is_alive() + assert delivered.is_set() + with pytest.raises(RuntimeError): + channel.dispatch(lambda: None) + monkeypatch.setattr(module.AsyncoreConnection, '_connect_socket', + lambda self: setattr(self, '_socket', ours)) + monkeypatch.setattr(module.AsyncoreConnection, '_send_options_message', lambda self: None) + connection = module.AsyncoreConnection(connect_timeout=1) + assert recovered.wait(5) + assert loop._thread is not old_runner + assert loop._thread.is_alive() + assert module._dispatcher_map[ours.fileno()] is connection + assert connection.socket is ours + with pytest.raises(RuntimeError): + channel.dispatch(lambda: None) + with pytest.raises(RuntimeError): + ReactorChannel(module.AsyncoreConnection) + assert not loop._management_callbacks + finally: + fail.set() + finish_failure.set() + if channel is not None: + channel.close() + if connection is not None: + connection.close() + loop._cleanup() + ours.close() + peer.close() + + @pytest.mark.parametrize('reactor', ['asyncio', 'libev', 'asyncore']) @pytest.mark.parametrize('count', [1, 130]) def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, reactor, count): @@ -339,10 +578,12 @@ def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, react connection.initialize_reactor() dispatch, runner = connection._management_dispatch, connection._loop_thread def stop(): - connection._loop.call_soon_threadsafe(stopping.set) - connection._loop.call_soon_threadsafe(connection._loop.stop) + stopping.set() def cleanup(): + if runner.is_alive(): + connection._loop.call_soon_threadsafe(connection._loop.stop) runner.join(5) + assert not runner.is_alive() connection._loop.close() else: loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() @@ -357,9 +598,16 @@ def cleanup(): def hold(): entered.set() assert release.wait(5) + if reactor == 'asyncio': + # Stop in the current ready-queue turn, before any of the drain + # handles queued while this ordinary loop callback was held. + connection._loop.stop() stopper = None try: - dispatch(hold) + if reactor == 'asyncio': + connection._loop.call_soon_threadsafe(hold) + else: + dispatch(hold) assert entered.wait(5) def completion(): observations.append(get_ident()) From ed1c8c4ffb1f4333b5ba2d03897f584d95453c0d Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 03:19:48 -0400 Subject: [PATCH 04/13] tests: compare disposal thread objects after manager shutdown Manager shutdown can retire a pool owner before the executor creates its worker. PyPy can reuse that owner's native thread ID, so comparing saved integer IDs falsely reports owner-thread disposal. Retain actual Thread objects and verify worker execution context. Preserve the exactly-once disposal and released-credit assertions; an exactly-once owner-disposal negative control still fails the owner-exclusion assertion. --- tests/unit/test_management_runtime.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index 6853bd1989..0bcd94a651 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -1,7 +1,7 @@ from collections import deque from concurrent.futures import ( Future, ThreadPoolExecutor, as_completed, wait, TimeoutError) -from threading import Barrier, Event, Lock, Thread, get_ident +from threading import Barrier, Event, Lock, Thread, current_thread, get_ident import pytest @@ -502,6 +502,10 @@ def test_completed_executor_future_does_not_dispose_on_registering_owner( manager, dispatch = runtime resource = object() disposal_threads = [] + disposal_contexts = [] + def dispose(value): + disposal_threads.append(current_thread()) + disposal_contexts.append(execution_context()) with ThreadPoolExecutor(1) as executor: class ClosingExecutor: def submit(self, function): @@ -514,13 +518,15 @@ def submit(self, function): return physical future = manager.submit(lambda record: manager.work( record, ClosingExecutor(), lambda: resource, - dispose=lambda value: disposal_threads.append(get_ident()))) + dispose=dispose)) with pytest.raises(ManagementClosed): future.result(5) manager.join(5) assert len(disposal_threads) == 1 - assert disposal_threads[0] not in { - thread.ident for thread in manager._threads} + # Native thread IDs may be reused after close retires an owner. Keep the + # actual Thread objects to distinguish that owner from a later worker. + assert all(disposal_threads[0] is not thread for thread in manager._threads) + assert disposal_contexts == ['worker'] assert manager.counts()['ordinary'] == 0 From 3bfddb78027df1e2020dc2e6c331ea715e121587 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 05:21:44 -0400 Subject: [PATCH 05/13] tests: handle unavailable dependencies in standalone reactor tests Share optional reactor imports with the connection fixture so the driver's DependencyException skips only unavailable reactor variants. Keep other runtime and assertion failures visible, and cover both behaviors. --- tests/unit/test_management_reactors.py | 53 +++++++++++++++++--------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index bcb8539d44..1eb43abb9e 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -12,6 +12,31 @@ ManagementRuntime, ReactorChannel, execution_context) +def reactor_module(reactor): + try: + return importlib.import_module('cassandra.io.' + reactor + 'reactor') + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + + +@pytest.mark.parametrize('failure', [ImportError, DependencyException]) +def test_unavailable_reactor_import_is_skipped(monkeypatch, failure): + def unavailable(name): + raise failure('reactor dependency unavailable') + monkeypatch.setattr(importlib, 'import_module', unavailable) + with pytest.raises(pytest.skip.Exception, match='reactor dependency unavailable'): + reactor_module('libev') + + +@pytest.mark.parametrize('failure', [RuntimeError, AssertionError]) +def test_reactor_import_errors_are_not_hidden(monkeypatch, failure): + def broken(name): + raise failure('supported reactor is broken') + monkeypatch.setattr(importlib, 'import_module', broken) + with pytest.raises(failure, match='supported reactor is broken'): + reactor_module('asyncore') + + @pytest.fixture(autouse=True) def close_borrowed_channels(monkeypatch): channels = [] @@ -29,12 +54,7 @@ def tracked(self, *args, **kwargs): @pytest.fixture(params=['asyncio', 'libev', 'asyncore']) def connection_class(request): - try: - module = importlib.import_module('cassandra.io.' + { - 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', - 'asyncore': 'asyncorereactor'}[request.param]) - except (ImportError, DependencyException) as exc: - pytest.skip(str(exc)) + module = reactor_module(request.param) return getattr(module, {'asyncio': 'AsyncioConnection', 'libev': 'LibevConnection', 'asyncore': 'AsyncoreConnection'}[request.param]) @@ -155,10 +175,7 @@ async def consume(): def test_libev_dispatch_enqueued_during_idle_exit_is_not_lost(): - try: - module = importlib.import_module('cassandra.io.libevreactor') - except (ImportError, DependencyException) as exc: - pytest.skip(str(exc)) + module = reactor_module('libev') loop = module.LibevLoop() native = loop._loop returned, release, first, second = Event(), Event(), Event(), Event() @@ -188,7 +205,7 @@ def __getattr__(self, name): def test_libev_retired_runner_does_not_claim_replacement_work(monkeypatch): - module = pytest.importorskip('cassandra.io.libevreactor') + module = reactor_module('libev') loop = module.LibevLoop() native, drain = loop._loop, loop._drain_deferred retired, release_old, replacement, release_new, delivered = [Event() for _ in range(5)] @@ -238,7 +255,7 @@ def completion(): @pytest.mark.parametrize('arming_path', ['acquire', 'keepalive']) def test_libev_keeper_arming_cannot_outlive_cleanup(monkeypatch, arming_path): - module = pytest.importorskip('cassandra.io.libevreactor') + module = reactor_module('libev') loop = module.LibevLoop() native = loop._management_keeper arming, resume, stopped, armed = [Event() for _ in range(4)] @@ -379,7 +396,7 @@ def holder(): @pytest.mark.parametrize('reactor', ['libev', 'asyncore']) def test_startup_failure_does_not_acknowledge_concurrent_dispatch(monkeypatch, reactor): - module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + module = reactor_module(reactor) if reactor == 'asyncore': monkeypatch.setattr(module, '_dispatcher_map', {}) class Dispatcher: @@ -459,7 +476,7 @@ def __exit__(self, *args): def test_asyncore_dispatch_accepted_during_runner_failure_is_delivered(monkeypatch): - module = pytest.importorskip('cassandra.io.asyncorereactor') + module = reactor_module('asyncore') monkeypatch.setattr(module, '_dispatcher_map', {}) entered, fail, delivered = Event(), Event(), Event() class Dispatcher: @@ -497,7 +514,7 @@ def completion(): def test_asyncore_ordinary_connection_recovers_without_reviving_management(monkeypatch): import socket - module = pytest.importorskip('cassandra.io.asyncorereactor') + module = reactor_module('asyncore') monkeypatch.setattr(module, '_dispatcher_map', {}) entered, fail, finish_failure, recovered = [Event() for _ in range(4)] class Dispatcher: @@ -566,9 +583,7 @@ def close(self): @pytest.mark.parametrize('reactor', ['asyncio', 'libev', 'asyncore']) @pytest.mark.parametrize('count', [1, 130]) def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, reactor, count): - module = pytest.importorskip('cassandra.io.' + { - 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', - 'asyncore': 'asyncorereactor'}[reactor]) + module = reactor_module(reactor) entered, release, delivered, stopping = [Event() for _ in range(4)] if reactor == 'asyncore': monkeypatch.setattr(module, '_dispatcher_map', {}) @@ -655,7 +670,7 @@ def test_fork_reinitialization_closes_inherited_descriptors(monkeypatch, reactor from types import SimpleNamespace if not hasattr(os, 'fork'): pytest.skip('fork unavailable') - module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + module = reactor_module(reactor) connection = getattr(module, {'libev': 'LibevConnection', 'asyncore': 'AsyncoreConnection'}[reactor]) connection.initialize_reactor() From 6e114c1711ad7cd3c4ea4814961661d10b2dbe9a Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 05:21:44 -0400 Subject: [PATCH 06/13] libev: serialize management release admission with cleanup Hold the existing startup lock across the release shutdown check and callback admission. Cleanup can then either drain an accepted release on the runner or let the caller detach after shutdown has begun. Exercise both orders with the native loop and actual cleanup, observing the admission lock rather than relying on a sleep to establish the race. --- cassandra/io/libevreactor.py | 14 +++-- tests/unit/test_management_reactors.py | 83 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index a4ee93bb63..a9ca5da42b 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -285,13 +285,15 @@ def detach(): completed() if current_thread() is self._thread: detach() - elif self._shutdown or self._thread is None or not self._thread.is_alive(): - # Shutdown closes admission before the runner finishes draining. - # Release remains idempotent while that runner unwinds. - with self._lock: - detach() else: - self.call_soon(detach) + # Cleanup closes admission under this same lock. Either queue + # the release before shutdown or detach after it has begun. + with self._lock_thread: + if self._shutdown or self._thread is None or not self._thread.is_alive(): + with self._lock: + detach() + else: + self.call_soon(detach) if current_thread() is self._thread: acquire() diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 1eb43abb9e..a0286f6fe1 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -318,6 +318,89 @@ def borrow(): loop._cleanup() +@pytest.mark.parametrize('cleanup_order', ['during-release', 'before-release']) +def test_libev_borrow_release_completes_during_cleanup(monkeypatch, cleanup_order): + module = reactor_module('libev') + loop = module.LibevLoop() + release_borrow = loop.borrow_management() + entered, finish, admission_attempted, stopped, completed = [Event() for _ in range(5)] + errors, observations, blocked = [], [], [] + admission_lock, keeper = loop._lock_thread, loop._management_keeper + dispatch = loop.call_soon + def hold(): + entered.set() + assert finish.wait(5) + def cleanup(): + try: + loop._cleanup() + except BaseException as exc: + errors.append(exc) + stopper = Thread(target=cleanup) + class ObservedLock: + def __enter__(self): + if current_thread() is stopper: + acquired = admission_lock.acquire(False) + blocked.append(not acquired) + admission_attempted.set() + if acquired: + return self + admission_lock.acquire() + return self + def __exit__(self, *args): + admission_lock.release() + class ObservedKeeper: + def stop(self): + keeper.stop() + stopped.set() + def __getattr__(self, name): + return getattr(keeper, name) + def racing_dispatch(callback): + stopper.start() + assert admission_attempted.wait(5) + if not blocked[0]: + # The unprotected release allows cleanup to commit shutdown here. + assert stopped.wait(5) + dispatch(callback) + def detached(): + observations.append(current_thread()) + completed.set() + try: + dispatch(hold) + assert entered.wait(5) + runner = loop._thread + monkeypatch.setattr(loop, '_lock_thread', ObservedLock()) + monkeypatch.setattr(loop, '_management_keeper', ObservedKeeper()) + if cleanup_order == 'during-release': + monkeypatch.setattr(loop, 'call_soon', racing_dispatch) + else: + stopper.start() + assert stopped.wait(5) + assert loop._shutdown + release_borrow(detached) + assert stopped.wait(5) + finish.set() + assert completed.wait(5) + stopper.join(5) + runner.join(5) + assert not stopper.is_alive() + assert not runner.is_alive() + assert errors == [] + assert not loop._management_users + assert not loop._management_callbacks + assert len(observations) == 1 + if cleanup_order == 'during-release': + assert blocked[0], 'cleanup passed the release admission boundary' + assert observations == [runner] + else: + assert observations == [current_thread()] + finally: + finish.set() + if stopper.ident is not None: + stopper.join(5) + loop._cleanup() + release_borrow(lambda: None) + + def test_chaining_application_loop_closes_when_management_join_raises(monkeypatch, connection_class): # Exercise the actual chaining test's failure cleanup, then supply our own # fallback cleanup so a negative control cannot leak a non-daemon thread. From 85876a36e311dbb479353543b84dce8bb8b1da99 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 08:32:54 -0400 Subject: [PATCH 07/13] libev: release native callbacks from unpublished reactor candidates Failed startup leaves an unpublished loop outside atexit ownership. Its C watchers retain bound callbacks that Python's cyclic GC cannot traverse, so stopping them alone leaks each failed candidate. Release native watcher references after cleanup and actual runner exit. If the bounded join expires, let the retiring runner release them after accepted callbacks drain. Test repeated thread-start failures and a native runner held beyond cleanup's join. --- cassandra/io/libevreactor.py | 31 +++++++++-- tests/unit/test_management_reactors.py | 76 ++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index a9ca5da42b..12983dbd8b 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -86,6 +86,8 @@ def __init__(self): self._management_callbacks = deque() self._management_users = set() self._cleanup_started = False + self._dispose_on_exit = False + self._runner_exited = True self._preparer = libev.Prepare(self._loop, self._loop_will_run) # prevent _preparer from keeping the loop from returning @@ -105,6 +107,7 @@ def maybe_start(self): if not self._started: log.debug("Starting libev event loop") self._started = True + self._runner_exited = False self._thread = Thread(target=self._run_loop, name="event_loop") self._thread.daemon = True try: @@ -157,6 +160,28 @@ def _run_loop(self): if shutdown: while self._management_callbacks: self._dispatch_management() + with self._lock: + self._runner_exited = True + if self._dispose_on_exit: + self._release_loop() + + def _cleanup_unpublished(self): + # An unpublished candidate has no future owner. If bounded cleanup + # cannot join its runner, that runner releases the native callbacks + # after returning from libev and draining accepted work. + self._cleanup() + with self._lock: + self._dispose_on_exit = True + if (self._thread is None or not self._thread.is_alive() + or self._runner_exited): + self._release_loop() + + def _release_loop(self): + # Native watchers retain bound callbacks without GC traversal support. + # Stop/join must precede this outside the single surviving fork thread. + self._preparer = self._loop_timer = self._management_keeper = None + self._notifier = None + self._loop = None def _cleanup(self): with self._lock_thread: @@ -218,9 +243,7 @@ def _cleanup_after_fork(self): self._management_users.clear() # Break watcher/callback cycles and release the native loop, whose # destructor closes its internal wakeup descriptors in the child. - self._preparer = self._loop_timer = self._management_keeper = None - self._notifier = None - self._loop = None + self._release_loop() def add_timer(self, timer): self._timers.add_timer(timer) @@ -485,7 +508,7 @@ def initialize_reactor(cls): if not ready.wait(5): raise RuntimeError("Libev reactor startup did not complete") except BaseException: - candidate._cleanup() + candidate._cleanup_unpublished() raise _global_loop = candidate diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index a0286f6fe1..b3f9c5cbef 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -477,6 +477,82 @@ def holder(): os.waitpid(pid, 0) +def test_failed_libev_startup_releases_unpublished_native_callbacks(monkeypatch): + import gc + import weakref + module = reactor_module('libev') + original_factory = module.LibevLoop + candidates = [] + def candidate(): + loop = original_factory() + candidates.append(weakref.ref(loop)) + return loop + class FailedStart: + def start(self): + raise RuntimeError('native runner was not started') + def is_alive(self): + return False + monkeypatch.setattr(module, '_global_loop', None) + monkeypatch.setattr(module, 'LibevLoop', candidate) + monkeypatch.setattr(module, 'Thread', lambda **kwargs: FailedStart()) + for _ in range(3): + with pytest.raises(RuntimeError, match='native runner was not started'): + module.LibevConnection.initialize_reactor() + assert module._global_loop is None + gc.collect() + assert len(candidates) == 3 + assert all(candidate() is None for candidate in candidates) + + +def test_unpublished_libev_disposal_waits_for_actual_native_exit(monkeypatch): + module = reactor_module('libev') + original_factory = module.LibevLoop + entered, release = Event(), Event() + candidates = [] + class PausedNativeLoop: + def __init__(self, native): + self.native = native + def __getattr__(self, name): + return getattr(self.native, name) + def start(self): + entered.set() + assert release.wait(15) + self.native.start() + def candidate(): + loop = original_factory() + loop._loop = PausedNativeLoop(loop._loop) + candidates.append(loop) + return loop + class ReadyTimeout(Event): + def wait(self, timeout=None): + assert entered.wait(5) + return False + monkeypatch.setattr(module, '_global_loop', None) + monkeypatch.setattr(module, 'LibevLoop', candidate) + monkeypatch.setattr(module, 'Event', ReadyTimeout) + try: + with pytest.raises(RuntimeError, match='startup did not complete'): + module.LibevConnection.initialize_reactor() + loop = candidates[0] + assert loop._thread.is_alive() + assert loop._loop is not None + assert loop._preparer is not None + # A repeated request must also retain native state until actual exit. + loop._cleanup_unpublished() + assert loop._loop is not None + finally: + release.set() + for loop in candidates: + loop._thread.join(5) + assert not loop._thread.is_alive() + assert module._global_loop is None + assert loop._loop is None + assert loop._preparer is None + assert loop._loop_timer is None + assert loop._management_keeper is None + assert loop._notifier is None + + @pytest.mark.parametrize('reactor', ['libev', 'asyncore']) def test_startup_failure_does_not_acknowledge_concurrent_dispatch(monkeypatch, reactor): module = reactor_module(reactor) From 353cf910976fd4929b045d91c0ddb8348bb837b1 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 08:32:54 -0400 Subject: [PATCH 08/13] tests: keep watcher cleanup off the shared libev singleton Restoring only shutdown and the preparer leaves the shared singleton's cleanup latch set, so its later teardown silently returns. Exercise the same watcher-stop assertions on a private reactor and release its native callbacks afterward. Verify the shared singleton and lifecycle flags stay unchanged. --- tests/unit/io/test_libevreactor.py | 39 ++++++++++++------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index 7d30acd616..f101802429 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -96,37 +96,28 @@ def test_watchers_are_finished(self): @test_category connection """ - from cassandra.io.libevreactor import _global_loop - reactor_needs_restore = False + from cassandra.io import libevreactor + shared_loop = libevreactor._global_loop + shared_state = shared_loop._shutdown, shared_loop._cleanup_started + private_loop = LibevLoop() try: - with patch.object(_global_loop, "_thread"),\ - patch.object(_global_loop, "notify"): - + with patch.object(libevreactor, "_global_loop", private_loop),\ + patch.object(private_loop, "_thread"),\ + patch.object(private_loop, "notify"): self.make_connection() - # We have to make a copy because the connections shouldn't - # be alive when we verify them - live_connections = set(_global_loop._live_conns) - - # This simulates the process ending without cluster.shutdown() - # being called, then with atexit _cleanup for libevreactor would - # be called - reactor_needs_restore = True - libev__cleanup(_global_loop) + # Keep the watchers alive while simulating atexit cleanup. + live_connections = set(private_loop._live_conns) + libev__cleanup(private_loop) for conn in live_connections: assert conn._write_watcher.stop.mock_calls assert conn._read_watcher.stop.mock_calls - finally: - if reactor_needs_restore: - _global_loop._shutdown = False - # _cleanup stopped the prepare watcher; restart it so the shared - # singleton loop is left in a working state for subsequent tests - # (otherwise timers would never be scheduled and tests would hang). - _global_loop._preparer.start() - # _cleanup took the preparer's loop reference back; drop it - # again as in LibevLoop.__init__, or the loop never returns - _global_loop._loop.unref() + private_loop._cleanup_unpublished() + + self.assertIs(libevreactor._global_loop, shared_loop) + self.assertEqual( + (shared_loop._shutdown, shared_loop._cleanup_started), shared_state) class _InstrumentedLock(object): From 1a2abdf2a90e12f531cf447244af155f6ef13b2e Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 08:32:54 -0400 Subject: [PATCH 09/13] tests: preserve parent reactor ownership in the fork fixture Initialization readiness does not stop the parent libev runner. Its next prepare callback can observe the inherited-descriptor fixture while the fork test is running, so that connection must support normal prepare reads and live-set updates must preserve existing snapshots. Track the descriptor through a context manager with copy-on-write updates under the state lock. Gate an actual parent prepare callback around the attachment and retain the child-close and parent-socket assertions. --- tests/unit/test_management_reactors.py | 149 ++++++++++++++++++------- 1 file changed, 106 insertions(+), 43 deletions(-) diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index b3f9c5cbef..803ffa40d6 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -2,6 +2,8 @@ import asyncio import importlib +from collections import deque +from contextlib import contextmanager from concurrent.futures import CancelledError, Future from threading import Event, Thread, current_thread, get_ident @@ -820,13 +822,83 @@ def completion(): cleanup() +@contextmanager +def inherited_descriptor(module, reactor, sock): + from types import SimpleNamespace + inherited = module._global_loop + if reactor == 'libev': + class Connection: + _socket = sock + _read_watcher = _write_watcher = None + _write_watcher_is_active = False + deque = deque() + conn = Connection() + with inherited._lock: + inherited._live_conns = inherited._live_conns | {conn} + try: + yield conn + finally: + with inherited._lock: + inherited._live_conns = inherited._live_conns - {conn} + else: + descriptor = sock.fileno() + module._dispatcher_map[descriptor] = SimpleNamespace(socket=sock) + try: + yield + finally: + module._dispatcher_map.pop(descriptor, None) + + +def test_inherited_libev_descriptor_is_safe_during_parent_prepare(monkeypatch): + import socket + module = reactor_module('libev') + entered, release, prepared = Event(), Event(), Event() + errors = [] + original_prepare = module.LibevLoop._loop_will_run + def observed_prepare(loop, watcher): + try: + original_prepare(loop, watcher) + except BaseException as exc: + errors.append(exc) + raise + finally: + prepared.set() + monkeypatch.setattr(module.LibevLoop, '_loop_will_run', observed_prepare) + loop = module.LibevLoop() + monkeypatch.setattr(module, '_global_loop', loop) + ours, peer = socket.socketpair() + def hold(): + entered.set() + assert release.wait(5) + try: + loop.call_soon(hold) + assert entered.wait(5) + previous = loop._live_conns + with inherited_descriptor(module, 'libev', ours) as conn: + installed = loop._live_conns + assert conn not in previous + assert conn in installed + release.set() + assert prepared.wait(5) + assert not errors + assert conn in installed + assert conn not in loop._live_conns + finally: + release.set() + try: + loop._cleanup_unpublished() + assert loop._thread is None or not loop._thread.is_alive() + finally: + ours.close() + peer.close() + + @pytest.mark.parametrize('reactor', ['libev', 'asyncore']) def test_fork_reinitialization_closes_inherited_descriptors(monkeypatch, reactor): import os import select import signal import socket - from types import SimpleNamespace if not hasattr(os, 'fork'): pytest.skip('fork unavailable') module = reactor_module(reactor) @@ -835,49 +907,40 @@ def test_fork_reinitialization_closes_inherited_descriptors(monkeypatch, reactor connection.initialize_reactor() inherited = module._global_loop ours, peer = socket.socketpair() - if reactor == 'libev': - class Connection: - _socket = ours - _read_watcher = _write_watcher = None - conn = Connection() - inherited._live_conns.add(conn) - else: - module._dispatcher_map[ours.fileno()] = SimpleNamespace(socket=ours) - original_factory = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor]) - def checked_factory(): - assert ours.fileno() == -1 - if reactor == 'libev': - assert inherited._loop is None - assert inherited._notifier is None - else: - assert inherited._loop_dispatcher.write_fd is None - return original_factory() - monkeypatch.setattr(module, original_factory.__name__, checked_factory) - read_fd, write_fd = os.pipe() - pid = os.fork() - if pid == 0: - try: - os.close(read_fd) - connection.initialize_reactor() - os.write(write_fd, b'closed') - finally: - os._exit(0) - os.close(write_fd) try: - ready, _, _ = select.select([read_fd], [], [], 5) - if not ready: - os.kill(pid, signal.SIGKILL) - assert ready - assert os.read(read_fd, 6) == b'closed' - # The child closed only its descriptor copies. - ours.send(b'x') - assert peer.recv(1) == b'x' + with inherited_descriptor(module, reactor, ours): + original_factory = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor]) + def checked_factory(): + assert ours.fileno() == -1 + if reactor == 'libev': + assert inherited._loop is None + assert inherited._notifier is None + else: + assert inherited._loop_dispatcher.write_fd is None + return original_factory() + monkeypatch.setattr(module, original_factory.__name__, checked_factory) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + connection.initialize_reactor() + os.write(write_fd, b'closed') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready + assert os.read(read_fd, 6) == b'closed' + # The child closed only its descriptor copies. + ours.send(b'x') + assert peer.recv(1) == b'x' + finally: + os.close(read_fd) + os.waitpid(pid, 0) finally: - if reactor == 'libev': - inherited._live_conns.discard(conn) - else: - module._dispatcher_map.pop(ours.fileno(), None) ours.close() peer.close() - os.close(read_fd) - os.waitpid(pid, 0) From 965a0ddfbb1e3eef79ea543d2bc3ec46cf687822 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 10:55:27 -0400 Subject: [PATCH 10/13] libev: reject inline management borrows after shutdown An accepted callback can run on the native reactor while cleanup drains its queue. Inline channel borrowing bypasses call_soon's shutdown check, so it can add a token after the keeper has stopped and return a channel that cannot accept work. Check shutdown under the state lock before inline acquisition. Exercise the actual native runner and committed keeper shutdown, retaining queue drain and channel-release assertions. --- cassandra/io/libevreactor.py | 5 +- tests/unit/test_management_reactors.py | 69 ++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 12983dbd8b..ce7ddfb816 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -319,7 +319,10 @@ def detach(): self.call_soon(detach) if current_thread() is self._thread: - acquire() + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + acquire() else: self.call_soon(acquire) if not ready.wait(5): diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 803ffa40d6..58c4c0a077 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -479,6 +479,75 @@ def holder(): os.waitpid(pid, 0) +def test_libev_shutdown_drain_rejects_inline_channel_borrow(monkeypatch): + module = reactor_module('libev') + loop = module.LibevLoop() + monkeypatch.setattr(module, '_global_loop', loop) + entered, release, stopped, finished = [Event() for _ in range(4)] + errors, rejected, observed, channels = [], [], [], [] + native_keeper = loop._management_keeper + class ObservedKeeper: + def stop(self): + native_keeper.stop() + stopped.set() + def __getattr__(self, name): + return getattr(native_keeper, name) + def hold(): + entered.set() + assert release.wait(5) + def attempt(): + try: + channels.append(ReactorChannel(module.LibevConnection)) + except RuntimeError as exc: + rejected.append(str(exc)) + finally: + observed.append(current_thread()) + finished.set() + def cleanup(): + try: + loop._cleanup() + except BaseException as exc: + errors.append(exc) + stopper = Thread(target=cleanup) + try: + channels.append(ReactorChannel(module.LibevConnection)) + with monkeypatch.context() as keeper_patch: + keeper_patch.setattr(loop, '_management_keeper', ObservedKeeper()) + before = set(loop._management_users) + loop.call_soon(hold) + assert entered.wait(5) + runner = loop._thread + loop.call_soon(attempt) + stopper.start() + assert stopped.wait(5) + with loop._lock: + assert loop._shutdown + release.set() + assert finished.wait(5) + stopper.join(5) + runner.join(5) + assert not stopper.is_alive() + assert not runner.is_alive() + assert errors == [] + assert observed == [runner] + assert rejected == ['Libev reactor is shutting down'], { + 'channels': len(channels), 'tokens': len(loop._management_users), + 'previous_tokens': len(before)} + assert set(loop._management_users) == before + assert not loop._management_callbacks + finally: + release.set() + if stopper.ident is not None: + stopper.join(5) + try: + for channel in channels: + detached = Event() + channel.close(detached.set) + assert detached.wait(5) + finally: + loop._cleanup_unpublished() + + def test_failed_libev_startup_releases_unpublished_native_callbacks(monkeypatch): import gc import weakref From be3027a39003ea32ce78efb02a7ed0deb30177eb Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 10:55:27 -0400 Subject: [PATCH 11/13] tests: make inherited asyncore descriptors safe to poll The fork fixture publishes its inherited socket in the live dispatcher map. A real parent poll can inspect it before fork, so the stub must support readable and writable probes without failing the management service. Keep both probes inert and exercise all three real dispatcher paths in bounded subprocesses. Preserve fork descriptor ownership assertions and verify accepted and subsequent callbacks drain without service failure. --- tests/unit/test_management_reactors.py | 73 +++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 58c4c0a077..7912744aae 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -911,13 +911,84 @@ class Connection: inherited._live_conns = inherited._live_conns - {conn} else: descriptor = sock.fileno() - module._dispatcher_map[descriptor] = SimpleNamespace(socket=sock) + module._dispatcher_map[descriptor] = SimpleNamespace( + socket=sock, readable=lambda: False, writable=lambda: False) try: yield finally: module._dispatcher_map.pop(descriptor, None) +@pytest.mark.parametrize('dispatcher_name', [ + '_AsyncorePipeDispatcher', '_AsyncoreUDPDispatcher', '_BusyWaitDispatcher']) +def test_inherited_asyncore_descriptor_survives_actual_poll(dispatcher_name): + import subprocess + import sys + reactor_module('asyncore') + # A running shared runner also consults the process-global map. Keep the + # real polling/cleanup regression in its own process, including failures. + script = ( + 'import sys, pytest; ' + 'from tests.unit.test_management_reactors import ' + '_assert_inherited_asyncore_descriptor_survives_poll as check; ' + 'patch = pytest.MonkeyPatch(); ' + '\ntry: check(patch, sys.argv[1])' + '\nfinally: patch.undo()') + result = subprocess.run( + [sys.executable, '-c', script, dispatcher_name], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=30) + assert result.returncode == 0, result.stdout + + +def _assert_inherited_asyncore_descriptor_survives_poll(monkeypatch, dispatcher_name): + import socket + module = reactor_module('asyncore') + monkeypatch.setattr(module, '_dispatcher_map', {}) + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', + getattr(module, dispatcher_name)) + loop = module.AsyncoreLoop() + monkeypatch.setattr(module, '_global_loop', loop) + entered, release, polled, initial, delivered = [Event() for _ in range(5)] + actual_poll = loop._loop_dispatcher.loop + errors, snapshots = [], [] + def observed_poll(timeout): + entered.set() + assert release.wait(5) + snapshots.append(set(module._dispatcher_map)) + try: + actual_poll(timeout) + except BaseException as exc: + errors.append(exc) + raise + finally: + polled.set() + monkeypatch.setattr(loop._loop_dispatcher, 'loop', observed_poll) + ours, peer = socket.socketpair() + descriptor = ours.fileno() + try: + loop.call_soon(initial.set) + assert entered.wait(5) + with inherited_descriptor(module, 'asyncore', ours): + assert descriptor in module._dispatcher_map + release.set() + assert polled.wait(5) + assert descriptor in snapshots[0] + assert initial.wait(5) + assert errors == [], { + 'errors': errors, 'management_failed': loop._management_failed} + assert not loop._management_failed + loop.call_soon(delivered.set) + assert delivered.wait(5) + finally: + release.set() + try: + loop._cleanup() + assert loop._thread is None or not loop._thread.is_alive() + finally: + ours.close() + peer.close() + + def test_inherited_libev_descriptor_is_safe_during_parent_prepare(monkeypatch): import socket module = reactor_module('libev') From dce4ffbec71878431ebb2e98b5decf27b78f2fdf Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 10:55:28 -0400 Subject: [PATCH 12/13] tests: keep disposal executor open until physical completion Manager close can settle the logical future before the owner submits its physical work. Exiting the executor context at that point rejects the submission, so the disposal regression may never produce its resource. Wait for the real physical result before closing the executor, retaining exactly-once disposal, owner-thread exclusion, worker context and credit assertions. This ensures the test exercises completed-result ownership. --- tests/unit/test_management_runtime.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index 0bcd94a651..d4b7a89fe8 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -501,6 +501,7 @@ def test_completed_executor_future_does_not_dispose_on_registering_owner( runtime): manager, dispatch = runtime resource = object() + physical_completed = Event() disposal_threads = [] disposal_contexts = [] def dispose(value): @@ -515,12 +516,17 @@ def submit(self, function): # invoke that callback on this owner. Cleanup must already # have happened on the physical worker, exactly once. assert physical.result(5) is resource + physical_completed.set() return physical future = manager.submit(lambda record: manager.work( record, ClosingExecutor(), lambda: resource, dispose=dispose)) with pytest.raises(ManagementClosed): future.result(5) + # close() settles the logical operation before the owner necessarily + # reaches executor.submit(). Keep the real executor open until the + # physical result required by this regression has actually arrived. + assert physical_completed.wait(5) manager.join(5) assert len(disposal_threads) == 1 # Native thread IDs may be reused after close retires an owner. Keep the From 48ac0309091a040c89090468a27bafa4bd6af96d Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 11:24:56 -0400 Subject: [PATCH 13/13] tests: give the UDP poll fixture an ephemeral endpoint The real-poll regression selects the experimental UDP dispatcher, whose default localhost port can already belong to another process. A collision fails construction before the fixture exercises polling or callback drain. Bind the test-local subclass to port zero and use the actual bound address for notifications before validation runs. Assert the requested dispatcher was retained so busy-wait fallback cannot hide a broken test setup. Keep all existing polling, service, drain, descriptor and cleanup assertions. --- tests/unit/test_management_reactors.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 7912744aae..7eb293e7b2 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -944,8 +944,19 @@ def _assert_inherited_asyncore_descriptor_survives_poll(monkeypatch, dispatcher_ import socket module = reactor_module('asyncore') monkeypatch.setattr(module, '_dispatcher_map', {}) + dispatcher_class = getattr(module, dispatcher_name) + if dispatcher_name == '_AsyncoreUDPDispatcher': + class LocalUDPDispatcher(dispatcher_class): + bind_address = ('127.0.0.1', 0) + + def __init__(self): + super().__init__() + # validate() wakes this socket after construction, so keep + # its actual OS-assigned endpoint as the notify destination. + self.bind_address = self._socket.getsockname() + dispatcher_class = LocalUDPDispatcher monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', - getattr(module, dispatcher_name)) + dispatcher_class) loop = module.AsyncoreLoop() monkeypatch.setattr(module, '_global_loop', loop) entered, release, polled, initial, delivered = [Event() for _ in range(5)] @@ -966,6 +977,7 @@ def observed_poll(timeout): ours, peer = socket.socketpair() descriptor = ours.fileno() try: + assert isinstance(loop._loop_dispatcher, dispatcher_class) loop.call_soon(initial.set) assert entered.wait(5) with inherited_descriptor(module, 'asyncore', ours):