From 9076af09a8fd552acb18b4beffc900d7f2d95761 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Nov 2026 09:16:07 -0400 Subject: [PATCH 1/5] reactors: dispatch management completion on shared I/O loops --- cassandra/io/asyncioreactor.py | 46 +++- cassandra/io/asyncorereactor.py | 112 +++++++--- cassandra/io/libevreactor.py | 142 ++++++++++-- tests/unit/io/test_asyncioreactor.py | 8 +- tests/unit/io/test_libevreactor_shutdown.py | 12 ++ tests/unit/test_management_reactors.py | 225 ++++++++++++++++++++ 6 files changed, 490 insertions(+), 55 deletions(-) create mode 100644 tests/unit/test_management_reactors.py diff --git a/cassandra/io/asyncioreactor.py b/cassandra/io/asyncioreactor.py index 20fe79b851..1688a160f2 100644 --- a/cassandra/io/asyncioreactor.py +++ b/cassandra/io/asyncioreactor.py @@ -7,7 +7,7 @@ import os import socket import ssl -from threading import Lock, Thread +from threading import Event, Lock, Thread, current_thread log = logging.getLogger(__name__) @@ -258,13 +258,34 @@ def initialize_reactor(cls): cls._pid = os.getpid() if cls._loop is None: assert cls._loop_thread is None - cls._loop = asyncio.new_event_loop() - # daemonize so the loop will be shut down on interpreter - # shutdown - cls._loop_thread = Thread( - target=cls._loop.run_forever, daemon=True, name="asyncio_thread" - ) - cls._loop_thread.start() + loop = asyncio.new_event_loop() + thread = Thread(target=loop.run_forever, daemon=True, + name="asyncio_thread") + ready = Event() + try: + loop.call_soon(ready.set) + thread.start() + if not ready.wait(5): + raise RuntimeError("Asyncio reactor startup did not complete") + except BaseException: + if thread.is_alive(): + loop.call_soon_threadsafe(loop.stop) + thread.join(5) + if not thread.is_alive(): + loop.close() + raise + cls._loop, cls._loop_thread = loop, thread + + @classmethod + def _management_dispatch(cls, callback): + loop = cls._loop + if loop is None or loop.is_closed() or not cls._loop_thread.is_alive(): + raise RuntimeError("Asyncio reactor is unavailable") + loop.call_soon_threadsafe(callback) + + @classmethod + def _management_in_context(cls): + return current_thread() is cls._loop_thread @classmethod def create_timer(cls, timeout, callback): @@ -381,3 +402,12 @@ async def handle_read(self): log.debug("Connection %s closed by server", self) self.close() return + + +def _reset_management_startup_lock_after_fork(): + # A parent thread may own this lock when another thread forks. + AsyncioConnection._lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 4d19bb9849..c91e1a2ebf 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -18,7 +18,7 @@ import os import socket import sys -from threading import Lock, Thread, Event +from threading import Lock, Thread, Event, current_thread import time import sys import ssl @@ -210,6 +210,9 @@ class AsyncoreLoop(object): def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() + self._state_lock = Lock() + self._management_lock = Lock() + self._management_callbacks = deque() self._started = False self._shutdown = False @@ -228,21 +231,38 @@ def __init__(self): self._loop_dispatcher = dispatcher def maybe_start(self): - should_start = False - did_acquire = False - try: - did_acquire = self._loop_lock.acquire(False) - if did_acquire and not self._started: - self._started = True - should_start = True - finally: - if did_acquire: - self._loop_lock.release() - - if should_start: - self._thread = Thread(target=self._run_loop, name="asyncore_cassandra_driver_event_loop") - self._thread.daemon = True - self._thread.start() + with self._state_lock: + if self._shutdown or self._started: + return + self._started = True + self._thread = Thread(target=self._run_loop, + name="asyncore_cassandra_driver_event_loop", daemon=True) + try: + self._thread.start() + except BaseException: + self._started = False + self._thread = None + raise + + def call_soon(self, callback): + with self._management_lock: + if self._shutdown: + raise RuntimeError("Asyncore reactor is shutting down") + self._management_callbacks.append(callback) + self.maybe_start() + self.wake_loop() + + def _dispatch_management(self): + for _ in range(64): + with self._management_lock: + if not self._management_callbacks: + return + callback = self._management_callbacks.popleft() + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + self.wake_loop() def wake_loop(self): self._loop_dispatcher.notify_loop() @@ -254,10 +274,12 @@ def _run_loop(self): try: self._loop_dispatcher.loop(self.timer_resolution) self._timers.service_timeouts() + self._dispatch_management() except Exception as exc: self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) break - self._started = False + with self._state_lock: + self._started = False self._maybe_log_debug("Asyncore event loop ended") @@ -281,13 +303,14 @@ def add_timer(self, timer): def _cleanup(self): global _dispatcher_map - self._shutdown = True - if not self._thread: - return + with self._state_lock: + self._shutdown = True + self.wake_loop() log.debug("Waiting for event loop thread to join...") - self._thread.join(timeout=1.0) - if self._thread.is_alive(): + if self._thread is not None and self._thread is not current_thread(): + self._thread.join(timeout=1.0) + if self._thread is not None and self._thread.is_alive(): log.warning( "Event loop thread could not be joined, so shutdown may not be clean. " "Please call Cluster.shutdown() to avoid this.") @@ -306,6 +329,7 @@ def _cleanup(self): _global_loop = None +_initialization_lock = Lock() atexit.register(partial(_cleanup, _global_loop)) @@ -320,15 +344,32 @@ class AsyncoreConnection(Connection, asyncore.dispatcher): @classmethod def initialize_reactor(cls): - global _global_loop - if not _global_loop: - _global_loop = AsyncoreLoop() - else: - current_pid = os.getpid() - if _global_loop._pid != current_pid: - log.debug("Detected fork, clearing and reinitializing reactor state") - cls.handle_fork() - _global_loop = AsyncoreLoop() + global _global_loop, _dispatcher_map + with _initialization_lock: + if _global_loop is not None and _global_loop._pid != os.getpid(): + _dispatcher_map = {} + _global_loop = None + if _global_loop is None: + candidate = AsyncoreLoop() + ready = Event() + try: + candidate.call_soon(ready.set) + if not ready.wait(5): + raise RuntimeError("Asyncore reactor startup did not complete") + except BaseException: + candidate._cleanup() + raise + _global_loop = candidate + + @classmethod + def _management_dispatch(cls, callback): + if _global_loop is None: + raise RuntimeError("Asyncore reactor is unavailable") + _global_loop.call_soon(callback) + + @classmethod + def _management_in_context(cls): + return _global_loop is not None and current_thread() is _global_loop._thread @classmethod def handle_fork(cls): @@ -476,3 +517,12 @@ def writable(self): def readable(self): return self._readable or ((self.is_control_connection or self._continuous_paging_sessions) and not (self.is_defunct or self.is_closed)) + + +def _reset_management_startup_lock_after_fork(): + global _initialization_lock + _initialization_lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index f963d283ac..3f5dc0c371 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -17,7 +17,7 @@ import os import socket import ssl -from threading import Lock, RLock, Thread +from threading import Event, Lock, RLock, Thread, current_thread import time from cassandra import DependencyException @@ -83,6 +83,8 @@ def __init__(self): self._closed_conns = set() # connections closed while their thread held _lock; drained by the loop self._deferred_destroyed = [] + self._management_callbacks = deque() + self._management_users = set() self._preparer = libev.Prepare(self._loop, self._loop_will_run) # prevent _preparer from keeping the loop from returning @@ -91,6 +93,7 @@ def __init__(self): self._timers = TimerManager() self._loop_timer = libev.Timer(self._loop, self._on_loop_timer) + self._management_keeper = libev.Timer(self._loop, self._keep_management_alive) def maybe_start(self): should_start = False @@ -105,7 +108,13 @@ def maybe_start(self): if not self._shutdown: self._thread = Thread(target=self._run_loop, name="event_loop") self._thread.daemon = True - self._thread.start() + try: + self._thread.start() + except BaseException: + with self._lock: + self._started = False + self._thread = None + raise self._notifier.send() @@ -124,7 +133,7 @@ def _run_loop(self): # connection_created(), then observes _started == False and # starts a fresh thread). There is no interleaving in which # the new connection is invisible to both. - if not self._shutdown and self._live_conns: + if not self._shutdown and (self._live_conns or self._management_callbacks): log.debug("Restarting event loop") continue else: @@ -148,9 +157,6 @@ def _run_loop(self): def _cleanup(self): # TODO: unguarded write; self._notify() below wakes _run_loop to re-check. self._shutdown = True - if not self._thread: - return - # Stop the prepare watcher first to prevent race conditions. # It was unref'd when started, so ref the loop back first: stopping it # unbalanced leaves libev's active count at -1, and a loop thread still @@ -161,6 +167,7 @@ def _cleanup(self): self._loop.ref() self._preparer.stop() self._loop_timer.stop() + self._management_keeper.stop() # TODO: unguarded read; sets are copy-on-write, so at most a stale # snapshot (redundant idempotent close), never a torn read. @@ -176,9 +183,10 @@ def _cleanup(self): # PYTHON-752 Thread might have just been created and not started # Use longer timeout to allow proper cleanup with self._lock_thread: - self._thread.join(timeout=5.0) + if self._thread is not None and self._thread is not current_thread(): + self._thread.join(timeout=5.0) - if self._thread.is_alive(): + if self._thread is not None and self._thread.is_alive(): log.warning( "Event loop thread could not be joined, so shutdown may not be clean. " "Please call Cluster.shutdown() to avoid this.") @@ -203,6 +211,77 @@ def _update_timer(self): def _on_loop_timer(self): self._timers.service_timeouts() + def call_soon(self, callback): + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + self._management_callbacks.append(callback) + try: + self.maybe_start() + except BaseException: + with self._lock: + try: + self._management_callbacks.remove(callback) + except ValueError: + return # Already claimed: the dispatch was accepted. + raise + + def _keep_management_alive(self): + if self._management_users and not self._shutdown: + self._management_keeper.start(3600) + + def borrow_management(self): + token = object() + ready, abandoned = Event(), Event() + + def acquire(): + if abandoned.is_set(): + return + self._management_users.add(token) + self._management_keeper.start(3600) + if abandoned.is_set(): + release(lambda: None) + else: + ready.set() + + def release(completed): + def detach(): + self._management_users.discard(token) + if not self._management_users: + self._management_keeper.stop() + completed() + if current_thread() is self._thread: + detach() + elif self._shutdown or self._thread is None or not self._thread.is_alive(): + # Catastrophic service loss: no native runner can touch watchers. + with self._lock: + detach() + else: + self.call_soon(detach) + + if current_thread() is self._thread: + acquire() + else: + self.call_soon(acquire) + if not ready.wait(5): + abandoned.set() + release(lambda: None) + raise RuntimeError('Libev management borrow did not complete') + return release + + def _dispatch_management(self): + # Bounded batches keep management completions fair to socket I/O. + for _ in range(64): + with self._lock: + if not self._management_callbacks: + return + callback = self._management_callbacks.popleft() + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + self._notifier.send() + def notify(self): self._notifier.send() @@ -288,6 +367,7 @@ def _close_conns(conns): def _loop_will_run(self, prepare): changed = False self._drain_deferred() + self._dispatch_management() # TODO: unguarded read; copy-on-write set, self-heals next tick. for conn in self._live_conns: @@ -341,6 +421,7 @@ def _atexit_cleanup(): _global_loop = None +_initialization_lock = Lock() atexit.register(_atexit_cleanup) @@ -356,13 +437,35 @@ class LibevConnection(Connection): @classmethod def initialize_reactor(cls): global _global_loop - if not _global_loop: - _global_loop = LibevLoop() - else: - if _global_loop._pid != os.getpid(): - log.debug("Detected fork, clearing and reinitializing reactor state") - cls.handle_fork() - _global_loop = LibevLoop() + with _initialization_lock: + if _global_loop is not None and _global_loop._pid != os.getpid(): + # Inherited threads cannot be joined in the child. + _global_loop = None + if _global_loop is None: + candidate = LibevLoop() + ready = Event() + try: + candidate.call_soon(ready.set) + if not ready.wait(5): + raise RuntimeError("Libev reactor startup did not complete") + except BaseException: + candidate._cleanup() + raise + _global_loop = candidate + + @classmethod + def _management_dispatch(cls, callback): + if _global_loop is None: + raise RuntimeError("Libev reactor is unavailable") + _global_loop.call_soon(callback) + + @classmethod + def _management_borrow(cls): + return _global_loop.borrow_management() + + @classmethod + def _management_in_context(cls): + return _global_loop is not None and current_thread() is _global_loop._thread @classmethod def handle_fork(cls): @@ -520,3 +623,12 @@ def push(self, data): with self._deque_lock: self.deque.extend(chunks) _global_loop.notify() + + +def _reset_management_startup_lock_after_fork(): + global _initialization_lock + _initialization_lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/tests/unit/io/test_asyncioreactor.py b/tests/unit/io/test_asyncioreactor.py index 76aca20ef3..31bef870da 100644 --- a/tests/unit/io/test_asyncioreactor.py +++ b/tests/unit/io/test_asyncioreactor.py @@ -41,7 +41,13 @@ def tearDownClass(cls): if skip_me: return if ASYNCIO_AVAILABLE and AsyncioConnection._loop: - AsyncioConnection._loop.stop() + loop = AsyncioConnection._loop + loop.call_soon_threadsafe(loop.stop) + AsyncioConnection._loop_thread.join(5) + assert not AsyncioConnection._loop_thread.is_alive() + loop.close() + AsyncioConnection._loop = None + AsyncioConnection._loop_thread = None @property def create_timer(self): diff --git a/tests/unit/io/test_libevreactor_shutdown.py b/tests/unit/io/test_libevreactor_shutdown.py index af5b420888..27bfe1e3e5 100644 --- a/tests/unit/io/test_libevreactor_shutdown.py +++ b/tests/unit/io/test_libevreactor_shutdown.py @@ -26,6 +26,7 @@ import tempfile import os from pathlib import Path +from unittest.mock import patch from cassandra import DependencyException @@ -216,6 +217,17 @@ def test_cleanup_with_fix_properly_shuts_down(self): from cassandra.io import libevreactor from cassandra.io.libevreactor import _cleanup, _atexit_cleanup + # Simulated process exit must not stop the shared reactor used by later + # tests. Restore its singleton even if an assertion below fails. + singleton = patch.object(libevreactor, '_global_loop', None) + singleton.start() + self.addCleanup(singleton.stop) + def close_private_loop(): + loop = libevreactor._global_loop + if loop is not None and not loop._shutdown: + _cleanup(loop) + self.addCleanup(close_private_loop) + LibevConnection.initialize_reactor() # Verify the loop exists diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py new file mode 100644 index 0000000000..9c3686a70c --- /dev/null +++ b/tests/unit/test_management_reactors.py @@ -0,0 +1,225 @@ +"""Socket-free tests of the actual supported reactor completion adapters.""" + +import asyncio +import importlib +from concurrent.futures import CancelledError, Future +from threading import Event, Thread, get_ident + +import pytest + +from cassandra import DependencyException +from cassandra._management import ( + ManagementRuntime, ReactorChannel, execution_context) + + +@pytest.fixture(autouse=True) +def close_borrowed_channels(monkeypatch): + channels = [] + initialize = ReactorChannel.__init__ + def tracked(self, *args, **kwargs): + initialize(self, *args, **kwargs) + channels.append(self) + monkeypatch.setattr(ReactorChannel, '__init__', tracked) + yield + for channel in channels: + detached = Event() + channel.close(detached.set) + assert detached.wait(5) + + +@pytest.fixture(params=['asyncio', 'libev', 'asyncore']) +def connection_class(request): + try: + module = importlib.import_module('cassandra.io.' + { + 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', + 'asyncore': 'asyncorereactor'}[request.param]) + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + return getattr(module, {'asyncio': 'AsyncioConnection', + 'libev': 'LibevConnection', + 'asyncore': 'AsyncoreConnection'}[request.param]) + + +def test_idle_reactor_completion_runs_on_io_thread(connection_class): + channel = ReactorChannel(connection_class) + for _ in range(3): + finished = Event() + observations = [] + def on_io(): + assert execution_context() is None # Ordinary ResponseFuture context. + assert connection_class._management_in_context() + observations.append(get_ident()) + finished.set() + channel.dispatch(on_io) + assert finished.wait(5) + assert observations[0] != get_ident() + + +def test_clusters_borrow_reactor_without_closing_other_channel(connection_class): + channel = ReactorChannel(connection_class) + first = ManagementRuntime(channel.dispatch, on_finalized=channel.close) + second_channel = ReactorChannel(connection_class) + second = ManagementRuntime(second_channel.dispatch, on_finalized=second_channel.close) + callback_entered, release, other_done = Event(), Event(), Event() + handler_entered, handler_release = Event(), Event() + try: + def handler(record): + handler_entered.set() + assert handler_release.wait(5) + return 3 + future = first.submit(handler) + assert handler_entered.wait(5) + def hold(future): + callback_entered.set() + assert release.wait(5) + future.add_done_callback(hold) + handler_release.set() + assert future.result(5) == 3 + assert callback_entered.wait(5) + first.close() + assert not first.finalized.is_set() + # Shared reactor may be blocked by application code; native state still + # progresses and the second channel works as soon as that code returns. + other = second.submit(lambda record: 8) + other.add_done_callback(lambda f: other_done.set()) + assert other.result(5) == 8 + release.set() + assert other_done.wait(5) + first.join(5) + assert second.submit(lambda record: 9).result(5) == 9 + finally: + handler_release.set() + release.set() + first.close() + second.close() + first.join(5) + second.join(5) + + +@pytest.mark.parametrize('terminal', ['result', 'error', 'cancel']) +@pytest.mark.parametrize('registered', ['before', 'after']) +def test_asyncio_chaining_progresses_while_listener_waits(connection_class, terminal, registered): + channel = ReactorChannel(connection_class) + manager = ManagementRuntime(channel.dispatch, on_finalized=channel.close) + application_loop = asyncio.new_event_loop() + app_thread = Thread(target=application_loop.run_forever) + app_thread.start() + entered, complete = Event(), Event() + release_handler = Event() + outcomes = [] + try: + def listener(): + def handler(record): + assert release_handler.wait(5) + if terminal == 'error': + raise ValueError('expected') + return 12 + source = manager.submit(handler) + if registered == 'after': + if terminal == 'cancel': + source.cancel() + release_handler.set() + try: + source.result(5) + except (ValueError, CancelledError): + pass + async def consume(): + wrapped = asyncio.wrap_future(source) + entered.set() + return await wrapped + chained = asyncio.run_coroutine_threadsafe(consume(), application_loop) + if registered == 'before': + assert entered.wait(5) + if terminal == 'cancel': + source.cancel() + release_handler.set() + try: + outcomes.append(chained.result(5)) + except (ValueError, CancelledError) as exc: + outcomes.append(type(exc)) + complete.set() + notification = manager.notify(listener) + assert complete.wait(5) + notification.result(5) + assert outcomes == [{'result': 12, 'error': ValueError, 'cancel': CancelledError}[terminal]] + finally: + release_handler.set() + manager.close() + manager.join(5) + application_loop.call_soon_threadsafe(application_loop.stop) + app_thread.join(5) + application_loop.close() + + +def test_libev_dispatch_enqueued_during_idle_exit_is_not_lost(): + try: + module = importlib.import_module('cassandra.io.libevreactor') + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + loop = module.LibevLoop() + native = loop._loop + returned, release, first, second = Event(), Event(), Event(), Event() + class PausedExit: + paused = False + def start(self): + native.start() + if not self.paused: + self.paused = True + returned.set() + assert release.wait(5) + def __getattr__(self, name): + return getattr(native, name) + loop._loop = PausedExit() + try: + loop.call_soon(first.set) + assert first.wait(5) + assert returned.wait(5) + loop.call_soon(second.set) + release.set() + assert second.wait(5), 'idle exit lost a completion queued before exit committed' + loop._thread.join(5) + assert not loop._thread.is_alive() + finally: + release.set() + loop._cleanup() + + +def test_reactor_startup_lock_is_reset_in_fork_child(connection_class): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + channel = ReactorChannel(connection_class) + module = importlib.import_module(connection_class.__module__) + lock = (connection_class._lock if module.__name__.endswith('asyncioreactor') + else module._initialization_lock) + entered, release = Event(), Event() + def holder(): + with lock: + entered.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert entered.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + child_channel = ReactorChannel(connection_class) + os.write(write_fd, b'started') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'new child reactor entered inherited startup lock' + assert os.read(read_fd, 8) == b'started' + finally: + release.set() + thread.join(5) + os.close(read_fd) + os.waitpid(pid, 0) From 8008e60e019a0986b23082a1a4f6199490dcaae1 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 01:09:39 -0400 Subject: [PATCH 2/5] reactors: drain admitted completions before runner exit Serialize callback admission with startup and shutdown, and drain accepted callbacks on the reactor thread when dispatch stops. Reject later submissions after asyncore service loss and remove callbacks whose startup failed. Close inherited socket and dispatcher descriptors before creating the child reactor without acquiring parent locks. Cover startup failures, shutdown backlogs, loop failure and descriptor ownership with socket-free tests. --- cassandra/io/asyncioreactor.py | 46 ++++- cassandra/io/asyncorereactor.py | 65 +++++-- cassandra/io/libevreactor.py | 90 ++++++---- tests/unit/test_management_reactors.py | 236 +++++++++++++++++++++++++ 4 files changed, 389 insertions(+), 48 deletions(-) diff --git a/cassandra/io/asyncioreactor.py b/cassandra/io/asyncioreactor.py index 1688a160f2..03745f5ab8 100644 --- a/cassandra/io/asyncioreactor.py +++ b/cassandra/io/asyncioreactor.py @@ -1,4 +1,5 @@ import threading +from collections import deque from cassandra.connection import Connection, ConnectionShutdown import sys @@ -133,6 +134,7 @@ class AsyncioConnection(Connection): _lock = Lock() _loop_thread = None + _management_state = None _write_queue = None _write_queue_lock = None @@ -259,7 +261,34 @@ def initialize_reactor(cls): if cls._loop is None: assert cls._loop_thread is None loop = asyncio.new_event_loop() - thread = Thread(target=loop.run_forever, daemon=True, + pending, management_lock = deque(), Lock() + accepting = [True] + + def drain(final=False): + count = 0 + while final or count < 64: + with management_lock: + if not pending: + return + callback = pending.popleft() + count += 1 + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + with management_lock: + if pending and accepting[0]: + loop.call_soon(drain) + + def run(): + try: + loop.run_forever() + finally: + with management_lock: + accepting[0] = False + drain(final=True) + + thread = Thread(target=run, daemon=True, name="asyncio_thread") ready = Event() try: @@ -275,13 +304,24 @@ def initialize_reactor(cls): loop.close() raise cls._loop, cls._loop_thread = loop, thread + cls._management_state = (pending, management_lock, accepting, drain) @classmethod def _management_dispatch(cls, callback): loop = cls._loop - if loop is None or loop.is_closed() or not cls._loop_thread.is_alive(): + if cls._management_state is None: raise RuntimeError("Asyncio reactor is unavailable") - loop.call_soon_threadsafe(callback) + pending, lock, accepting, drain = cls._management_state + with lock: + if (not accepting[0] or loop is None or loop.is_closed() or + not cls._loop_thread.is_alive()): + raise RuntimeError("Asyncio reactor is unavailable") + pending.append(callback) + try: + loop.call_soon_threadsafe(drain) + except BaseException: + pending.remove(callback) + raise @classmethod def _management_in_context(cls): diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index c91e1a2ebf..fdd600f0e9 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -18,7 +18,7 @@ import os import socket import sys -from threading import Lock, Thread, Event, current_thread +from threading import Lock, RLock, Thread, Event, current_thread import time import sys import ssl @@ -137,6 +137,14 @@ def notify_loop(self): self._notified = True os.write(self.write_fd, b'x') + def close(self): + try: + asyncore.dispatcher.close(self) + finally: + if self.write_fd is not None: + os.close(self.write_fd) + self.write_fd = None + class _AsyncoreUDPDispatcher(_AsyncoreDispatcher): """ @@ -210,11 +218,12 @@ class AsyncoreLoop(object): def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() - self._state_lock = Lock() - self._management_lock = Lock() + self._state_lock = RLock() + self._management_lock = self._state_lock self._management_callbacks = deque() self._started = False self._shutdown = False + self._cleanup_started = False self._thread = None @@ -240,8 +249,9 @@ def maybe_start(self): try: self._thread.start() except BaseException: - self._started = False - self._thread = None + if not self._thread.is_alive(): + self._started = False + self._thread = None raise def call_soon(self, callback): @@ -249,8 +259,15 @@ def call_soon(self, callback): if self._shutdown: raise RuntimeError("Asyncore reactor is shutting down") self._management_callbacks.append(callback) - self.maybe_start() - self.wake_loop() + try: + self.maybe_start() + self.wake_loop() + except BaseException: + try: + self._management_callbacks.remove(callback) + except ValueError: + return + raise def _dispatch_management(self): for _ in range(64): @@ -270,16 +287,23 @@ def wake_loop(self): def _run_loop(self): log.debug("Starting asyncore event loop") with self._loop_lock: - while not self._shutdown: - try: + try: + while not self._shutdown: self._loop_dispatcher.loop(self.timer_resolution) self._timers.service_timeouts() self._dispatch_management() - except Exception as exc: - self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) - break - with self._state_lock: - self._started = False + except BaseException as exc: + self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) + finally: + # Commit service loss atomically with admission. Accepted + # callbacks still run on this runner before it relinquishes + # ownership; later submissions receive an explicit rejection. + with self._state_lock: + self._shutdown = True + while self._management_callbacks: + self._dispatch_management() + with self._state_lock: + self._started = False self._maybe_log_debug("Asyncore event loop ended") @@ -304,6 +328,9 @@ def _cleanup(self): global _dispatcher_map with self._state_lock: + if self._cleanup_started: + return + self._cleanup_started = True self._shutdown = True self.wake_loop() @@ -347,6 +374,16 @@ def initialize_reactor(cls): global _global_loop, _dispatcher_map with _initialization_lock: if _global_loop is not None and _global_loop._pid != os.getpid(): + # Close descriptors directly: Connection.close() can acquire + # parent-owned locks or schedule timers on a vanished runner. + for dispatcher in tuple(_dispatcher_map.values()): + try: + if dispatcher is _global_loop._loop_dispatcher: + dispatcher.close() + else: + dispatcher.socket.close() + except OSError: + pass _dispatcher_map = {} _global_loop = None if _global_loop is None: diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 3f5dc0c371..b8c100328a 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -70,7 +70,7 @@ def __init__(self): # re-entered by a thread that already holds it, via _is_owned(): # private, but threading.Condition relies on it too. self._lock = RLock() - self._lock_thread = Lock() + self._lock_thread = RLock() self._thread = None @@ -85,6 +85,7 @@ def __init__(self): self._deferred_destroyed = [] self._management_callbacks = deque() self._management_users = set() + self._cleanup_started = False self._preparer = libev.Prepare(self._loop, self._loop_will_run) # prevent _preparer from keeping the loop from returning @@ -96,27 +97,24 @@ def __init__(self): self._management_keeper = libev.Timer(self._loop, self._keep_management_alive) def maybe_start(self): - should_start = False - with self._lock: - if not self._started: - log.debug("Starting libev event loop") - self._started = True - should_start = True - - if should_start: - with self._lock_thread: - if not self._shutdown: + # Serialize all admissions with startup, including its failure path. + with self._lock_thread: + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + if not self._started: + log.debug("Starting libev event loop") + self._started = True self._thread = Thread(target=self._run_loop, name="event_loop") self._thread.daemon = True try: self._thread.start() except BaseException: - with self._lock: + if not self._thread.is_alive(): self._started = False self._thread = None raise - - self._notifier.send() + self._notifier.send() def _run_loop(self): while True: @@ -153,10 +151,18 @@ def _run_loop(self): self._close_conns(to_stop) break self._drain_deferred() + # Shutdown rejects new admissions; finish the accepted queue on this + # reactor thread even when cleanup has stopped the prepare watcher. + while self._management_callbacks: + self._dispatch_management() def _cleanup(self): - # TODO: unguarded write; self._notify() below wakes _run_loop to re-check. - self._shutdown = True + with self._lock_thread: + with self._lock: + if self._cleanup_started: + return + self._cleanup_started = True + self._shutdown = True # Stop the prepare watcher first to prevent race conditions. # It was unref'd when started, so ref the loop back first: stopping it # unbalanced leaves libev's active count at -1, and a loop thread still @@ -183,8 +189,9 @@ def _cleanup(self): # PYTHON-752 Thread might have just been created and not started # Use longer timeout to allow proper cleanup with self._lock_thread: - if self._thread is not None and self._thread is not current_thread(): - self._thread.join(timeout=5.0) + thread = self._thread + if thread is not None and thread is not current_thread(): + thread.join(timeout=5.0) if self._thread is not None and self._thread.is_alive(): log.warning( @@ -193,6 +200,26 @@ def _cleanup(self): log.debug("Event loop thread was joined") + def _cleanup_after_fork(self): + # Only this thread survived. Do not enter inherited Python locks, + # invoke connection callbacks, or join vanished parent threads. + self._shutdown = True + for conn in self._live_conns | self._new_conns | self._closed_conns: + try: + conn._socket.close() + except OSError: + pass + conn._read_watcher = conn._write_watcher = None + self._live_conns = self._new_conns = self._closed_conns = set() + self._deferred_destroyed = [] + self._management_callbacks.clear() + self._management_users.clear() + # Break watcher/callback cycles and release the native loop, whose + # destructor closes its internal wakeup descriptors in the child. + self._preparer = self._loop_timer = self._management_keeper = None + self._notifier = None + self._loop = None + def add_timer(self, timer): self._timers.add_timer(timer) self._notifier.send() # wake up in case this timer is earlier @@ -212,19 +239,20 @@ def _on_loop_timer(self): self._timers.service_timeouts() def call_soon(self, callback): - with self._lock: - if self._shutdown: - raise RuntimeError("Libev reactor is shutting down") - self._management_callbacks.append(callback) - try: - self.maybe_start() - except BaseException: + with self._lock_thread: with self._lock: - try: - self._management_callbacks.remove(callback) - except ValueError: - return # Already claimed: the dispatch was accepted. - raise + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + self._management_callbacks.append(callback) + try: + self.maybe_start() + except BaseException: + with self._lock: + try: + self._management_callbacks.remove(callback) + except ValueError: + return # Already claimed: the dispatch was accepted. + raise def _keep_management_alive(self): if self._management_users and not self._shutdown: @@ -439,7 +467,7 @@ def initialize_reactor(cls): global _global_loop with _initialization_lock: if _global_loop is not None and _global_loop._pid != os.getpid(): - # Inherited threads cannot be joined in the child. + _global_loop._cleanup_after_fork() _global_loop = None if _global_loop is None: candidate = LibevLoop() diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 9c3686a70c..29b5deff32 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -223,3 +223,239 @@ def holder(): thread.join(5) os.close(read_fd) os.waitpid(pid, 0) + + +@pytest.mark.parametrize('reactor', ['libev', 'asyncore']) +def test_startup_failure_does_not_acknowledge_concurrent_dispatch(monkeypatch, reactor): + module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + if reactor == 'asyncore': + monkeypatch.setattr(module, '_dispatcher_map', {}) + class Dispatcher: + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + Event().wait(0.001) + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() + starting, fail, second_entered, accepted, delivered = [Event() for _ in range(5)] + original_thread = module.Thread + calls = [] + class FailedStart: + def start(self): + starting.set() + assert fail.wait(5) + raise RuntimeError('thread creation failed') + def is_alive(self): + return False + def factory(*args, **kwargs): + calls.append(True) + return FailedStart() if len(calls) == 1 else original_thread(*args, **kwargs) + monkeypatch.setattr(module, 'Thread', factory) + errors, rejected_callbacks = [], [] + def first(): + try: + loop.call_soon(lambda: rejected_callbacks.append(True)) + except RuntimeError as exc: + errors.append(str(exc)) + def second(): + second_entered.set() + loop.call_soon(delivered.set) + accepted.set() + first_thread, second_thread = Thread(target=first), Thread(target=second) + try: + first_thread.start() + assert starting.wait(5) + second_thread.start() + assert second_entered.wait(5) + assert not accepted.wait(0.05) + fail.set() + first_thread.join(5) + second_thread.join(5) + assert accepted.is_set() + assert delivered.wait(5) + assert errors == ['thread creation failed'] + assert rejected_callbacks == [] + finally: + fail.set() + first_thread.join(5) + if second_thread.ident is not None: + second_thread.join(5) + loop._cleanup() + + +def test_asyncore_dispatch_accepted_during_runner_failure_is_delivered(monkeypatch): + module = pytest.importorskip('cassandra.io.asyncorereactor') + monkeypatch.setattr(module, '_dispatcher_map', {}) + entered, fail, delivered = Event(), Event(), Event() + class Dispatcher: + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + entered.set() + assert fail.wait(5) + raise RuntimeError('dispatcher failed') + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = module.AsyncoreLoop() + observations = [] + try: + loop.maybe_start() + assert entered.wait(5) + def completion(): + observations.append(get_ident()) + delivered.set() + loop.call_soon(completion) + fail.set() + loop._thread.join(5) + assert delivered.is_set() + assert observations == [loop._thread.ident] + with pytest.raises(RuntimeError, match='shutting down'): + loop.call_soon(lambda: None) + assert not loop._management_callbacks + finally: + fail.set() + loop._cleanup() + + +@pytest.mark.parametrize('reactor', ['asyncio', 'libev', 'asyncore']) +@pytest.mark.parametrize('count', [1, 130]) +def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, reactor, count): + module = pytest.importorskip('cassandra.io.' + { + 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', + 'asyncore': 'asyncorereactor'}[reactor]) + entered, release, delivered, stopping = [Event() for _ in range(4)] + if reactor == 'asyncore': + monkeypatch.setattr(module, '_dispatcher_map', {}) + if reactor == 'asyncio': + connection = type('IsolatedConnection', (module.AsyncioConnection,), + {'_loop': None, '_loop_thread': None}) + connection.initialize_reactor() + dispatch, runner = connection._management_dispatch, connection._loop_thread + def stop(): + connection._loop.call_soon_threadsafe(stopping.set) + connection._loop.call_soon_threadsafe(connection._loop.stop) + def cleanup(): + runner.join(5) + connection._loop.close() + else: + loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() + dispatch = loop.call_soon + def stop(): + stopping.set() + loop._cleanup() + def cleanup(): + loop._cleanup() + runner = None + observations = [] + def hold(): + entered.set() + assert release.wait(5) + stopper = None + try: + dispatch(hold) + assert entered.wait(5) + def completion(): + observations.append(get_ident()) + if len(observations) == count: + delivered.set() + for _ in range(count): + dispatch(completion) + if reactor == 'asyncio': + stop() + else: + runner = loop._thread + stopper = Thread(target=stop) + stopper.start() + assert stopping.wait(5) + # Cleanup commits shutdown under the admission lock before join. + for _ in range(500): + with (loop._lock if reactor == 'libev' else loop._state_lock): + if loop._shutdown: + break + Event().wait(0.001) + assert loop._shutdown + release.set() + assert delivered.wait(5) + runner.join(5) + assert not runner.is_alive() + if stopper is not None: + stopper.join(5) + assert not stopper.is_alive() + assert observations == [runner.ident] * count + with pytest.raises(RuntimeError): + dispatch(lambda: None) + finally: + release.set() + if stopper is not None: + stopper.join(5) + cleanup() + + +@pytest.mark.parametrize('reactor', ['libev', 'asyncore']) +def test_fork_reinitialization_closes_inherited_descriptors(monkeypatch, reactor): + import os + import select + import signal + import socket + from types import SimpleNamespace + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + connection = getattr(module, {'libev': 'LibevConnection', + 'asyncore': 'AsyncoreConnection'}[reactor]) + connection.initialize_reactor() + inherited = module._global_loop + ours, peer = socket.socketpair() + if reactor == 'libev': + class Connection: + _socket = ours + _read_watcher = _write_watcher = None + conn = Connection() + inherited._live_conns.add(conn) + else: + module._dispatcher_map[ours.fileno()] = SimpleNamespace(socket=ours) + original_factory = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor]) + def checked_factory(): + assert ours.fileno() == -1 + if reactor == 'libev': + assert inherited._loop is None + assert inherited._notifier is None + else: + assert inherited._loop_dispatcher.write_fd is None + return original_factory() + monkeypatch.setattr(module, original_factory.__name__, checked_factory) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + connection.initialize_reactor() + os.write(write_fd, b'closed') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready + assert os.read(read_fd, 6) == b'closed' + # The child closed only its descriptor copies. + ours.send(b'x') + assert peer.recv(1) == b'x' + finally: + if reactor == 'libev': + inherited._live_conns.discard(conn) + else: + module._dispatcher_map.pop(ours.fileno(), None) + ours.close() + peer.close() + os.close(read_fd) + os.waitpid(pid, 0) From 4747f2b6a34ac6717887807c02e4fdeecd51f16e Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 03:19:48 -0400 Subject: [PATCH 3/5] reactors: preserve runner ownership during teardown and recovery Drain libev completions after exit only for the shutdown owner, so a retired idle runner cannot claim a replacement runner's work. Stop a management keeper armed concurrently with cleanup or final borrower release. Separate permanent asyncore management failure from ordinary I/O shutdown. Drain accepted completions before retiring the failed runner, reject stale management channels, and allow ordinary connections to restart I/O. Gate tests at the actual admission, idle-exit and timer-arm boundaries. Exercise asyncio final draining before queued normal drains can run, and always stop the chaining test's application loop when management join fails. --- cassandra/io/asyncorereactor.py | 16 +- cassandra/io/libevreactor.py | 18 +- tests/unit/test_management_reactors.py | 266 ++++++++++++++++++++++++- 3 files changed, 277 insertions(+), 23 deletions(-) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index fdd600f0e9..79dbfeef2e 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -219,8 +219,8 @@ def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() self._state_lock = RLock() - self._management_lock = self._state_lock self._management_callbacks = deque() + self._management_failed = False self._started = False self._shutdown = False self._cleanup_started = False @@ -255,8 +255,8 @@ def maybe_start(self): raise def call_soon(self, callback): - with self._management_lock: - if self._shutdown: + with self._state_lock: + if self._shutdown or self._management_failed: raise RuntimeError("Asyncore reactor is shutting down") self._management_callbacks.append(callback) try: @@ -271,7 +271,7 @@ def call_soon(self, callback): def _dispatch_management(self): for _ in range(64): - with self._management_lock: + with self._state_lock: if not self._management_callbacks: return callback = self._management_callbacks.popleft() @@ -295,11 +295,11 @@ def _run_loop(self): except BaseException as exc: self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) finally: - # Commit service loss atomically with admission. Accepted - # callbacks still run on this runner before it relinquishes - # ownership; later submissions receive an explicit rejection. + # Management service loss is permanent for existing channels. + # Ordinary I/O can restart after this runner drains accepted + # completions and relinquishes ownership. with self._state_lock: - self._shutdown = True + self._management_failed = True while self._management_callbacks: self._dispatch_management() with self._state_lock: diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index b8c100328a..a4ee93bb63 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -137,6 +137,7 @@ def _run_loop(self): else: # all Connections have been closed, no active watchers log.debug("All Connections currently closed, event loop ended") + shutdown = self._shutdown self._started = False # A connection may have been closed after the last # _loop_will_run() pass, which is where closed @@ -151,10 +152,11 @@ def _run_loop(self): self._close_conns(to_stop) break self._drain_deferred() - # Shutdown rejects new admissions; finish the accepted queue on this - # reactor thread even when cleanup has stopped the prepare watcher. - while self._management_callbacks: - self._dispatch_management() + # Only the shutdown owner may drain after publishing stopped state. + # Ordinary idle exit allows a replacement runner to claim new work. + if shutdown: + while self._management_callbacks: + self._dispatch_management() def _cleanup(self): with self._lock_thread: @@ -257,6 +259,9 @@ def call_soon(self, callback): def _keep_management_alive(self): if self._management_users and not self._shutdown: self._management_keeper.start(3600) + # Cleanup or release can stop the timer while start() is in flight. + if self._shutdown or not self._management_users: + self._management_keeper.stop() def borrow_management(self): token = object() @@ -266,7 +271,7 @@ def acquire(): if abandoned.is_set(): return self._management_users.add(token) - self._management_keeper.start(3600) + self._keep_management_alive() if abandoned.is_set(): release(lambda: None) else: @@ -281,7 +286,8 @@ def detach(): if current_thread() is self._thread: detach() elif self._shutdown or self._thread is None or not self._thread.is_alive(): - # Catastrophic service loss: no native runner can touch watchers. + # Shutdown closes admission before the runner finishes draining. + # Release remains idempotent while that runner unwinds. with self._lock: detach() else: diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index 29b5deff32..bcb8539d44 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -3,7 +3,7 @@ import asyncio import importlib from concurrent.futures import CancelledError, Future -from threading import Event, Thread, get_ident +from threading import Event, Thread, current_thread, get_ident import pytest @@ -144,11 +144,14 @@ async def consume(): assert outcomes == [{'result': 12, 'error': ValueError, 'cancel': CancelledError}[terminal]] finally: release_handler.set() - manager.close() - manager.join(5) - application_loop.call_soon_threadsafe(application_loop.stop) - app_thread.join(5) - application_loop.close() + try: + manager.close() + manager.join(5) + finally: + application_loop.call_soon_threadsafe(application_loop.stop) + app_thread.join(5) + assert not app_thread.is_alive() + application_loop.close() def test_libev_dispatch_enqueued_during_idle_exit_is_not_lost(): @@ -184,6 +187,155 @@ def __getattr__(self, name): loop._cleanup() +def test_libev_retired_runner_does_not_claim_replacement_work(monkeypatch): + module = pytest.importorskip('cassandra.io.libevreactor') + loop = module.LibevLoop() + native, drain = loop._loop, loop._drain_deferred + retired, release_old, replacement, release_new, delivered = [Event() for _ in range(5)] + old_threads, observations = [], [] + def pause_retired(): + drain() + if not loop._started and not retired.is_set(): + old_threads.append(current_thread()) + retired.set() + assert release_old.wait(5) + class PausedReplacement: + def start(self): + if retired.is_set() and current_thread() is not old_threads[0]: + replacement.set() + assert release_new.wait(5) + native.start() + def __getattr__(self, name): + return getattr(native, name) + monkeypatch.setattr(loop, '_drain_deferred', pause_retired) + loop._loop = PausedReplacement() + def completion(): + observations.append((get_ident(), current_thread() is loop._thread)) + delivered.set() + try: + loop.call_soon(lambda: None) + assert retired.wait(5), 'runner did not publish its idle exit' + loop.call_soon(completion) + assert replacement.wait(5) + new_thread = loop._thread + assert new_thread is not old_threads[0] + release_old.set() + old_threads[0].join(5) + assert not old_threads[0].is_alive() + assert observations == [], 'retired runner consumed new-generation work' + release_new.set() + assert delivered.wait(5) + new_thread.join(5) + assert observations == [(new_thread.ident, True)] + assert not new_thread.is_alive() + finally: + release_old.set() + release_new.set() + loop._cleanup() + for thread in old_threads: + thread.join(5) + + +@pytest.mark.parametrize('arming_path', ['acquire', 'keepalive']) +def test_libev_keeper_arming_cannot_outlive_cleanup(monkeypatch, arming_path): + module = pytest.importorskip('cassandra.io.libevreactor') + loop = module.LibevLoop() + native = loop._management_keeper + arming, resume, stopped, armed = [Event() for _ in range(4)] + releases, errors = [], [] + class PausedKeeper: + active = False + def start(self, timeout): + arming.set() + assert resume.wait(5) + native.start(timeout) + self.active = True + armed.set() + def stop(self): + native.stop() + self.active = False + stopped.set() + keeper = PausedKeeper() + monkeypatch.setattr(loop, '_management_keeper', keeper) + def borrow(): + try: + releases.append(loop.borrow_management()) + except BaseException as exc: + errors.append(exc) + borrower = Thread(target=borrow) if arming_path == 'acquire' else None + stopper = None + try: + if borrower is not None: + borrower.start() + else: + loop._management_users.add(object()) + loop.call_soon(loop._keep_management_alive) + assert arming.wait(5) + runner = loop._thread + stopper = Thread(target=loop._cleanup) + stopper.start() + assert stopped.wait(5), 'cleanup did not stop the in-flight keeper' + assert loop._shutdown + resume.set() + assert armed.wait(5) + runner.join(5) + assert not runner.is_alive(), 'keeper was resurrected after cleanup' + assert not keeper.active + stopper.join(5) + assert not stopper.is_alive() + if borrower is not None: + borrower.join(5) + assert not borrower.is_alive() + assert errors == [] + assert len(releases) == 1 + finally: + resume.set() + keeper.stop() + loop.notify() + if borrower is not None: + borrower.join(5) + for release in releases: + release(lambda: None) + if stopper is not None: + stopper.join(5) + loop._cleanup() + + +def test_chaining_application_loop_closes_when_management_join_raises(monkeypatch, connection_class): + # Exercise the actual chaining test's failure cleanup, then supply our own + # fallback cleanup so a negative control cannot leak a non-daemon thread. + ReactorChannel(connection_class) + app_threads, app_loops = [], [] + make_loop, thread_class, join = asyncio.new_event_loop, Thread, ManagementRuntime.join + def new_loop(): + loop = make_loop() + app_loops.append(loop) + return loop + def new_thread(*args, **kwargs): + thread = thread_class(*args, **kwargs) + app_threads.append(thread) + return thread + def failed_join(manager, timeout): + join(manager, timeout) + raise RuntimeError('forced finalization failure') + monkeypatch.setattr(asyncio, 'new_event_loop', new_loop) + monkeypatch.setattr(__name__ + '.Thread', new_thread) + monkeypatch.setattr(ManagementRuntime, 'join', failed_join) + try: + with pytest.raises(RuntimeError, match='forced finalization failure'): + test_asyncio_chaining_progresses_while_listener_waits(connection_class, 'result', 'before') + assert len(app_threads) == len(app_loops) == 1 + assert not app_threads[0].is_alive() + assert app_loops[0].is_closed() + finally: + for loop, thread in zip(app_loops, app_threads): + if thread.is_alive(): + loop.call_soon_threadsafe(loop.stop) + thread.join(5) + if not thread.is_alive() and not loop.is_closed(): + loop.close() + + def test_reactor_startup_lock_is_reset_in_fork_child(connection_class): import os import select @@ -242,6 +394,7 @@ def close(self): monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() starting, fail, second_entered, accepted, delivered = [Event() for _ in range(5)] + admission_attempted = Event() original_thread = module.Thread calls = [] class FailedStart: @@ -266,11 +419,29 @@ def second(): loop.call_soon(delivered.set) accepted.set() first_thread, second_thread = Thread(target=first), Thread(target=second) + lock_name = '_lock_thread' if reactor == 'libev' else '_state_lock' + admission_lock = getattr(loop, lock_name) + blocked = [] + class ObservedLock: + def __enter__(self): + if get_ident() == second_thread.ident: + acquired = admission_lock.acquire(False) + blocked.append(not acquired) + admission_attempted.set() + if acquired: + return self + admission_lock.acquire() + return self + def __exit__(self, *args): + admission_lock.release() + setattr(loop, lock_name, ObservedLock()) try: first_thread.start() assert starting.wait(5) second_thread.start() assert second_entered.wait(5) + assert admission_attempted.wait(5) + assert blocked == [True], 'second admission did not block on provisional startup' assert not accepted.wait(0.05) fail.set() first_thread.join(5) @@ -324,6 +495,74 @@ def completion(): loop._cleanup() +def test_asyncore_ordinary_connection_recovers_without_reviving_management(monkeypatch): + import socket + module = pytest.importorskip('cassandra.io.asyncorereactor') + monkeypatch.setattr(module, '_dispatcher_map', {}) + entered, fail, finish_failure, recovered = [Event() for _ in range(4)] + class Dispatcher: + failed = False + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + if fail.is_set() and not self.failed: + entered.set() + assert finish_failure.wait(5) + self.failed = True + raise OSError('transient poll failure') + if self.failed: + recovered.set() + Event().wait(0.001) + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = module.AsyncoreLoop() + monkeypatch.setattr(module, '_global_loop', loop) + # This channel predates service loss and must never become usable again. + channel = None + ours, peer = socket.socketpair() + connection = None + try: + channel = ReactorChannel(module.AsyncoreConnection) + old_runner = loop._thread + fail.set() + assert entered.wait(5) + delivered = Event() + channel.dispatch(delivered.set) + finish_failure.set() + old_runner.join(5) + assert not old_runner.is_alive() + assert delivered.is_set() + with pytest.raises(RuntimeError): + channel.dispatch(lambda: None) + monkeypatch.setattr(module.AsyncoreConnection, '_connect_socket', + lambda self: setattr(self, '_socket', ours)) + monkeypatch.setattr(module.AsyncoreConnection, '_send_options_message', lambda self: None) + connection = module.AsyncoreConnection(connect_timeout=1) + assert recovered.wait(5) + assert loop._thread is not old_runner + assert loop._thread.is_alive() + assert module._dispatcher_map[ours.fileno()] is connection + assert connection.socket is ours + with pytest.raises(RuntimeError): + channel.dispatch(lambda: None) + with pytest.raises(RuntimeError): + ReactorChannel(module.AsyncoreConnection) + assert not loop._management_callbacks + finally: + fail.set() + finish_failure.set() + if channel is not None: + channel.close() + if connection is not None: + connection.close() + loop._cleanup() + ours.close() + peer.close() + + @pytest.mark.parametrize('reactor', ['asyncio', 'libev', 'asyncore']) @pytest.mark.parametrize('count', [1, 130]) def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, reactor, count): @@ -339,10 +578,12 @@ def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, react connection.initialize_reactor() dispatch, runner = connection._management_dispatch, connection._loop_thread def stop(): - connection._loop.call_soon_threadsafe(stopping.set) - connection._loop.call_soon_threadsafe(connection._loop.stop) + stopping.set() def cleanup(): + if runner.is_alive(): + connection._loop.call_soon_threadsafe(connection._loop.stop) runner.join(5) + assert not runner.is_alive() connection._loop.close() else: loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() @@ -357,9 +598,16 @@ def cleanup(): def hold(): entered.set() assert release.wait(5) + if reactor == 'asyncio': + # Stop in the current ready-queue turn, before any of the drain + # handles queued while this ordinary loop callback was held. + connection._loop.stop() stopper = None try: - dispatch(hold) + if reactor == 'asyncio': + connection._loop.call_soon_threadsafe(hold) + else: + dispatch(hold) assert entered.wait(5) def completion(): observations.append(get_ident()) From babdd7274738baeb23fafb9b2712d33df2c397a4 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sun, 15 Nov 2026 03:19:48 -0400 Subject: [PATCH 4/5] tests: compare disposal thread objects after manager shutdown Manager shutdown can retire a pool owner before the executor creates its worker. PyPy can reuse that owner's native thread ID, so comparing saved integer IDs falsely reports owner-thread disposal. Retain actual Thread objects and verify worker execution context. Preserve the exactly-once disposal and released-credit assertions; an exactly-once owner-disposal negative control still fails the owner-exclusion assertion. --- tests/unit/test_management_runtime.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index 6853bd1989..0bcd94a651 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -1,7 +1,7 @@ from collections import deque from concurrent.futures import ( Future, ThreadPoolExecutor, as_completed, wait, TimeoutError) -from threading import Barrier, Event, Lock, Thread, get_ident +from threading import Barrier, Event, Lock, Thread, current_thread, get_ident import pytest @@ -502,6 +502,10 @@ def test_completed_executor_future_does_not_dispose_on_registering_owner( manager, dispatch = runtime resource = object() disposal_threads = [] + disposal_contexts = [] + def dispose(value): + disposal_threads.append(current_thread()) + disposal_contexts.append(execution_context()) with ThreadPoolExecutor(1) as executor: class ClosingExecutor: def submit(self, function): @@ -514,13 +518,15 @@ def submit(self, function): return physical future = manager.submit(lambda record: manager.work( record, ClosingExecutor(), lambda: resource, - dispose=lambda value: disposal_threads.append(get_ident()))) + dispose=dispose)) with pytest.raises(ManagementClosed): future.result(5) manager.join(5) assert len(disposal_threads) == 1 - assert disposal_threads[0] not in { - thread.ident for thread in manager._threads} + # Native thread IDs may be reused after close retires an owner. Keep the + # actual Thread objects to distinguish that owner from a later worker. + assert all(disposal_threads[0] is not thread for thread in manager._threads) + assert disposal_contexts == ['worker'] assert manager.counts()['ordinary'] == 0 From 75450522328fa39152b45750b2ce83e55f7c0640 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Nov 2026 09:23:16 -0400 Subject: [PATCH 5/5] lifecycle: adapt management producers and synchronous APIs Deliver existing lifecycle notifications through the bounded management runtime while retaining publication locks and causal observation ordering. Reserve synchronous API admission before effects, allow bounded callback reentry between policy phases, and reject blocking calls on owner or I/O threads. Keep ordinary query routing on its existing path. Gate transaction notifications in the scheduler, release nested async admission through its active drainer, and restore idle state after executor rejection or runtime loss. Report unknown-host overload explicitly and release host-UP handling after failed pool admission. Coordinate lazy runtime installation with shutdown and defer managed cleanup until physical pool work and the completion barrier retire. Cover saturation, reentry, startup, shutdown, nested recovery and collection with controlled interleavings on rebuilt Cython and pure Python runtimes. --- cassandra/_lifecycle.py | 370 ++++++++++-- cassandra/_management.py | 28 +- cassandra/cluster.py | 98 +++- tests/unit/test_cluster.py | 8 +- tests/unit/test_control_connection.py | 2 + tests/unit/test_lifecycle_coordination.py | 667 +++++++++++++++++++++- tests/unit/test_management_reactors.py | 49 +- tests/unit/test_management_runtime.py | 48 ++ 8 files changed, 1186 insertions(+), 84 deletions(-) diff --git a/cassandra/_lifecycle.py b/cassandra/_lifecycle.py index b16819456c..5fa97c1157 100644 --- a/cassandra/_lifecycle.py +++ b/cassandra/_lifecycle.py @@ -23,11 +23,14 @@ from contextlib import contextmanager from functools import partial, wraps import logging -from threading import local, RLock +from threading import Event, Lock, local, RLock import weakref from weakref import WeakValueDictionary from cassandra import DriverException +from cassandra._management import (ManagementRuntime, ReactorChannel, + ManagementClosed, ManagementOverloaded, assert_synchronous_api, execution_context, + physical_work_context, performing_cleanup, compatibility_call, compatibility_work, policy_call) __all__ = () @@ -181,12 +184,19 @@ class LifecycleCoordinator: join the queue instead of recursively overtaking the current lifecycle. """ - def __init__(self, cluster=None): + def __init__(self, cluster=None, runtime=None): self._cluster = weakref.proxy(cluster) if cluster is not None else None + self._cluster_ref = weakref.ref(cluster) if cluster is not None else lambda: None self._lock = RLock() self._local = local() self._events = deque() + self._pending_observations = {} + self._runtime = runtime + self._runtime_lock = RLock() self._draining = False + self._active_event = False + self._idle = Event() + self._idle.set() self._requested = 0 self._published = 0 self._force_revisions = deque() @@ -209,7 +219,12 @@ def close(self): if self._closed: return self._closed = True + for event in self._events: + event[1].set() self._events.clear() + self._pending_observations.clear() + if not self._active_event: + self._idle.set() self._force_revisions.clear() hosts = list(self._retired.values()) if self._cluster is not None: @@ -231,6 +246,12 @@ def close(self): handler = host.get_and_set_reconnection_handler(None) if handler is not None: reconnectors.append(handler) + # Startup does not hold the publication lock while it handshakes with + # a reactor. Join its installation boundary only after releasing that + # lock, so a first-use race cannot install a runtime behind shutdown. + with self._runtime_lock: + if self._runtime is not None: + self._runtime.close() for handler in reconnectors: try: handler.cancel() @@ -269,7 +290,7 @@ def submit_readiness(self, host, callback): with self.transaction(): with host.lock: generation = getattr(host, '_readiness_generation', 0) - self.submit(callback, host, generation) + self.submit_observation(host, 'readiness', callback, host, generation) def readiness_is_current(self, host, generation=None): return (self.is_current(host) and @@ -396,7 +417,7 @@ def transaction(self, drain=True): finally: self._local.depth -= 1 finally: - if drain and not getattr(self._local, 'depth', 0): + if (drain or self._draining) and not getattr(self._local, 'depth', 0): self.drain() def reserve_refresh(self, force_token_rebuild=False): @@ -421,41 +442,204 @@ def accept_refresh(self, revision): self._published = revision return True - def submit(self, callback, *args, **kwargs): - """Queue a notification; delivery never has a synchronous result.""" + def _ensure_runtime(self): + # First use occurs after Cluster dependencies have been constructed. + # Never wait for a reactor handshake while holding publication locks. + with self._runtime_lock: + if self.closed: + raise ManagementClosed('Lifecycle coordinator is closed') + if self._runtime is None: + if self._cluster is None: + from cassandra.io.asyncioreactor import AsyncioConnection + connection_class = AsyncioConnection + else: + connection_class = self._cluster.connection_class + runtime = ManagementRuntime(None) + try: + runtime.start() + channel = ReactorChannel(connection_class) + runtime._dispatch = channel.dispatch + runtime._on_finalized = channel.close + if self.closed: + raise ManagementClosed('Lifecycle coordinator closed during startup') + except BaseException: + runtime.close() + runtime.join(5) + raise + self._runtime = runtime + weakref.finalize(self, runtime.close) + if self._cluster is not None: + # The coordinator stores only a proxy; the Cluster can be + # collected even while this runtime is idle. + cluster = self._cluster_ref() + if cluster is not None: + weakref.finalize(cluster, runtime.close) + return self._runtime + + def _queue_event(self, callback): + if self.closed: + return None + try: + runtime = self._ensure_runtime() + except ManagementClosed: + return None + ready = Event() + event = [callback, ready, None] + + def deliver(): + with self._lock: + if self.closed or event not in self._events: + return + self._events.remove(event) + self._active_event = True + try: + self._local.delivering = True + callback() + except Exception: + log.exception("Error delivering host lifecycle notification") + finally: + self._local.delivering = False + with self._lock: + self._active_event = False + if not self._events: + self._draining = False + self._idle.set() + with self._lock: if self.closed: - return - self._events.append(partial(callback, *args, **kwargs)) + return None + try: + event[2] = runtime.notify(deliver, ready=ready) + except ManagementClosed: + return None + self._events.append(event) + if self._draining and not getattr(self._local, 'depth', 0): + # An active drainer already owns delivery. In particular an + # asynchronous producer inside its callback must not depend + # on a separate executor draining again after that callback. + runtime.release_notifications([ready]) + return event + + def submit(self, callback, *args, **kwargs): + """Admit an ordered notification on the serial compatibility lane.""" + self._queue_event(partial(callback, *args, **kwargs)) if not getattr(self._local, 'depth', 0): self.drain() - def submit_async(self, executor, callback, *args, **kwargs): - """Queue before executor handoff so later notifications cannot overtake.""" - event = partial(callback, *args, **kwargs) + def _defer_observation(self, host, family, callback, removed=False): + """One pending batch per known incarnation; no per-call overflow list. + + These are unadmitted, waiter-free compatibility observations. Accepted + batches keep their original credits and generation checks. Removal + replaces pending positive demand; families retain only their latest + generation, not a history of every producer call. + """ + if family not in ('readiness', 'location', 'pool-results', 'down', 'remove'): + raise ValueError('Unknown lifecycle observation family') + if self.closed: + return False + try: + runtime = self._ensure_runtime() + except ManagementClosed: + return False with self._lock: if self.closed: - return None - self._events.append(event) + return False + known = (self.is_current(host) or self._retired.get(id(host)) is host) + if not known: + # There is no lifecycle owner for an unknown incarnation. + # Report rejection to the producer instead of pretending an + # unconsumed refresh-demand bit will recover its completion. + return False + entry = self._pending_observations.setdefault(id(host), (host, {})) + pending = entry[1] + if removed: + pending.clear() + elif 'remove' in pending: + return True + pending.pop(family, None) + pending[family] = callback + runtime.register_health(host) + + def deliver(): + with self._lock: + entry = self._pending_observations.pop(id(host), None) + callbacks = tuple(entry[1].values()) if entry is not None else () + self._active_event = self._draining = True + self._idle.clear() + try: + self._local.delivering = True + for callback in callbacks: + try: + callback() + except Exception: + log.exception('Error delivering deferred lifecycle observation') + finally: + self._local.delivering = False + with self._lock: + self._active_event = False + if not self._events: + self._draining = False + self._idle.set() + if host._is_removed: + runtime.unregister_health(host) + + return runtime.observe_negative(host, None, deliver, removed=removed) + + def submit_observation(self, host, family, callback, *args, **kwargs): + notification = partial(callback, *args, **kwargs) + try: + self.submit(notification) + except ManagementOverloaded: + if (not self._defer_observation(host, family, notification) and + not self.closed and not host._is_removed): + raise + + def _retain_negative(self, host, callback, removed=False): + return self._defer_observation(host, 'remove' if removed else 'down', + callback, removed=removed) + + def submit_negative(self, host, callback, *args, removed=False, **kwargs): + """Retain a native negative transition when ordinary delivery is full.""" + notification = partial(callback, *args, **kwargs) + try: + self.submit(notification) + except ManagementOverloaded: + self._retain_negative(host, notification, removed) + + def submit_async(self, executor, callback, *args, **kwargs): + """Retain admission-before-handoff behavior for existing producers.""" + try: + event = self._queue_event(partial(callback, *args, **kwargs)) + except ManagementOverloaded: + if getattr(callback, '_negative_lifecycle', False): + host = args[1] + if self._retain_negative(host, partial(callback, *args, **kwargs)): + completed = Future() + completed.set_result(None) + return completed + raise + if event is None: + return None try: return executor.submit(self.drain) except Exception: log.exception("Failed to schedule host lifecycle notifications") - # If no drainer has claimed this event, failed handoff cancels - # admission. Otherwise a concurrent submit already delivered it. with self._lock: - try: + if event in self._events: self._events.remove(event) - except ValueError: - if self.closed: - return None - else: + self._runtime.release_notifications([event[1]]) + if not self._events and not self._active_event: + self._draining = False + self._idle.set() + return None + if self.closed: return None completed = Future() completed.set_result(None) return completed - def collect_futures(self, futures, callback): + def collect_futures(self, futures, callback, host=None): """Deliver all results once, with aggregation owned by the FIFO. Register only after the caller has scheduled every operation. Worker @@ -464,6 +648,31 @@ def collect_futures(self, futures, callback): delivered as results so the caller can apply its recovery policy. Closing the coordinator discards outstanding completion notifications. """ + if host is not None: + # Physical outcomes remain in their original futures; retain only + # one completion observation for this existing Host incarnation. + pending = set(futures) + lock = Lock() + results = [] + + def completed(future): + with lock: + if future not in pending: + return + pending.remove(future) + try: + results.append(future.result()) + except Exception as exc: + results.append(exc) + finished = not pending + if finished: + self.submit_observation(host, 'pool-results', callback, results) + + if not pending: + self.submit_observation(host, 'pool-results', callback, results) + for future in tuple(pending): + future.add_done_callback(completed) + return pending = set(futures) results = [] @@ -632,8 +841,9 @@ def retire(self, host, trigger_reconciliation=True, refresh_nodes=True): with self.transaction(): if not self._retire(host): return - self.submit(self._cluster._notify_host_removed, host, - trigger_reconciliation, True, refresh_nodes) + self.submit_negative(host, self._cluster._notify_host_removed, host, + trigger_reconciliation, True, refresh_nodes, + removed=True) def remove(self, host, trigger_reconciliation=True, refresh_nodes=True): with self.transaction(): @@ -642,8 +852,9 @@ def remove(self, host, trigger_reconciliation=True, refresh_nodes=True): return self._retire(host) log.info("Cassandra host %s removed", host) - self.submit(self._cluster._notify_host_removed, host, - trigger_reconciliation, True, refresh_nodes) + self.submit_negative(host, self._cluster._notify_host_removed, host, + trigger_reconciliation, True, refresh_nodes, + removed=True) def publish_host(self, candidate, signal, refresh_nodes, reconcile_pools_on_failure, replace_existing_id): @@ -734,8 +945,8 @@ def _queue_location(self, host, datacenter, rack): token = object() host._pending_location = desired host._pending_location_token = token - self.submit( - self._notify_location_changed, host, datacenter, rack, token) + self.submit_observation( + host, 'location', self._notify_location_changed, host, datacenter, rack, token) return True def _notify_location_changed(self, host, datacenter, rack, token): @@ -760,29 +971,94 @@ def _notify_location_changed(self, host, datacenter, rack, token): host._pending_location_token = None def drain(self): + """Release transaction batches; preserve idle synchronous observations. + + An already active lane is never joined by another producer or a + reentrant listener. Native operation results do not depend on this lane. + """ with self._lock: - if self._draining or self.closed: + if self.closed: return - self._draining = True - try: - while True: - with self._lock: - if self.closed or not self._events: + events = tuple(self._events) + should_wait = not self._draining and not self.delivering + if events: + self._draining = True + self._idle.clear() + if events: + self._runtime.release_notifications(event[1] for event in events) + reactor_context = (self._cluster is not None and + getattr(self._cluster.connection_class, + '_management_in_context', lambda: False)()) + if (should_wait and not reactor_context and + execution_context() not in ('cluster', 'pool', 'bridge', 'worker')): + while not self._idle.wait(0.05): + # Runtime infrastructure failure may discard accepted + # notifications without running our delivery closure. Never + # leave a caller waiting for callbacks that cannot now run. + runtime = self._runtime + if runtime is not None and runtime.closed.is_set(): + with self._lock: self._events.clear() - self._draining = False - return - callback = self._events.popleft() - try: - self._local.delivering = True - callback() - except Exception: - log.exception("Error delivering host lifecycle notification") - finally: - self._local.delivering = False - except BaseException: - with self._lock: - self._draining = False - raise + self._pending_observations.clear() + if not self._active_event: + self._draining = False + self._idle.set() + + +def managed_context(cluster): + if cluster is None: + return False # Bare Session.__new__ tests have no runtime lifetime. + return (execution_context() is not None or + getattr(getattr(cluster, 'connection_class', None), + '_management_in_context', lambda: False)() is True) + + +def defer_cleanup(cluster, callback): + runtime = cluster._lifecycle._runtime + release = runtime.retain_cleanup() if runtime is not None else lambda: None + def cleanup(): + try: + with physical_work_context(cleanup=True): + callback() + finally: + release() + try: + return cluster.executor.submit(cleanup) + except BaseException as exc: + if runtime is not None: + runtime.cleanup_error = exc + release() + log.exception('Failed to submit management lifetime cleanup') + raise + + +def synchronous_api(method): + @wraps(method) + def invoke(self, *args, **kwargs): + cluster = self if hasattr(self, '_lifecycle') else self.cluster + assert_synchronous_api(cluster.connection_class) + if cluster.is_shutdown or not cluster._management_start_allowed(): + return method(self, *args, **kwargs) + coordinator = cluster._lifecycle + runtime = coordinator._ensure_runtime() + with compatibility_call(runtime): + return method(self, *args, **kwargs) + return invoke + + +def observation_event(family): + def decorate(method): + @wraps(method) + def dispatch(self, host, *args, **kwargs): + return self._lifecycle.submit_observation( + host, family, method, self, host, *args, **kwargs) + return dispatch + return decorate + + +def negative_event(method): + method._negative_lifecycle = True + return event(method) def event(method): diff --git a/cassandra/_management.py b/cassandra/_management.py index 7bb9e245de..241b49b3a7 100644 --- a/cassandra/_management.py +++ b/cassandra/_management.py @@ -728,19 +728,32 @@ def physical_work(): receive(None, exc) return _PENDING - def notify(self, callback, *, wait=False, timeout=None, budget='ordinary'): + def notify(self, callback, *, wait=False, timeout=None, budget='ordinary', + ready=None): + """Admit FIFO delivery, optionally gated by a caller publication batch. + + A gate is scheduler state: handlers never wait on the submitting + thread. Release it through ``release_notifications`` after publication. + """ self._check_pid() def admit(record): with self._condition: if self._closing or record.handler_done or record.released: return _PENDING record.notification = True - self._notifications.append((record, callback)) + self._notifications.append((record, callback, ready)) self._condition.notify_all() return _PENDING return self.submit(admit, wait=wait, timeout=timeout, budget=budget, kind='notification') + def release_notifications(self, gates): + self._check_pid() + with self._condition: + for gate in gates: + gate.set() + self._condition.notify_all() + @contextmanager def synchronous_reentry(self): """Reserve one L aggregate and a bounded prerequisite window.""" @@ -1047,11 +1060,14 @@ def _execute(self, record, callback, kind): def _run_notifications(self): while True: with self._condition: - while not self._notifications and not self._closing: + while (not self._closing and + (not self._notifications or + (self._notifications[0][2] is not None and + not self._notifications[0][2].is_set()))): self._condition.wait() if self._closing: return - record, callback = self._notifications.popleft() + record, callback, ready = self._notifications.popleft() with _entered('callback', record): try: value = callback() @@ -1066,7 +1082,7 @@ def _run_notifications(self): with self._condition: record.notification = False self._release_locked(record) - record = callback = value = None + record = callback = ready = value = None def retain_cleanup(self): """Retain one cleanup obligation independently of load.""" @@ -1097,7 +1113,7 @@ def close(self, wait=False): for record in tuple(self._records): self._finish(record, error=ManagementClosed( 'Management lifetime ended')) - for record, _ in self._notifications: + for record, _, _ in self._notifications: record.notification = False self._release_locked(record) self._notifications.clear() diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cd779725ed..3f6918e088 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -690,6 +690,7 @@ def _profiles_without_explicit_lbps(self): for n in names ) + @_lifecycle.policy_call def distance(self, host): if not self.pools_allowed: return HostDistance.IGNORED @@ -699,14 +700,17 @@ def distance(self, host): HostDistance.REMOTE if HostDistance.REMOTE in distances else \ HostDistance.IGNORED + @_lifecycle.policy_call def populate(self, cluster, hosts): for p in self.profiles.values(): p.load_balancing_policy.populate(cluster, hosts) + @_lifecycle.policy_call def check_supported(self): for p in self.profiles.values(): p.load_balancing_policy.check_supported() + @_lifecycle.policy_call def on_up(self, host): if not self.pools_allowed: return @@ -715,6 +719,7 @@ def on_up(self, host): return p.load_balancing_policy.on_up(host) + @_lifecycle.policy_call def on_down(self, host): if not self.pools_allowed: return @@ -723,12 +728,14 @@ def on_down(self, host): if host._is_removed: return + @_lifecycle.policy_call def on_add(self, host): for p in self.profiles.values(): if host._is_removed: return p.load_balancing_policy.on_add(host) + @_lifecycle.policy_call def on_remove(self, host): for p in self.profiles.values(): p.load_balancing_policy.on_remove(host) @@ -1764,6 +1771,10 @@ def __init__(self, schema_metadata_enabled, token_metadata_enabled, schema_meta_page_size=schema_metadata_page_size) + @staticmethod + def _management_start_allowed(): + return not _cluster_scheduler_shutdown_started + def _resolve_hostnames(self): raw_contact_points = [] for cp in [cp for cp in self.contact_points if not isinstance(cp, EndPoint)]: @@ -2084,6 +2095,7 @@ def protocol_downgrade(self, host_endpoint, previous_version): "https://python-driver.docs.scylladb.com/stable/api/cassandra/cluster.html#cassandra.cluster.Cluster.protocol_version", self.protocol_version, new_version, host_endpoint) self.protocol_version = new_version + @_lifecycle.policy_call def _populate_hosts(self): self.profile_manager.populate( weakref.proxy(self), self.metadata.all_hosts()) @@ -2091,6 +2103,7 @@ def _populate_hosts(self): weakref.proxy(self), self.metadata.all_hosts() ) + @_lifecycle.synchronous_api def connect(self, keyspace=None, wait_for_all_pools=False): """ Creates and returns a new :class:`~.Session` object. @@ -2102,6 +2115,7 @@ def connect(self, keyspace=None, wait_for_all_pools=False): established or attempted. Default is `False`, which means it will return when the first successful connection is established. Remaining pools are added asynchronously. """ + _lifecycle.assert_synchronous_api(self.connection_class) reject_late_connect = False shutdown_rejected_cluster = False with self._lock: @@ -2207,6 +2221,14 @@ def shutdown(self): else: self.is_shutdown = True + if _lifecycle.managed_context(self): + try: + self._shutdown_future = _lifecycle.defer_cleanup( + self, partial(self._shutdown, wait_for_executor=False)) + finally: + self._lifecycle.close() + return + # Once interpreter scheduler cleanup has begun, executor teardown is # imminent and this call may itself be running on an executor worker. self._shutdown( @@ -2216,7 +2238,7 @@ def _shutdown(self, wait_for_executor): """Finish cleanup after shutdown has been claimed under ``self._lock``.""" self._lifecycle.close() - if self._lifecycle.delivering: + if self._lifecycle.delivering or _lifecycle.execution_context() is not None: wait_for_executor = False if self._idle_heartbeat: @@ -2385,12 +2407,12 @@ def _on_up_prepared(self, host, generation): if futures: self._lifecycle.collect_futures( futures, partial(self._on_up_pools_completed, host, - generation=generation)) + generation=generation), host=host) except Exception: log.exception("Unexpected failure handling node %s being marked up:", host) for future in futures: future.cancel() - + self._on_up_preparation_failed(host, generation) raise else: if not futures: @@ -2471,7 +2493,7 @@ def _start_reconnector(self, host, is_host_addition, return reconnector @run_in_executor - @_lifecycle.event + @_lifecycle.negative_event def on_down_potentially_blocking(self, host, is_host_addition, down_event_generation, on_add_reconnection=None): @@ -2843,9 +2865,9 @@ def pools_completed(futures_results): addition_token=addition_token) return - self._lifecycle.collect_futures(futures, pools_completed) + self._lifecycle.collect_futures(futures, pools_completed, host=host) - @_lifecycle.event + @_lifecycle.observation_event('readiness') def _finalize_add(self, host, set_up=True, generation=None, addition_token=None, advance_health=True): if not self._lifecycle.mark_ready( @@ -3135,6 +3157,7 @@ def get_control_connection_host(self): connection = self.control_connection._connection return self.control_connection._get_host_for_connection(connection) + @_lifecycle.synchronous_api def refresh_schema_metadata(self, max_schema_agreement_wait=None): """ Synchronously refresh all schema metadata. @@ -3148,9 +3171,11 @@ def refresh_schema_metadata(self, max_schema_agreement_wait=None): An Exception is raised if schema refresh fails for any reason. """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("Schema metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_keyspace_metadata(self, keyspace, max_schema_agreement_wait=None): """ Synchronously refresh keyspace metadata. This applies to keyspace-level information such as replication @@ -3158,10 +3183,12 @@ def refresh_keyspace_metadata(self, keyspace, max_schema_agreement_wait=None): See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.KEYSPACE, keyspace=keyspace, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("Keyspace metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None): """ Synchronously refresh table metadata. This applies to a table, and any triggers or indexes attached @@ -3169,30 +3196,36 @@ def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TABLE, keyspace=keyspace, table=table, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("Table metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_materialized_view_metadata(self, keyspace, view, max_schema_agreement_wait=None): """ Synchronously refresh materialized view metadata. See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TABLE, keyspace=keyspace, table=view, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("View metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_user_type_metadata(self, keyspace, user_type, max_schema_agreement_wait=None): """ Synchronously refresh user defined type metadata. See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TYPE, keyspace=keyspace, type=user_type, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("User Type metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_user_function_metadata(self, keyspace, function, max_schema_agreement_wait=None): """ Synchronously refresh user defined function metadata. @@ -3201,10 +3234,12 @@ def refresh_user_function_metadata(self, keyspace, function, max_schema_agreemen See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.FUNCTION, keyspace=keyspace, function=function, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("User Function metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreement_wait=None): """ Synchronously refresh user defined aggregate metadata. @@ -3213,10 +3248,12 @@ def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreem See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.AGGREGATE, keyspace=keyspace, aggregate=aggregate, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("User Aggregate metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_nodes(self, force_token_rebuild=False): """ Synchronously publish refreshed node-list and token metadata. @@ -3231,6 +3268,7 @@ def refresh_nodes(self, force_token_rebuild=False): An Exception is raised if node refresh fails for any reason. """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_node_list_and_token_map(force_token_rebuild): raise DriverException("Node list was not refreshed. See log for details.") @@ -3561,12 +3599,15 @@ def default_serial_consistency_level(self, cl): _pool_generations = None _pool_intents = None _pool_creation_work = None + # Bare Session.__new__ fixtures have no admitted runtime lifetime. + _management_runtime = None _profile_manager = None _metrics = None _request_init_callbacks = None def __init__(self, cluster, hosts, keyspace=None): self.cluster = cluster + self._management_runtime = cluster._lifecycle._ensure_runtime() self.hosts = hosts self.keyspace = keyspace @@ -4233,11 +4274,20 @@ def shutdown(self): for future in initial_connect_futures if future in pool_creation_work) + cleanup = partial(self._finish_shutdown, initial_connect_work, + initial_connect_futures) + if (_lifecycle.managed_context(self.cluster) and + not _lifecycle.performing_cleanup()): + self._shutdown_future = _lifecycle.defer_cleanup(self.cluster, cleanup) + return + cleanup() + + def _finish_shutdown(self, initial_connect_work, initial_connect_futures): # PYTHON-673. If shutdown was called shortly after session init, avoid # a race by cancelling any initial connection attempts haven't started, # then blocking on any that have. for future in initial_connect_work: - future.cancel() + _cancel_and_notify(future) wait_futures(initial_connect_work) for future in initial_connect_futures: @@ -4284,21 +4334,25 @@ def _pool_intents_locked(self): return pool_intents def _submit_pool_creation_work(self, completion, fn, *args, **kwargs): - """Submit pool creation and retain its work behind its public proxy.""" - with self._lock: - work_future = self.submit(fn, *args, **kwargs) - add_done_callback = ( - getattr(work_future, 'add_done_callback', None) - if work_future is not None else None) + """Retain one charged physical result behind each legacy pool proxy.""" + def work(): + with _lifecycle.physical_work_context(): + return fn(*args, **kwargs) + + with _lifecycle.compatibility_work(self._management_runtime) as attach: + with self._lock: + work_future = self.submit(work) + add_done_callback = ( + getattr(work_future, 'add_done_callback', None) + if work_future is not None else None) + if callable(add_done_callback): + work_items = self._pool_creation_work + if work_items is None: + work_items = self._pool_creation_work = {} + work_items[completion] = work_future + attach(work_future) if callable(add_done_callback): - work = self._pool_creation_work - if work is None: - work = self._pool_creation_work = {} - work[completion] = work_future - - if callable(add_done_callback): - add_done_callback(partial( - self._pool_creation_work_done, completion)) + add_done_callback(partial(self._pool_creation_work_done, completion)) return work_future def _pool_creation_work_done(self, completion, work_future): @@ -5064,6 +5118,7 @@ def on_remove(self, host, trigger_reconciliation=True): else: self.remove_pool(host) + @_lifecycle.synchronous_api def set_keyspace(self, keyspace): """ Set the default keyspace for all queries made through this Session. @@ -5073,6 +5128,7 @@ def set_keyspace(self, keyspace): :class:`.InvalidRequest` if another Session is using the shared connection or another keyspace change is in progress. """ + _lifecycle.assert_synchronous_api(self.cluster.connection_class) self.execute('USE %s' % (protect_name(keyspace),)) def _set_keyspace_for_all_pools(self, keyspace, callback): diff --git a/tests/unit/test_cluster.py b/tests/unit/test_cluster.py index 063020ca6b..716f285264 100644 --- a/tests/unit/test_cluster.py +++ b/tests/unit/test_cluster.py @@ -2731,7 +2731,7 @@ def test_initialized_cluster_connect_after_scheduler_cleanup_is_rejected(self): allow_control_connection_query_fallback= ControlConnectionQueryFallback.SkipPoolCreation, ) -with patch.object(cluster.connection_class, 'initialize_reactor'), \ +with patch.object(cluster.connection_class, 'initialize_reactor', wraps=cluster.connection_class.initialize_reactor), \ patch.object(cluster.control_connection, 'connect'), \ patch.object(cluster, '_populate_hosts'), \ patch.object(cluster.profile_manager, 'check_supported'): @@ -2792,7 +2792,7 @@ def test_late_connect_does_not_join_reentrant_executor_shutdown(self): allow_control_connection_query_fallback= ControlConnectionQueryFallback.SkipPoolCreation, ) -with patch.object(cluster.connection_class, 'initialize_reactor'), \ +with patch.object(cluster.connection_class, 'initialize_reactor', wraps=cluster.connection_class.initialize_reactor), \ patch.object(cluster.control_connection, 'connect'), \ patch.object(cluster, '_populate_hosts'), \ patch.object(cluster.profile_manager, 'check_supported'): @@ -4130,7 +4130,7 @@ def shutdown_before_submit(fn, *args, **kwargs): cluster.sessions = (session, live_session) cluster._finalize_add = Mock() - def collect_futures(futures, callback): + def collect_futures(futures, callback, host=None): callback([future.exception(timeout=5) or future.result(timeout=5) for future in futures]) @@ -6692,7 +6692,7 @@ class _ResumableConnection(Connection): supports_tls_session_resumption = True -class _NonResumableConnection(Connection): +class _NonResumableConnection(connection_class): supports_tls_session_resumption = False diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 7d2b7e7359..53ef1f9d92 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -47,6 +47,7 @@ ConstantReconnectionPolicy, ExponentialReconnectionPolicy, IdentityTranslator) +from tests import connection_class from tests.unit.utils import new_session_with_pool_state PEER_IP = "foobar" @@ -161,6 +162,7 @@ def remove_host_by_host_id(self, host_id, endpoint=None): class MockCluster(object): + connection_class = connection_class max_schema_agreement_wait = 5 profile_manager = ProfileManager() reconnection_policy = ConstantReconnectionPolicy(2) diff --git a/tests/unit/test_lifecycle_coordination.py b/tests/unit/test_lifecycle_coordination.py index db22525ad4..6c5ae2d72a 100644 --- a/tests/unit/test_lifecycle_coordination.py +++ b/tests/unit/test_lifecycle_coordination.py @@ -13,7 +13,7 @@ # limitations under the License. from concurrent.futures import CancelledError, Future -from threading import Barrier, Event, Lock, Thread, get_ident +from threading import Barrier, Event, Lock, RLock, Thread, get_ident from unittest.mock import Mock, patch import pytest @@ -33,6 +33,23 @@ from tests.unit.utils import new_session_with_pool_state +@pytest.fixture(autouse=True) +def close_test_coordinators(monkeypatch): + coordinators = [] + initialize = LifecycleCoordinator.__init__ + + def tracked(self, *args, **kwargs): + initialize(self, *args, **kwargs) + coordinators.append(self) + + monkeypatch.setattr(LifecycleCoordinator, '__init__', tracked) + yield + for coordinator in coordinators: + coordinator.close() + if coordinator._runtime is not None: + coordinator._runtime.join(5) + + def test_pool_results_wait_for_every_operation_and_preserve_failures(): coordinator = LifecycleCoordinator() callback = Mock() @@ -91,7 +108,7 @@ def hold_drainer(): assert not drainer.is_alive() assert len(observations) == 1 thread_id, results = observations[0] - assert thread_id == drainer.ident + assert thread_id == coordinator._runtime._threads[3].ident assert sorted(results) == list(range(8)) @@ -179,7 +196,8 @@ def finish_cleanup(): session.remove_pool.assert_called_once_with(host) session.update_created_pools.assert_called_once_with() assert observations == [ - ('notification', drainer.ident), ('reconcile', drainer.ident)] + ('notification', cluster._lifecycle._runtime._threads[3].ident), + ('reconcile', cluster._lifecycle._runtime._threads[3].ident)] @pytest.mark.parametrize('completion', ['queued', 'late']) @@ -885,7 +903,7 @@ def shutdown_from_listener(_host): [result], lambda results: cluster._on_up_pools_completed(host, results)) finished.result(timeout=5) assert cluster.is_shutdown - assert shutdown_returned.is_set() + assert shutdown_returned.wait(5) assert cluster._lifecycle.closed assert not host._currently_handling_node_up listener.on_up.assert_called_once_with(host) @@ -2575,6 +2593,7 @@ def take_then_shutdown(replacement): host.get_and_set_reconnection_handler = take_then_shutdown cluster.remove_host(host, refresh_nodes=False) + cluster._lifecycle._runtime.join(5) handler.cancel.assert_called_once_with() @@ -2874,3 +2893,643 @@ def stop(): if shutdown is not None: assert not shutdown.is_alive() assert not shutdown_errors + + +@pytest.mark.parametrize('transition', ['down', 'remove']) +@pytest.mark.parametrize('backlog', ['io', 'callbacks']) +def test_negative_producer_survives_full_ordinary_admission(topology, transition, backlog): + from cassandra._management import ManagementOverloaded + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + cluster._discount_down_events = False + cluster._start_reconnector = Mock() + cluster.control_connection.on_down = Mock() + cluster.control_connection.on_remove = Mock() + cluster._lifecycle._ensure_runtime() + manager = cluster._lifecycle._runtime + manager.limit = 2 + entered, release = Event(), Event() + listener = Mock() + callback = listener.on_down if transition == 'down' else listener.on_remove + cluster.register_listener(listener) + physical = [] + if backlog == 'io': + class ControlledExecutor: + def submit(self, function): + future = Future() + physical.append(future) + entered.set() + return future + executor = ControlledExecutor() + futures = [manager.submit(lambda r: manager.work(r, executor, lambda: 1)) + for _ in range(2)] + assert entered.wait(5) + # C processes both admission envelopes before the transition is sent. + # Polling native record state uses a condition, not a timing delay. + with manager._condition: + assert manager._condition.wait_for( + lambda: all(f._record.physical for f in futures), timeout=5) + else: + def blocked_listener(): + entered.set() + assert release.wait(5) + first = manager.notify(blocked_listener) + assert entered.wait(5) + second = manager.notify(lambda: None) + futures = [first, second] + try: + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None) + if transition == 'down': + assert cluster.on_down(host, False) + assert host.is_up is False + else: + cluster.remove_host(host, refresh_nodes=False) + assert host._is_removed + assert cluster.metadata.get_host_by_host_id(host.host_id) is None + callback.assert_not_called() + assert len(manager._health) == 1 + finally: + release.set() + for future in physical: + future.set_result(None) + # An explicit C observation runs after native negative slot consumption. + # Completion is signalled by the actual application callback. + observed = Event() + callback.side_effect = lambda *args: observed.set() + if callback.call_count: + observed.set() + assert observed.wait(5) + callback.assert_called_once_with(host) + for future in futures: + future.result(5) + + +def test_deferred_observation_replacement_keeps_latest_causal_order(topology): + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + entered, release, complete = Event(), Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + observed = [] + try: + cluster._lifecycle.submit_readiness( + host, lambda *args: observed.append('superseded-readiness')) + cluster._lifecycle.update_location(host, 'dc2', 'rack2') + cluster._lifecycle.submit_readiness( + host, lambda *args: (observed.append((host.datacenter, host.rack)), complete.set())) + assert len(cluster._lifecycle._pending_observations) == 1 + assert list(cluster._lifecycle._pending_observations[id(host)][1]) == ['location', 'readiness'] + finally: + release.set() + active.result(5) + assert complete.wait(5) + assert observed == [('dc2', 'rack2')] + + +def test_deferred_positive_observation_follows_admitted_events(topology): + cluster, _ = topology + coordinator = cluster._lifecycle + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = coordinator._ensure_runtime() + manager.limit = 2 + entered, release, completed = Event(), Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + order = [] + try: + with coordinator.transaction(drain=False): + coordinator.submit(lambda: order.append('admitted')) + coordinator.submit_observation( + host, 'readiness', lambda: (order.append('deferred'), completed.set())) + assert manager.counts()['ordinary'] == 2 + assert id(host) in coordinator._pending_observations + release.set() + coordinator.drain() + assert completed.wait(5) + assert order == ['admitted', 'deferred'] + active.result(5) + finally: + release.set() + + +@pytest.mark.parametrize('running', [False, True]) +def test_up_pool_admission_failure_releases_claim_and_cancels_started_futures(topology, running): + from cassandra._management import ManagementOverloaded, compatibility_work + cluster, _ = topology + coordinator = cluster._lifecycle + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + host.set_down() + generation = getattr(host, '_readiness_generation', 0) + manager = coordinator._ensure_runtime() + manager.limit = 1 + entered, release = Event(), Event() + occupied = manager.submit(lambda record: (entered.set(), release.wait(5))) + assert entered.wait(5) + started = Future() + if running: + assert started.set_running_or_notify_cancel() + first, second = Mock(), Mock() + first.add_or_renew_pool.return_value = started + def submit_pool(*args, **kwargs): + with compatibility_work(manager): + pytest.fail('saturated pool work was admitted') + second.add_or_renew_pool.side_effect = submit_pool + cluster.sessions = [first, second] + cluster._cleanup_failed_on_up_handling = Mock() + try: + assert coordinator.begin_up(host, generation) + with pytest.raises(ManagementOverloaded): + cluster._on_up_prepared(host, generation) + assert started.cancelled() is not running + cluster._cleanup_failed_on_up_handling.assert_called_once_with(host) + assert not host._currently_handling_node_up + if running: + # Cancellation cannot own a physical job that already started. + # Its native result remains available after failed-UP cleanup. + started.set_result(True) + assert started.result(0) is True + assert not host._currently_handling_node_up + assert coordinator.begin_up(host, generation), 'later UP was stranded' + coordinator.finish_up(host, generation) + finally: + release.set() + occupied.result(5) + + +def test_transaction_readiness_never_waits_inside_notification_handler(monkeypatch): + from cassandra import _lifecycle + from cassandra._management import execution_context + class GuardedEvent(Event): + def wait(self, timeout=None): + assert execution_context() != 'callback', 'handler waited for transaction caller' + return super().wait(timeout) + coordinator = LifecycleCoordinator() + manager = coordinator._ensure_runtime() + monkeypatch.setattr(_lifecycle, 'Event', GuardedEvent) + delivered = [] + with coordinator.transaction(drain=False): + coordinator.submit(lambda: delivered.append(True)) + # C must remain usable while the transaction caller owns publication. + assert manager.submit(lambda record: 23).result(5) == 23 + drainer = Thread(target=coordinator.drain) + drainer.start() + try: + drainer.join(2) + assert not drainer.is_alive(), 'transaction readiness stranded drain' + assert delivered == [True] + finally: + coordinator.close() + drainer.join(5) + + +def test_unknown_host_overload_is_explicit(topology): + from cassandra._management import ManagementOverloaded + from cassandra.pool import Host + from cassandra.policies import SimpleConvictionPolicy + cluster, _ = topology + coordinator = cluster._lifecycle + from uuid import uuid4 + unknown = Host(DefaultEndPoint('127.0.0.99'), SimpleConvictionPolicy, host_id=uuid4()) + manager = coordinator._ensure_runtime() + manager.limit = 1 + entered, release = Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + callback = Mock() + try: + with pytest.raises(ManagementOverloaded): + coordinator.submit_observation(unknown, 'pool-results', callback, [True]) + callback.assert_not_called() + assert not coordinator._pending_observations + assert manager._unknown_refresh is None + finally: + release.set() + active.result(5) + + +@pytest.mark.parametrize('race', [False, True]) +def test_coordinator_close_cannot_leave_first_runtime_open(monkeypatch, race): + from cassandra import _lifecycle + from cassandra._management import ManagementClosed + coordinator = LifecycleCoordinator() + entered, release, closing, closure_claimed = Event(), Event(), Event(), Event() + original_idle_set = coordinator._idle.set + def idle_set(): + if coordinator._closed: + closure_claimed.set() + original_idle_set() + coordinator._idle.set = idle_set + original_channel = _lifecycle.ReactorChannel + created, outcomes = [], [] + def channel(*args, **kwargs): + if race: + entered.set() + assert release.wait(5) + return original_channel(*args, **kwargs) + original_start = _lifecycle.ManagementRuntime.start + def start(runtime): + if runtime not in created: + created.append(runtime) + return original_start(runtime) + monkeypatch.setattr(_lifecycle, 'ReactorChannel', channel) + monkeypatch.setattr(_lifecycle.ManagementRuntime, 'start', start) + def first_use(): + try: + coordinator._ensure_runtime() + outcomes.append('started') + except ManagementClosed: + outcomes.append('closed') + if not race: + coordinator.close() + try: + first_use() + finally: + for runtime in created: + runtime.close() + runtime.join(5) + assert not created + assert outcomes == ['closed'] + return + starter = Thread(target=first_use) + stopper = Thread(target=lambda: (coordinator.close(), closing.set())) + starter.start() + try: + assert entered.wait(5) + stopper.start() + # close claims closure before waiting for provisional installation. + assert closure_claimed.wait(5) + release.set() + starter.join(5) + stopper.join(5) + assert not starter.is_alive() and not stopper.is_alive() + assert closing.is_set() + assert outcomes == ['closed'] + assert len(created) == 1 + created[0].join(5) + assert created[0].finalized.is_set() + assert created[0].closed.is_set() + assert coordinator._runtime is None + finally: + release.set() + starter.join(5) + if stopper.ident is not None: + stopper.join(5) + for runtime in created: + runtime.close() + runtime.join(5) + + +def test_runtime_closure_releases_waiting_lifecycle_drain(): + coordinator = LifecycleCoordinator() + manager = coordinator._ensure_runtime() + entered, release, waiting, finished = Event(), Event(), Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + with coordinator.transaction(drain=False): + coordinator.submit(lambda: pytest.fail('closed notification ran')) + original_wait = coordinator._idle.wait + def observed_wait(timeout=None): + waiting.set() + return original_wait(timeout) + coordinator._idle.wait = observed_wait + drainer = Thread(target=lambda: (coordinator.drain(), finished.set())) + drainer.start() + try: + assert waiting.wait(5) + manager.close() + assert finished.wait(2), 'runtime closure stranded drain' + assert not coordinator._events + assert coordinator._idle.is_set() + finally: + release.set() + coordinator.close() + drainer.join(5) + from cassandra._management import ManagementClosed + with pytest.raises(ManagementClosed): + active.result(5) + + +def test_failed_async_handoff_releases_concurrent_drain(): + coordinator = LifecycleCoordinator() + manager = coordinator._ensure_runtime() + entered, reject, waiting, finished = Event(), Event(), Event(), Event() + callback_entered, release_callback = Event(), Event() + active = manager.notify(lambda: (callback_entered.set(), release_callback.wait(5))) + assert callback_entered.wait(5) + class RejectingExecutor: + def submit(self, callback): + entered.set() + assert reject.wait(5) + raise RuntimeError('executor rejected handoff') + submitter = Thread(target=lambda: coordinator.submit_async( + RejectingExecutor(), lambda: pytest.fail('withdrawn event ran'))) + original_wait = coordinator._idle.wait + def observed_wait(timeout=None): + waiting.set() + return original_wait(timeout) + coordinator._idle.wait = observed_wait + submitter.start() + drainer = Thread(target=lambda: (coordinator.drain(), finished.set())) + try: + assert entered.wait(5) + # Keep callback delivery behind an actual active notification while + # the other producer releases this event's transaction barrier. + # The callback lane must not consume the event before withdrawal. + drainer.start() + assert waiting.wait(5) + reject.set() + submitter.join(5) + assert finished.wait(2), 'withdrawn last event stranded drain' + assert not coordinator._events + assert coordinator._idle.is_set() + assert not coordinator._draining + finally: + reject.set() + release_callback.set() + active.result(5) + coordinator.close() + submitter.join(5) + if drainer.ident is not None: + drainer.join(5) + + +def test_async_submission_from_active_delivery_reuses_drainer(): + coordinator = LifecycleCoordinator() + executor = Mock() + order = [] + def callback(): + coordinator.submit_async(executor, lambda: order.append('inner')) + order.append('outer') + submitter = Thread(target=lambda: coordinator.submit(callback)) + submitter.start() + try: + submitter.join(2) + assert not submitter.is_alive(), 'nested asynchronous admission lost its active drainer' + assert order == ['outer', 'inner'] + executor.submit.assert_called_once_with(coordinator.drain) + assert not coordinator._events + finally: + coordinator.close() + submitter.join(5) + + +@pytest.mark.parametrize('api', ['connect', 'refresh', 'keyspace']) +def test_actual_listener_sync_wrappers_finish_at_full_ordinary_capacity(topology, api): + cluster, connection = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + cluster._is_setup = True + cluster.executor.shutdown() + from concurrent.futures import ThreadPoolExecutor + cluster.executor = ThreadPoolExecutor(1) + session = new_session_with_pool_state() + session.cluster = cluster + session.execute = lambda statement: cluster.executor.submit(lambda: None).result(5) + session._initial_connect_futures = set() + session.shutdown = Mock() + session._default_consistency_level = None + deferred = Event() + def create(keyspace): + cluster._lifecycle.submit_readiness(host, lambda *args: deferred.set()) + return session + cluster._new_session = create + entered, release, completed = Event(), Event(), Event() + results = [] + def listener(): + entered.set() + assert release.wait(5) + if api == 'connect': + results.append(cluster.connect(wait_for_all_pools=True) is session) + elif api == 'refresh': + # Actual topology reader/publisher, with its socket response mocked. + connection.peer_results[1][1][3:5] = ['new-dc', 'new-rack'] + connection.wait_for_responses.return_value = _node_meta_results( + connection.local_results, connection.peer_results) + cluster.refresh_nodes() + results.append(True) + else: + session.set_keyspace('ks') + results.append(True) + completed.set() + notification = manager.notify(listener) + assert entered.wait(5) + assert manager._credits == 1 + release.set() + assert completed.wait(5) + notification.result(5) + assert results == [True] + assert manager._reentry_active is False + if api == 'connect': + assert deferred.wait(5) + + +def test_actual_sync_refresh_waits_for_admission_before_side_effects(topology): + cluster, _ = topology + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + gate = Event() + held = manager.submit(lambda record: gate.wait(5)) + started, finished = Event(), Event() + control = cluster.control_connection.refresh_node_list_and_token_map + cluster.control_connection.refresh_node_list_and_token_map = Mock(return_value=True) + def caller(): + started.set() + cluster.refresh_nodes() + finished.set() + thread = Thread(target=caller) + thread.start() + try: + assert started.wait(5) + assert not finished.is_set() + cluster.control_connection.refresh_node_list_and_token_map.assert_not_called() + # Admission waits without retaining locks required by owner publication. + with cluster._lifecycle.transaction(drain=False), cluster._lock: + assert manager._credits == 1 + gate.set() + held.result(5) + assert finished.wait(5) + cluster.control_connection.refresh_node_list_and_token_map.assert_called_once_with(False) + finally: + gate.set() + thread.join(5) + cluster.control_connection.refresh_node_list_and_token_map = control + + +@pytest.mark.parametrize('raises', [False, True]) +def test_listener_policy_reentry_releases_native_phase_before_nested_refresh(topology, raises): + from cassandra.cluster import EXEC_PROFILE_DEFAULT + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + cluster._is_setup = True + observed = [] + class ReentrantPolicy(RoundRobinPolicy): + def distance(self, host): + from cassandra._management import execution_context + observed.append((execution_context(), manager._reentry_active)) + cluster.refresh_nodes() + if raises: + raise ValueError('policy rejected candidate') + return HostDistance.LOCAL + policy = ReentrantPolicy() + profile = ProfileManager() + profile.profiles[EXEC_PROFILE_DEFAULT] = ExecutionProfile(load_balancing_policy=policy) + cluster.profile_manager = profile + session = new_session_with_pool_state() + session.cluster = cluster + session.shutdown = Mock() + session._initial_connect_futures = set() + def create(keyspace): + cluster.profile_manager.distance(host) + return session + cluster._new_session = create + complete = Event() + def listener(): + if raises: + with pytest.raises(ValueError, match='policy rejected'): + cluster.connect() + else: + assert cluster.connect() is session + complete.set() + notification = manager.notify(listener) + assert complete.wait(5) + notification.result(5) + assert observed == [('callback', False)] + assert manager._reentry_active is False + + +def test_listener_connect_windows_real_pool_fanout_with_one_physical_worker(topology): + from concurrent.futures import ThreadPoolExecutor + from contextlib import nullcontext + from uuid import UUID + cluster, _ = topology + # Keep real Cluster.connect and Session construction; replace socket/pool + # construction only. 128 pools exceed the dedicated 64-result window. + cluster.metadata._hosts = {} + cluster.metadata._host_id_by_endpoint = {} + for number in range(128): + host, _ = cluster.add_host(DefaultEndPoint('127.0.0.%d' % (number + 1)), + signal=False, host_id=UUID(int=number + 10)) + host.set_up() + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + cluster._is_setup = True + cluster.executor.shutdown() + cluster.executor = ThreadPoolExecutor(1) + entered, release, connected = Event(), Event(), Event() + pools, sessions = [], [] + def create_pool(host, distance, session, **kwargs): + if not pools: + entered.set() + assert release.wait(5) + pool = Mock(host=host, host_distance=distance, is_shutdown=False, + _keyspace=None, _membership_lock=RLock()) + pool._lock_usable_connection.return_value = nullcontext(True) + pool.shutdown.side_effect = lambda: setattr(pool, 'is_shutdown', True) + pools.append(pool) + return pool + def listener(): + sessions.append(cluster.connect(wait_for_all_pools=True)) + connected.set() + with patch('cassandra.cluster.HostConnection', side_effect=create_pool): + notification = manager.notify(listener) + try: + assert entered.wait(5) + with manager._condition: + assert manager._condition.wait_for(lambda: manager._reentry_credits == 64, timeout=5) + assert manager._credits == 1 + assert manager._reentry_active + assert not connected.is_set() + finally: + release.set() + assert connected.wait(5) + notification.result(5) + assert len(pools) == len(sessions[0]._pools) == 128 + assert manager._reentry_credits == 0 + assert not manager._reentry_active + + +@pytest.mark.parametrize('context', ['cluster', 'pool', 'io']) +def test_managed_cluster_shutdown_defers_initial_pool_wait_and_retains_cleanup(topology, context): + from concurrent.futures import ThreadPoolExecutor + from contextlib import nullcontext + from cassandra._management import ManagementClosed + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + cluster.executor.shutdown() + cluster.executor = ThreadPoolExecutor(1) + session = new_session_with_pool_state() + session.cluster = cluster + session._management_runtime = manager + session._profile_manager = cluster.profile_manager + session._initial_connect_futures = set() + session.keyspace = None + session.is_shutdown = False + cluster.sessions.add(session) + entered, release, returned = Event(), Event(), Event() + candidate = Mock(host=host, is_shutdown=False, _keyspace=None, + _membership_lock=RLock()) + candidate._lock_usable_connection.return_value = nullcontext(True) + def construct(*args, **kwargs): + entered.set() + assert release.wait(5) + return candidate + with patch('cassandra.cluster.HostConnection', side_effect=construct): + initial = session.add_or_renew_pool(host, False) + session._initial_connect_futures.add(initial) + assert entered.wait(5) + def shutdown(*args): + cluster.shutdown() + returned.set() + try: + if context == 'io': + cluster.connection_class._management_dispatch(shutdown) + else: + manager.submit(shutdown, pool=object() if context == 'pool' else None) + assert returned.wait(5), 'shutdown waited on invoking execution context' + assert cluster.is_shutdown + assert manager.closed.is_set() + assert not manager.finalized.is_set() + assert manager._cleanup_obligations == 1 + finally: + release.set() + manager.join(5) + assert session.is_shutdown + candidate.shutdown.assert_called_once_with() + assert cluster.executor._shutdown + assert manager._cleanup_obligations == 0 + assert not manager._records + + +def test_managed_close_submission_rejection_is_recorded_without_inline_cleanup(topology): + cluster, _ = topology + manager = cluster._lifecycle._ensure_runtime() + original_submit = cluster.executor.submit + rejected, returned = RuntimeError('cleanup executor rejected'), Event() + effects = [] + cluster.executor.submit = Mock(side_effect=rejected) + def handler(record): + try: + with pytest.raises(RuntimeError, match='cleanup executor rejected'): + cluster.shutdown() + effects.append('returned') + finally: + returned.set() + try: + manager.submit(handler) + assert returned.wait(5) + manager.join(5) + assert manager.cleanup_error is rejected + assert effects == ['returned'] + assert manager._cleanup_obligations == 0 + # Transport cleanup has not been falsely reported as run. + assert not cluster.scheduler.is_shutdown + finally: + cluster.executor.submit = original_submit + cluster._shutdown(wait_for_executor=True) diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py index bcb8539d44..768258cad2 100644 --- a/tests/unit/test_management_reactors.py +++ b/tests/unit/test_management_reactors.py @@ -4,12 +4,15 @@ import importlib from concurrent.futures import CancelledError, Future from threading import Event, Thread, current_thread, get_ident +from types import SimpleNamespace +from unittest.mock import Mock import pytest from cassandra import DependencyException from cassandra._management import ( - ManagementRuntime, ReactorChannel, execution_context) + ManagementContextError, ManagementRuntime, ReactorChannel, execution_context) +from cassandra.cluster import Cluster, Session @pytest.fixture(autouse=True) @@ -40,7 +43,7 @@ def connection_class(request): 'asyncore': 'AsyncoreConnection'}[request.param]) -def test_idle_reactor_completion_runs_on_io_thread(connection_class): +def test_idle_reactor_completion_and_raw_io_sync_guard(connection_class): channel = ReactorChannel(connection_class) for _ in range(3): finished = Event() @@ -48,6 +51,17 @@ def test_idle_reactor_completion_runs_on_io_thread(connection_class): def on_io(): assert execution_context() is None # Ordinary ResponseFuture context. assert connection_class._management_in_context() + cluster = SimpleNamespace(connection_class=connection_class, + _lifecycle=None, control_connection=Mock()) + session = SimpleNamespace(cluster=cluster, execute=Mock()) + for call in (lambda: Cluster.connect(cluster), + lambda: Cluster.refresh_nodes(cluster), + lambda: Cluster.refresh_schema_metadata(cluster), + lambda: Session.set_keyspace(session, 'ks')): + with pytest.raises(ManagementContextError): + call() + cluster.control_connection.assert_not_called() + session.execute.assert_not_called() observations.append(get_ident()) finished.set() channel.dispatch(on_io) @@ -707,3 +721,34 @@ def checked_factory(): peer.close() os.close(read_fd) os.waitpid(pid, 0) + + +def test_channel_startup_failure_unwinds_all_management_threads(): + from cassandra._lifecycle import LifecycleCoordinator + class FailingConnection: + @classmethod + def initialize_reactor(cls): + pass + @classmethod + def _management_in_context(cls): + return False + @classmethod + def _management_dispatch(cls, callback): + raise RuntimeError('bridge admission failed') + cluster = SimpleNamespace(connection_class=FailingConnection, is_shutdown=False) + # SimpleNamespace is not weak-referenceable; use an ordinary test lifetime. + class Lifetime: + pass + lifetime = Lifetime() + lifetime.connection_class = FailingConnection + lifetime.is_shutdown = False + coordinator = LifecycleCoordinator(lifetime) + import threading + before = {thread.ident for thread in threading.enumerate() + if thread.name.startswith('cassandra-management-')} + with pytest.raises(RuntimeError, match='bridge admission failed'): + coordinator._ensure_runtime() + assert coordinator._runtime is None + after = {thread.ident for thread in threading.enumerate() + if thread.name.startswith('cassandra-management-')} + assert after == before diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index 0bcd94a651..4a16b9881c 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -835,6 +835,54 @@ def result_queued(): assert completion.ident not in {thread.ident for thread in manager._threads} +@pytest.mark.parametrize('health_notification', [False, True]) +def test_detached_coordinator_does_not_keep_runtime_threads_alive( + health_notification): + import gc + import weakref + from functools import partial + from cassandra._lifecycle import LifecycleCoordinator + coordinator = LifecycleCoordinator() + coordinator.submit(lambda: None) + manager = coordinator._runtime + reference = weakref.ref(coordinator) + release = Event() + try: + if health_notification: + entered, delivered = Event(), Event() + def hold(): + entered.set() + assert release.wait(5) + def completed(callback): + callback() + delivered.set() + manager.limit = 1 + manager.notify(hold, wait=True) + assert entered.wait(5) + incarnation = object() + manager.register_health(incarnation) + manager.observe_negative( + incarnation, None, partial(completed, coordinator.drain)) + release.set() + assert delivered.wait(5) + # Lifecycle idle is signalled inside its callback, before the native + # notification reservation finishes. A FIFO fence passes the earlier + # C admission and L callback frames before testing idle retention. + manager.notify(lambda: None, wait=True).result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: not manager._records, 5) + del coordinator + gc.collect() + assert reference() is None + manager.join(5) + assert all(not thread.is_alive() for thread in manager._threads) + finally: + release.set() + manager.close() + manager.join(5) + + def test_synchronous_admission_waits_without_allocating_pending_records( runtime, monkeypatch): manager, dispatch = runtime