From d76d975db8d484fa706eeb96a9ee8a0cd6607995 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Nov 2026 09:13:54 -0400 Subject: [PATCH 1/3] management: add bounded native runtime Provide serial cluster ownership, fair pool runners and a callback lane with bounded admission so management work can progress independently of slow listeners and completion callbacks. Keep physical work on existing executors and reserve its result delivery before submission. Charge coalesced waiters separately, retain ordinary callback capacity until delivery, and release native reentry capacity after completion. Yield the reactor during caller-to-owner callback handoff, then resume its charged batch when the competing drainer returns. Keep terminal publication atomic while scanning shared callback batches outside future conditions, so standard bulk waits cannot invert their lock order. Preserve cancellation isolation, guaranteed close handles, late-result disposal and fork rejection before inherited locks. Reject mixed bulk waits inside their cleanup context so earlier fresh locks unwind after fork. Guard standard Future waits against owner self-waits without changing concurrent.futures globally. Cover overload, ordering, callback reentry, shutdown, failure unwinding, resource retention, interrupted admission and cancellation with controlled interleavings. Settle controlled physical futures even when regression assertions fail, preserving the original failure during fixture cleanup. Reactor adapters and existing driver entry points follow in dependent changes. Refs #1257. --- cassandra/_management.py | 1275 +++++++++++++++++++ tests/unit/test_management_runtime.py | 1633 +++++++++++++++++++++++++ 2 files changed, 2908 insertions(+) create mode 100644 cassandra/_management.py create mode 100644 tests/unit/test_management_runtime.py diff --git a/cassandra/_management.py b/cassandra/_management.py new file mode 100644 index 0000000000..5bdd5f94e3 --- /dev/null +++ b/cassandra/_management.py @@ -0,0 +1,1275 @@ +# Copyright ScyllaDB, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0 + +"""Bounded management execution contexts. + +No query-path work belongs here. + +Owners handle short, nonblocking messages. Physical work belongs to +an existing executor and lifecycle notifications to the serial callback +lane. Charged future callback batches run on the selected reactor. After +a batch retires, external registrations may drain through the same +serialized adapter on their caller. C/P attach to a current charged +batch. State not yet migrated still requires its original publication +locks. +""" + +from collections import deque +from concurrent.futures import Future, TimeoutError +from contextlib import contextmanager +from dataclasses import dataclass +from functools import wraps +from itertools import chain +import logging +import os +from threading import Condition, Event, Lock, Thread, current_thread, local +import time + +from cassandra import DriverException + +log = logging.getLogger(__name__) +_context = local() +_PENDING = object() + + +class ManagementClosed(DriverException): + """The original management lifetime has ended.""" + + +class ManagementSuperseded(DriverException): + """The operation was superseded without losing its waiters.""" + + +class ManagementOverloaded(DriverException): + """No reservation is available for a nonblocking submission.""" + + +class ManagementDeliveryFailed(DriverException): + """Delivery failed while native outcomes remain terminal.""" + + +class ManagementContextError(DriverException): + """A blocking management call would wait on its own context.""" + + +def execution_context(): + return getattr(_context, 'kind', None) + + +def assert_can_wait(): + if execution_context() in ('cluster', 'pool', 'bridge'): + raise ManagementContextError( + 'Blocking management call on an owner or reactor') + + +def _future_access(method): + @wraps(method) + def access(self, *args, **kwargs): + self._record.runtime._check_pid() + previous = getattr(_context, 'future', None) + _context.future = self + try: + return method(self, *args, **kwargs) + finally: + _context.future = previous + return access + + +class _FutureCondition(Condition): + """Keep bulk-wait rejection inside its cleanup context.""" + + def __init__(self, runtime): + super().__init__() + self._runtime = runtime + self._acquire = self.acquire + self._release = self.release + self.acquire = self._checked_acquire + self.release = self._checked_release + + def _checked_acquire(self, *args, **kwargs): + # Stdlib bulk acquisition does not unwind an acquisition failure. + # Skip inherited locks; the Future state guard rejects inside the + # bulk context, whose exit then releases earlier fresh locks. + if self._runtime._pid != os.getpid(): + return False + return self._acquire(*args, **kwargs) + + def _checked_release(self): + if self._runtime._pid == os.getpid(): + return self._release() + + def __enter__(self): + self._runtime._check_pid() + return super().__enter__() + + +@contextmanager +def _entered(kind, record=None): + previous = (getattr(_context, 'kind', None), + getattr(_context, 'record', None)) + _context.kind, _context.record = kind, record + try: + yield + finally: + _context.kind, _context.record = previous + + +@dataclass(frozen=True) +class Envelope: + operation: int + kind: str + origin: object + lifetime: object + identity: tuple + + +class OperationFuture(Future): + """Native futures complete without running callbacks on owners.""" + + def __init__(self, record): + super().__init__() + self._record = record + self._condition = _FutureCondition(record.runtime) + self._budget = 'ordinary' + self._delivery_error = None + self._callback_cursor = 0 + self._callbacks_running = False + self._callback_waiters = set() + self._delivery_record = record + self._publishing = False + + @property + def _state(self): + self._record.runtime._check_pid() + state = self.__state + if (state not in ('CANCELLED_AND_NOTIFIED', 'FINISHED') and + getattr(_context, 'future', None) is not self): + # Bulk waits read state after acquiring ALL future conditions. + # Reject before waiter installation, where their context unwinds + # every lock. Own nonblocking methods use scoped access above. + assert_can_wait() + return state + + @_state.setter + def _state(self, value): + self.__state = value + + @_future_access + def __repr__(self): + return super().__repr__() + + @_future_access + def done(self): + self._record.runtime._check_pid() + return super().done() + + @_future_access + def running(self): + self._record.runtime._check_pid() + return super().running() + + @_future_access + def cancelled(self): + self._record.runtime._check_pid() + return super().cancelled() + + @_future_access + def result(self, timeout=None): + self._record.runtime._check_pid() + if not self.done(): + assert_can_wait() + return super().result(timeout) + + @_future_access + def exception(self, timeout=None): + self._record.runtime._check_pid() + if not self.done(): + assert_can_wait() + return super().exception(timeout) + + @_future_access + def cancel(self): + self._record.runtime._check_pid() + cancelled = super().cancel() + if cancelled: + # Future.cancel() alone leaves concurrent.futures.wait() asleep. + with self._condition: + if self._state == 'CANCELLED': + self.set_running_or_notify_cancel() + return cancelled + + @_future_access + def set_result(self, result): + return super().set_result(result) + + @_future_access + def set_exception(self, exception): + return super().set_exception(exception) + + @_future_access + def set_running_or_notify_cancel(self): + return super().set_running_or_notify_cancel() + + def _invoke_callbacks(self): + if not self._publishing: + self._record.runtime._schedule_callbacks(self) + + def add_done_callback(self, fn): + self._record.runtime._check_pid() + if self._delivery_error is not None: + raise self._delivery_error + if self._budget in ('reentry', 'aggregate'): + raise ManagementContextError( + 'Private synchronous waiters have no callback batch') + while True: + record = self._delivery_record + owner = execution_context() in ('cluster', 'pool') + target = (getattr(_context, 'record', None) + if record.released and owner else record) + if target is None: + raise ManagementContextError( + 'Owner callback has no reservation') + with target.runtime._condition: + with self._condition: + if record is not self._delivery_record: + continue + if self._delivery_error is not None: + raise self._delivery_error + if record.released and owner: + if target.released or not target.budgets['ordinary']: + raise ManagementContextError( + 'Owner callback has no reservation') + self._delivery_record = target + target.foreign_futures.add(self) + self._done_callbacks.append(fn) + terminal = self.done() + if not terminal: + return + if not target.released: + target.runtime._queue_delivery_locked(target) + return + # Released callbacks share a caller-side drainer; a concurrent + # registrant may leave its callback to the active drainer. + self._drain_callbacks() + return + + def _drain_callbacks(self, record=None): + """Drain the adapter's serialized callback stream outside C/P.""" + with self._condition: + if self._callbacks_running: + if record is not None: + self._callback_waiters.add(record) + return False + self._callbacks_running = True + original = self._delivery_record + while True: + with self._condition: + if (original is not self._delivery_record or + self._callback_cursor == len(self._done_callbacks)): + self._callbacks_running = False + resume, self._callback_waiters = ( + self._callback_waiters, set()) + if original is not self._delivery_record: + # An owner took responsibility for callbacks appended + # during a caller-side drain. Hand them back to its + # reactor batch after this callback has returned. + resume.add(self._delivery_record) + break + callback = self._done_callbacks[self._callback_cursor] + self._callback_cursor += 1 + try: + callback(self) + except BaseException: + log.exception('Error in management completion callback') + for waiting in resume: + waiting.runtime._resume_delivery(waiting) + return True + + +class _DeliveryTicket: + """One cancellable completion ticket per charged record.""" + + def __init__(self, callback): + self._lock = Lock() + self._callback = callback + self._active = False + + def __call__(self): + with self._lock: + callback = self._callback + if callback is None: + return + self._active = True + try: + callback() + finally: + with self._lock: + self._callback = None + self._active = False + + def cancel(self): + with self._lock: + if self._active: + return False + self._callback = None + return True + + +class _Record: + def __init__(self, runtime, envelope, handler, pool, key): + self.runtime, self.envelope, self.handler = runtime, envelope, handler + self.pool, self.key = pool, key + self.budgets = dict.fromkeys(('ordinary', 'reentry', 'aggregate'), 0) + self.private_released = False + self.futures = [] + self.foreign_futures = set() + self.physical = 0 + self.mailbox_pending = 0 + self.started = Event() + self.handler_done = False + self.delivery = False + self.delivery_ticket = None + self.delivery_again = False + self.notification = False + self.released = False + self.dispatched = False + self.result = None + self.error = None + + +class _LifetimeHandle: + def __init__(self, runtime): + self.runtime = runtime + self.closed = Event() + + +class ManagementRuntime: + """One C owner, two P runners and one callback lane per Cluster. + + ``dispatch`` schedules a callable on the shared reactor, never + inline on C/P. It must remain usable until ``finalized``. A + controlled dispatcher may be supplied by unit tests; production + obtains it from its reactor adapter. + + Catastrophic loss of that validated shared service closes + admission and records ``completion_error``. Native terminal + outcomes remain immutable; undeliverable application callbacks + are explicitly marked failed while physical cleanup continues. + Normal startup, idle and close use the adapter's lifetime + guarantee rather than this failure policy. + """ + + def __init__(self, dispatch, limit=1024, pool_limit=64, reentry_window=64, + on_finalized=None): + self._dispatch = dispatch + self.limit, self.pool_limit = limit, pool_limit + self.reentry_window = reentry_window + self.lifetime = object() + self._pid = os.getpid() + self._condition = Condition() + self._cluster = deque() + self._pools = {} + self._ready = deque() + self._active_pools = set() + self._notifications = deque() + self._records = set() + self._keys = {} + self._pool_credits = {} + self._credits = 0 + self._reentry_credits = 0 + self._reentry_active = False + self._sequence = 0 + self._started = False + self._closing = False + self.completion_error = None + self.cleanup_error = None + self._threads = [] + self._exited = 0 + self._cleanup_obligations = 0 + self._finalizing = False + self._on_finalized = on_finalized + self._health = {} + self._unknown_refresh = None + self._lifetimes = set() + self.closed = Event() + self.finalized = Event() + self.high_water = 0 + self.high_waters = dict.fromkeys( + ('ordinary', 'reentry', 'aggregate', 'health', 'close', 'delivery', + 'mailbox'), 0) + + def _observe_counts_locked(self): + counts = { + 'ordinary': self._credits, 'reentry': self._reentry_credits, + 'aggregate': int(self._reentry_active), + 'health': len(self._health), + 'close': len(self._lifetimes) + self._cleanup_obligations, + 'delivery': sum(record.delivery for record in self._records), + 'mailbox': sum(record.mailbox_pending for record in self._records)} + for kind, count in counts.items(): + self.high_waters[kind] = max(self.high_waters[kind], count) + return counts + + def counts(self): + self._check_pid() + with self._condition: + return self._observe_counts_locked() + + def start(self): + self._check_pid() + with self._condition: + self._check_pid() + if self._closing: + raise ManagementClosed('Management lifetime is closed') + if self._started: + return + self._started = True + targets = [('cluster', self._run_cluster), + ('pool', self._run_pool), ('pool', self._run_pool), + ('callback', self._run_notifications)] + try: + for kind, target in targets: + thread = Thread(target=self._run, args=(kind, target), + name='cassandra-management-' + kind, + daemon=True) + thread.start() + self._threads.append(thread) + except BaseException: + self._closing = True + self._condition.notify_all() + # Started threads exit once this startup lock is released. + raise + + def _check_pid(self): + if self._pid != os.getpid(): + raise ManagementClosed( + 'Inherited management runtime cannot run after fork') + + def assert_owner(self, pool=None): + self._check_pid() + record = getattr(_context, 'record', None) + if (execution_context() != ('cluster' if pool is None else 'pool') or + record is None or record.runtime is not self or + record.pool is not pool): + raise ManagementContextError( + 'Management state accessed outside its owner') + + def _capacity(self, pool, budget): + if budget == 'aggregate': + return not self._reentry_active + if budget == 'reentry': + return self._reentry_credits < self.reentry_window + return (self._credits < self.limit and + (pool is None or + self._pool_credits.get(pool, 0) < self.pool_limit)) + + def submit(self, handler, *, pool=None, key=None, identity=(), + kind='operation', origin=None, wait=False, timeout=None, + budget='ordinary', reserve=0): + if budget not in ('ordinary', 'reentry', 'aggregate'): + raise ManagementContextError('Unknown management admission budget') + if reserve not in (0, 1) or (reserve and key is not None): + raise ManagementContextError( + 'Physical fanout requires separate admitted records') + if wait: + assert_can_wait() + self.start() + deadline = None if timeout is None else time.monotonic() + timeout + with self._condition: + self._check_pid() + while not self._closing and not self._capacity(pool, budget): + if not wait: + raise ManagementOverloaded( + 'Management admission capacity exhausted') + remaining = (None if deadline is None + else deadline - time.monotonic()) + if remaining is not None and remaining <= 0: + raise TimeoutError() + self._condition.wait(remaining) + if self._closing: + raise ManagementClosed('Management lifetime is closed') + if budget == 'aggregate': + self._reentry_active = True + elif budget == 'reentry': + self._reentry_credits += 1 + else: + self._credits += 1 + if pool is not None: + self._pool_credits[pool] = ( + self._pool_credits.get(pool, 0) + 1) + self.high_water = max(self.high_water, self._credits) + coalesce = (pool, key) if key is not None else None + record = self._keys.get(coalesce) if coalesce is not None else None + if record is None: + self._sequence += 1 + envelope = Envelope(self._sequence, kind, origin, + self.lifetime, tuple(identity)) + record = _Record(self, envelope, handler, pool, coalesce) + record.physical = reserve + self._records.add(record) + if coalesce is not None: + self._keys[coalesce] = record + self._enqueue_locked(record, lambda: self._handle(record)) + future = OperationFuture(record) + future._budget = budget + record.futures.append(future) + record.budgets[budget] += 1 + self._observe_counts_locked() + self._condition.notify_all() + return future + + def _enqueue_locked(self, record, callback, reserved=False): + record.mailbox_pending += 1 + if record.pool is None: + self._cluster.append((record, callback, reserved)) + else: + queue = self._pools.setdefault(record.pool, deque()) + queue.append((record, callback, reserved)) + if (record.pool not in self._active_pools and + record.pool not in self._ready): + self._ready.append(record.pool) + self._condition.notify_all() + + def _handle(self, record): + if record.handler_done: + return + record.dispatched = True + record.started.set() + try: + outcome = record.handler(record) + if outcome is _PENDING: + return + except BaseException as exc: + self._finish(record, error=exc) + else: + self._finish(record, outcome) + + def _finish(self, record, result=None, error=None): + with self._condition: + if record.handler_done: + return + record.handler_done, record.result, record.error = ( + True, result, error) + record.started.set() + if record.key is not None: + self._keys.pop(record.key, None) + for future in record.futures: + with future._condition: + if not future.done(): + # Keep cancellation and terminal publication atomic, + # but defer the cross-future delivery scan until this + # condition is released. Stdlib bulk waits acquire + # these conditions in a different order. + future._publishing = True + try: + if error is None: + future.set_result(result) + else: + future.set_exception(error) + finally: + future._publishing = False + self._queue_delivery_locked(record) + self._release_locked(record) + + def supersede(self, future): + self._check_pid() + self._finish(future._record, + error=ManagementSuperseded('Operation superseded')) + + def work(self, record, executor, function, *, accept=lambda value: value, + dispose=lambda value: None): + """Reserve result delivery before physical work, on C/P only.""" + self.assert_owner(record.pool) + with self._condition: + if (getattr(_context, 'record', None) is not record or + record.released or record.handler_done or record.physical): + raise ManagementContextError( + 'Physical work requires its current unused ' + 'result reservation') + record.physical = 1 + self._condition.notify_all() + def physical_work(): + with _entered('worker', record): + return function() + + try: + physical = executor.submit(physical_work) + except BaseException as exc: + with self._condition: + record.physical -= 1 + self._finish(record, error=exc) + return _PENDING + + def completed(future): + with _entered('worker', record): + complete(future) + + def complete(future): + try: + value, error = future.result(), None + except BaseException as exc: + value, error = None, exc + with self._condition: + if not self._closing: + self._enqueue_locked( + record, lambda: returned(value, error), reserved=True) + return + # Close retains this obligation without owner readmission. + try: + if error is None: + dispose(value) + except BaseException: + log.exception('Error disposing late management result') + finally: + with self._condition: + record.physical -= 1 + self._release_locked(record) + + def returned(value, error): + try: + if self._closing or record.handler_done: + if error is None: + dispose(value) + return + if error is not None: + self._finish(record, error=error) + else: + try: + accepted = accept(value) + except BaseException as exc: + try: + dispose(value) + finally: + self._finish(record, error=exc) + else: + self._finish(record, accepted) + finally: + with self._condition: + record.physical -= 1 + self._release_locked(record) + physical.add_done_callback(completed) + return _PENDING + + def notify(self, callback, *, wait=False, timeout=None, budget='ordinary'): + self._check_pid() + def admit(record): + with self._condition: + if self._closing or record.handler_done or record.released: + return _PENDING + record.notification = True + self._notifications.append((record, callback)) + self._condition.notify_all() + return _PENDING + return self.submit(admit, wait=wait, timeout=timeout, budget=budget, + kind='notification') + + @contextmanager + def synchronous_reentry(self): + """Reserve one L aggregate and a bounded prerequisite window.""" + self._check_pid() + assert_can_wait() + current = getattr(_context, 'record', None) + if (execution_context() != 'callback' or current is None or + current.runtime is not self): + yield 'ordinary' + return + with self._condition: + if self._reentry_active: + raise ManagementContextError( + 'Recursive native reentry aggregate') + if self._closing: + raise ManagementClosed('Management lifetime is closed') + self._reentry_active = True + try: + yield 'reentry' + finally: + self._check_pid() + with self._condition: + self._reentry_active = False + self._condition.notify_all() + + def register_health(self, incarnation): + self.start() + with self._condition: + if self._closing: + raise ManagementClosed('Management lifetime is closed') + self._health.setdefault( + id(incarnation), (incarnation, [None, None, False])) + self._observe_counts_locked() + + def observe_negative(self, incarnation, apply, notification=None, + removed=False): + """Apply incarnation health despite ordinary admission load.""" + self._check_pid() + with self._condition: + if self._closing: + return False + entry = self._health.get(id(incarnation)) + slot = (entry[1] if entry is not None and entry[0] is incarnation + else None) + if slot is None: + self._unknown_refresh = True + return False + # An already pending removal is terminal for this incarnation. + if slot[2] and not removed: + return True + slot[:] = [apply, notification, removed] + self._condition.notify_all() + return True + + def take_refresh_demand(self): + self.assert_owner() + with self._condition: + demand, self._unknown_refresh = self._unknown_refresh, None + return bool(demand) + + def unregister_health(self, incarnation): + self._check_pid() + with self._condition: + self._health.pop(id(incarnation), None) + + def register_lifetime(self): + """Return a caller-owned handle for an active lifetime.""" + self._check_pid() + with self._condition: + if self._closing: + raise ManagementClosed('Management lifetime is closed') + handle = _LifetimeHandle(self) + self._lifetimes.add(handle) + self._observe_counts_locked() + return handle + + def close_lifetime(self, handle, retire): + self._check_pid() + if (not isinstance(handle, _LifetimeHandle) or + handle.runtime is not self): + raise ManagementContextError( + 'Close handle belongs to another runtime') + with self._condition: + closed = handle.closed + if closed.is_set(): + return closed + # Retirement is native, short and must not run application code. + retire() + closed.set() + self._lifetimes.discard(handle) + return closed + + def _schedule_callbacks(self, future): + with self._condition: + record = future._record + self._queue_delivery_locked(record) + + def _delivery_failed_locked(self, record, error): + for future in chain(record.futures, record.foreign_futures): + with future._condition: + future._delivery_error = error + future._done_callbacks.clear() + future._callback_cursor = 0 + + def _queue_delivery_locked(self, record): + if record.delivery: + return + if not any( + future.done() and + future._callback_cursor < len(future._done_callbacks) + for future in chain(record.futures, record.foreign_futures)): + return + if self.completion_error is not None: + self._delivery_failed_locked(record, self.completion_error) + return + record.delivery = True + self._observe_counts_locked() + self._dispatch_delivery_locked(record) + + def _resume_delivery(self, record): + """Resume a deferred batch when its drainer returns.""" + self._check_pid() + with self._condition: + if record.released or not record.delivery: + return + if record.delivery_ticket is not None: + record.delivery_again = True + else: + self._dispatch_delivery_locked(record) + + def _dispatch_delivery_locked(self, record): + if self.completion_error is not None: + self._delivery_failed_locked(record, self.completion_error) + record.delivery = False + self._release_locked(record) + return + record.delivery_ticket = _DeliveryTicket(lambda: self._deliver(record)) + try: + self._dispatch(record.delivery_ticket) + except BaseException as exc: + record.delivery = False + error = ManagementDeliveryFailed( + 'Completion reactor rejected management delivery') + error.__cause__ = exc + self.completion_error = error + for pending in tuple(self._records): + self._delivery_failed_locked(pending, error) + ticket = pending.delivery_ticket + if ticket is None or ticket.cancel(): + pending.delivery = False + pending.delivery_ticket = None + self._release_locked(pending) + # Infrastructure failure closes admission. Native outcomes already + # published are immutable; remaining waiters receive shutdown. + # Inactive queued tickets are abandoned; an active delivery keeps + # its credit until it returns. Undeliverable callbacks are never + # retried or executed as application code on an owner. + log.exception('Management reactor completion delivery failed') + self.close() + + def _deliver(self, record): + with _entered('bridge', record): + with self._condition: + record.delivery_again = False + while True: + with self._condition: + futures = tuple(chain(record.futures, + record.foreign_futures)) + stalled = False + for future in futures: + if future.done(): + if not future._drain_callbacks(record): + stalled = True + with self._condition: + if any( + future.done() and future._callback_cursor < + len(future._done_callbacks) + for future in chain(record.futures, + record.foreign_futures)): + if stalled: + # Another caller is executing a callback. Keep the + # charged batch and yield I/O until its handoff. + record.delivery_ticket = None + if record.delivery_again: + record.delivery_again = False + self._dispatch_delivery_locked(record) + return + continue + record.delivery = False + record.delivery_ticket = None + record.delivery_again = False + self._release_locked(record) + return + + def _release_locked(self, record): + if (record.released or not record.handler_done or record.physical or + record.mailbox_pending): + return + # Coalesced waiters share physical work, but private native completion + # must release its own window independently of ordinary callback debt. + if not record.private_released: + if record.budgets['aggregate']: + self._reentry_active = False + self._reentry_credits -= record.budgets['reentry'] + record.private_released = True + self._condition.notify_all() + if record.delivery or record.notification: + return + record.released = True + count = record.budgets['ordinary'] + if count: + self._credits -= count + if record.pool is not None: + remaining = self._pool_credits[record.pool] - count + if remaining: + self._pool_credits[record.pool] = remaining + else: + del self._pool_credits[record.pool] + self._records.discard(record) + record.handler = None + self._condition.notify_all() + self._maybe_finalized_locked() + + def _run(self, kind, target): + with _entered(kind): + try: + target() + finally: + with self._condition: + self._exited += 1 + self._maybe_finalized_locked() + + def _health_ready(self): + return any(slot[0] is not None or + (slot[1] is not None and self._capacity(None, 'ordinary')) + for _, slot in self._health.values()) + + def _run_cluster(self): + while True: + with self._condition: + while (not self._cluster and not self._closing and + not self._health_ready()): + self._condition.wait() + if self._closing and not self._cluster: + return + health = next((slot for _, slot in self._health.values() + if slot[0] is not None), None) + if health is not None: + callback, health[0] = health[0], None + self._sequence += 1 + record = _Record( + self, Envelope(self._sequence, 'health', None, + self.lifetime, ()), None, None, None) + self._records.add(record) + native_health = True + else: + pending = next((slot for _, slot in self._health.values() + if slot[1] is not None and + self._capacity(None, 'ordinary')), None) + if pending is not None: + notification, pending[1] = pending[1], None + self.notify(notification) + notification = pending = None + continue + record, callback, reserved = self._cluster.popleft() + record.mailbox_pending -= 1 + native_health = False + self._execute(record, callback, 'cluster') + if native_health: + self._finish(record) + with self._condition: + self._release_locked(record) + record = callback = health = None + + def _run_pool(self): + while True: + with self._condition: + while not self._ready and not self._closing: + self._condition.wait() + if self._closing and not self._ready: + return + pool = self._ready.popleft() + self._active_pools.add(pool) + record, callback, reserved = self._pools[pool].popleft() + record.mailbox_pending -= 1 + self._execute(record, callback, 'pool') + with self._condition: + self._active_pools.remove(pool) + self._release_locked(record) + if self._pools[pool]: + self._ready.append(pool) + else: + del self._pools[pool] + self._condition.notify_all() + record = callback = pool = None + + def _execute(self, record, callback, kind): + with _entered(kind, record): + try: + callback() + except BaseException as exc: + self._finish(record, error=exc) + + def _run_notifications(self): + while True: + with self._condition: + while not self._notifications and not self._closing: + self._condition.wait() + if self._closing: + return + record, callback = self._notifications.popleft() + with _entered('callback', record): + try: + value = callback() + except BaseException as exc: + self._finish(record, error=exc) + else: + self._finish(record, value) + finally: + # Drop a bound adapter before publishing released credits; + # an idle lane must not root its last coordinator. + callback = None + with self._condition: + record.notification = False + self._release_locked(record) + record = callback = value = None + + def retain_cleanup(self): + """Retain one cleanup obligation independently of load.""" + self._check_pid() + with self._condition: + if self._finalizing: + raise ManagementClosed('Management cleanup lifetime has ended') + self._cleanup_obligations += 1 + self._observe_counts_locked() + completed = Event() + def release(): + self._check_pid() + with self._condition: + if not completed.is_set(): + completed.set() + self._cleanup_obligations -= 1 + self._maybe_finalized_locked() + return release + + def close(self, wait=False): + self._check_pid() + if wait: + assert_can_wait() + with self._condition: + self._check_pid() + if not self._closing: + self._closing = True + for record in tuple(self._records): + self._finish(record, error=ManagementClosed( + 'Management lifetime ended')) + for record, _ in self._notifications: + record.notification = False + self._release_locked(record) + self._notifications.clear() + def retained_results(queue): + retained = deque() + for item in queue: + if item[2]: + retained.append(item) + else: + item[0].mailbox_pending -= 1 + self._release_locked(item[0]) + return retained + self._cluster = retained_results(self._cluster) + for pool, queue in tuple(self._pools.items()): + retained = retained_results(queue) + if retained or pool in self._active_pools: + self._pools[pool] = retained + else: + del self._pools[pool] + self._ready = deque(pool for pool in self._ready + if pool in self._pools) + self._keys.clear() + self._health.clear() + for handle in self._lifetimes: + handle.closed.set() + self._lifetimes.clear() + self._condition.notify_all() + self.closed.set() + self._maybe_finalized_locked() + if wait and execution_context() not in ('callback', 'worker'): + self.closed.wait() + return self.closed + + def _maybe_finalized_locked(self): + if (self._closing and not self._records and + not self._cleanup_obligations + and self._exited == len(self._threads) + and not self._finalizing): + self._finalizing = True + if self._on_finalized is None: + self.finalized.set() + else: + try: + self._on_finalized(self.finalized.set) + except BaseException: + log.exception( + 'Error detaching management completion channel') + self.finalized.set() + + def join(self, timeout=None): + self._check_pid() + assert_can_wait() + if execution_context() in ('callback', 'worker'): + raise ManagementContextError('Cannot join own management resource') + if not self.finalized.wait(timeout): + raise TimeoutError() + for thread in self._threads: + if thread is not current_thread(): + thread.join(timeout) + + +class ReactorChannel: + """Borrow dispatch without owning the process-global I/O service. + + Libev holds its I/O runner live until the final admitted delivery + drains; thread startup failures consequently precede runtime + admission. Closing one channel releases only its borrow, never + another Cluster's reactor. + """ + + def __init__(self, connection_class): + dispatch = getattr(connection_class, '_management_dispatch', None) + in_context = getattr(connection_class, '_management_in_context', None) + if dispatch is None or in_context is None: + raise ManagementContextError( + 'Connection class has no management completion adapter') + self._closed = False + self._dispatch = dispatch + self._release = None + connection_class.initialize_reactor() + borrow = getattr(connection_class, '_management_borrow', None) + if borrow is not None: + self._release = borrow() + elif not in_context(): + ready = Event() + dispatch(ready.set) + if not ready.wait(5): + raise ManagementContextError( + 'Management reactor channel startup did not complete') + + def dispatch(self, callback): + if self._closed: + raise ManagementClosed('Management completion channel detached') + self._dispatch(callback) + + def close(self, completed=lambda: None): + if self._closed: + completed() + return + self._closed = True + release, self._release = self._release, None + self._dispatch = None + if release is None: + completed() + else: + release(completed) + + +def assert_synchronous_api(connection_class): + assert_can_wait() + in_context = getattr(connection_class, '_management_in_context', None) + if in_context is not None and in_context(): + raise ManagementContextError( + 'Blocking management call on a reactor thread') + + +@contextmanager +def physical_work_context(cleanup=False): + """Mark physical work; cleanup may finish child closes inline.""" + previous = getattr(_context, 'cleanup', False) + _context.cleanup = cleanup + try: + with _entered('worker'): + yield + finally: + _context.cleanup = previous + + +def performing_cleanup(): + return getattr(_context, 'cleanup', False) + + +class _CompatibilityCall: + """Admit legacy phases while their state writers still use locks. + + The calling stack performs existing blocking work. A reserved + physical obligation holds its credit; C never waits for that + stack. Policy wrappers release a completed phase before arbitrary + policy reentry, then acquire the next phase after the policy + returns. Later migrations replace these bodies with native + asynchronous owner continuations. + """ + + def __init__(self, runtime): + self.runtime = runtime + current = getattr(_context, 'record', None) + self.budget = ('aggregate' if execution_context() == 'callback' and + current is not None and current.runtime is runtime + else 'ordinary') + self.record = None + + def acquire(self): + if self.record is not None: + return + if self.budget == 'aggregate' and self.runtime._reentry_active: + raise ManagementContextError('Recursive native reentry aggregate') + future = self.runtime.submit(lambda record: _PENDING, wait=True, + budget=self.budget, reserve=1) + self.record = future._record + try: + self.record.started.wait() + if future.done(): + future.result() + except BaseException: + self.release() + raise + + def release(self): + record, self.record = self.record, None + if record is not None: + with self.runtime._condition: + self.runtime._finish(record) + record.physical -= 1 + self.runtime._release_locked(record) + + +@contextmanager +def compatibility_call(runtime): + previous = getattr(_context, 'call', None) + call = _CompatibilityCall(runtime) + call.acquire() + _context.call = call + try: + yield + finally: + call.release() + _context.call = previous + + +def policy_call(method): + """Run user policy between released compatibility phases.""" + @wraps(method) + def call_policy(*args, **kwargs): + call = getattr(_context, 'call', None) + if call is None: + return method(*args, **kwargs) + call.release() + _context.call = None + try: + result = method(*args, **kwargs) + finally: + _context.call = call + call.acquire() + return result + return call_policy + + +@contextmanager +def compatibility_work(runtime): + """Charge physical work; L fanout uses its independent window.""" + if runtime is None: + # Internal tests construct bare Sessions with __new__, without a + # registered production lifetime or management admission contract. + yield lambda work: None + return + assert_can_wait() + call = getattr(_context, 'call', None) + same_call = call is not None and call.runtime is runtime + budget = ('reentry' if same_call and call.budget == 'aggregate' + else 'ordinary') + if same_call and budget == 'ordinary': + # Admission cannot retain a phase's credit while waiting for a child + # record in a full window, or retain a Session lock needed by workers. + call.release() + future = runtime.submit(lambda record: _PENDING, wait=same_call, + budget=budget, reserve=1) + record = future._record + attached = False + + def complete(work): + with runtime._condition: + runtime._finish(record) + record.physical -= 1 + runtime._release_locked(record) + + def attach(work): + nonlocal attached + if (work is not None and + callable(getattr(work, 'add_done_callback', None))): + attached = True + work.add_done_callback(complete) + try: + record.started.wait() + if future.done(): + future.result() + yield attach + finally: + if not attached: + complete(None) + if same_call and budget == 'ordinary': + call.acquire() diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py new file mode 100644 index 0000000000..1f8e829ba6 --- /dev/null +++ b/tests/unit/test_management_runtime.py @@ -0,0 +1,1633 @@ +from collections import deque +from concurrent.futures import Future, ThreadPoolExecutor, wait, TimeoutError +from threading import Barrier, Event, Lock, Thread, get_ident + +import pytest + +from cassandra._management import ( + ManagementClosed, ManagementContextError, ManagementOverloaded, + ManagementRuntime, execution_context) + + +class Dispatcher: + def __init__(self): + self.queue = deque() + self.lock = Lock() + + def __call__(self, callback): + with self.lock: + self.queue.append(callback) + + def drain(self): + while True: + with self.lock: + if not self.queue: + return + callback = self.queue.popleft() + callback() + + +@pytest.fixture +def runtime(): + dispatch = Dispatcher() + runtime = ManagementRuntime(dispatch, limit=4, pool_limit=2) + yield runtime, dispatch + runtime.close() + dispatch.drain() + runtime.join(5) + + +def test_fifo_one_owner_and_exception_progress(runtime): + manager, dispatch = runtime + observed = [] + owner = [] + def handler(record): + manager.assert_owner() + observed.append(record.envelope.operation) + owner.append(get_ident()) + return record.envelope.operation + futures = [manager.submit(handler) for _ in range(4)] + assert [f.result(5) for f in futures] == observed == [1, 2, 3, 4] + assert len(set(owner)) == 1 + def fail(record): + raise ValueError('bad operation') + with pytest.raises(ValueError): + manager.submit(fail).result(5) + assert manager.submit(handler).result(5) == 6 + + +def test_native_completion_preserves_delivery_capacity(runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + def handler(record): + entered.set() + assert release.wait(5) + return 7 + futures = [manager.submit(handler) for _ in range(4)] + order = [] + for i, future in enumerate(futures): + future.add_done_callback( + lambda f, i=i: order.append((i, execution_context()))) + assert entered.wait(5) + release.set() + assert all(f.result(5) == 7 for f in futures) + assert wait(futures, timeout=5).not_done == set() + with manager._condition: + assert manager._credits == 4 + assert len(dispatch.queue) == 4 + for _ in range(20): + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None) + futures[0].add_done_callback( + lambda f: order.append(('extra', execution_context()))) + assert len(dispatch.queue) == 4 + dispatch.drain() + assert order == [(0, 'bridge'), ('extra', 'bridge'), (1, 'bridge'), + (2, 'bridge'), (3, 'bridge')] + assert manager._credits == 0 + + +def test_coalesced_cancel_waiters_and_physical_work(runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + attempts = [] + with ThreadPoolExecutor(1) as executor: + def physical(): + attempts.append(True) + entered.set() + assert release.wait(5) + return 42 + first = manager.submit( + lambda r: manager.work(r, executor, physical), key='same') + assert entered.wait(5) + second = manager.submit( + lambda r: pytest.fail('duplicate handler'), key='same') + assert first is not second + assert first.cancel() + assert wait([first], timeout=1).not_done == set() + with pytest.raises(TimeoutError): + second.result(0) + release.set() + assert second.result(5) == 42 + assert attempts == [True] + + +def test_late_result_disposal_does_not_need_admission(runtime): + manager, dispatch = runtime + entered, release, disposed = Event(), Event(), Event() + resource = object() + with ThreadPoolExecutor(1) as executor: + def physical(): + entered.set() + assert release.wait(5) + return resource + future = manager.submit(lambda r: manager.work( + r, executor, physical, + dispose=lambda value: disposed.set() + if value is resource else None)) + assert entered.wait(5) + manager.close() + with pytest.raises(ManagementClosed): + future.result(5) + assert not manager.finalized.is_set() + release.set() + assert disposed.wait(5) + manager.join(5) + + +def test_callback_lane_wait_does_not_block_native_or_reactor_completion( + runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + complete = Event() + future = manager.notify(lambda: (entered.set(), release.wait(5))) + try: + assert entered.wait(5) + result = manager.submit(lambda record: (complete.wait(5), 9)[1]) + seen = [] + result.add_done_callback( + lambda f: seen.append((f.result(), execution_context()))) + complete.set() + assert result.result(5) == 9 + # Native publication precedes enqueue; fence that owner phase before + # observing the independently locked dispatcher. + with manager._condition: + assert len(dispatch.queue) == 1 + dispatch.drain() + assert seen == [(9, 'bridge')] + assert not future.done() + manager.close() + assert future.done() + finally: + complete.set() + release.set() + + +def test_owner_wait_fails_and_pool_handlers_are_serial(runtime): + manager, dispatch = runtime + pending = Future() + entered, release = Event(), Event() + pool = object() + def hold(record): + manager.assert_owner(pool) + entered.set() + assert release.wait(5) + first = manager.submit(hold, pool=pool) + assert entered.wait(5) + second = manager.submit(lambda record: None, pool=pool) + assert not second.done() + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None, pool=pool) + release.set() + first.result(5) + second.result(5) + def self_wait(record): + other = manager.submit(lambda r: 1) + with pytest.raises(ManagementContextError): + other.result() + manager.submit(self_wait).result(5) + + +def test_executor_rejection_releases_reservation(runtime): + manager, dispatch = runtime + class Rejected: + def submit(self, function): + raise RuntimeError('executor closed') + with pytest.raises(RuntimeError, match='executor closed'): + manager.submit( + lambda r: manager.work(r, Rejected(), lambda: None)).result(5) + with manager._condition: + assert manager._credits == 0 + + +def test_health_and_close_bypass_full_delivery_and_io_capacity(runtime): + manager, dispatch = runtime + # Fill ordinary credits with terminal records awaiting callback delivery. + gate = Event() + futures = [manager.submit(lambda record: gate.wait(5)) for _ in range(4)] + for future in futures: + future.add_done_callback(lambda f: None) + gate.set() + for future in futures: + future.result(5) + assert manager._credits == 4 + incarnation = object() + manager.register_health(incarnation) + native, notified = Event(), Event() + assert manager.observe_negative( + incarnation, native.set, notified.set, removed=True) + assert native.wait(5) + assert not notified.is_set() + # A pending removal cannot be overwritten by a positive/less terminal + # event. + assert manager.observe_negative( + incarnation, lambda: pytest.fail('old health')) + child_one = manager.register_lifetime() + child_two = manager.register_lifetime() + effects = [] + one = manager.close_lifetime(child_one, lambda: effects.append(1)) + assert manager.close_lifetime( + child_one, lambda: pytest.fail('duplicate')) is one + assert manager.close_lifetime( + child_two, lambda: effects.append(2)).is_set() + assert effects == [1, 2] + dispatch.drain() + assert notified.wait(5) + + +def test_callback_reserved_window_progresses_at_full_delivery_capacity( + runtime): + manager, dispatch = runtime + gate = Event() + entered, release, completed = Event(), Event(), Event() + def listener(): + entered.set() + assert release.wait(5) + with manager.synchronous_reentry() as budget: + assert budget == 'reentry' + with ThreadPoolExecutor(1) as executor: + for _ in range(100): + future = manager.submit( + lambda r: manager.work(r, executor, lambda: 11), + budget=budget, wait=True) + assert future.result(5) == 11 + completed.set() + notification = manager.notify(listener) + assert entered.wait(5) + futures = [manager.submit(lambda record: gate.wait(5)) for _ in range(3)] + for future in futures: + future.add_done_callback(lambda f: None) + gate.set() + for future in futures: + future.result(5) + assert manager._credits == 4 + release.set() + assert completed.wait(5) + notification.result(5) + assert manager._reentry_credits == 0 + + +def test_superseded_physical_result_is_disposed(runtime): + from cassandra._management import ManagementSuperseded + manager, dispatch = runtime + entered, release, disposed = Event(), Event(), Event() + with ThreadPoolExecutor(1) as executor: + def physical(): + entered.set() + assert release.wait(5) + return object() + future = manager.submit( + lambda r: manager.work( + r, executor, physical, dispose=lambda value: disposed.set())) + assert entered.wait(5) + manager.supersede(future) + with pytest.raises(ManagementSuperseded): + future.result(5) + release.set() + assert disposed.wait(5) + + +@pytest.mark.parametrize('failed_phase', [1, 2, 3, 4]) +def test_start_failure_and_repeated_close_leave_no_threads( + monkeypatch, failed_phase): + import cassandra._management as management + real_start = Thread.start + starts = [] + def start(thread): + starts.append(thread) + if len(starts) == failed_phase: + raise RuntimeError('thread startup failed') + return real_start(thread) + dispatch = Dispatcher() + manager = ManagementRuntime(dispatch) + try: + with monkeypatch.context() as patcher: + patcher.setattr(management.Thread, 'start', start) + with pytest.raises(RuntimeError, match='thread startup failed'): + manager.start() + finally: + manager.close() + manager.join(5) + assert all(not thread.is_alive() for thread in manager._threads) + for _ in range(100): + manager = ManagementRuntime(dispatch) + try: + assert manager.submit(lambda record: 1).result(5) == 1 + first = manager.close() + assert manager.close() is first + finally: + manager.close() + manager.join(5) + assert not manager._records + assert all(not thread.is_alive() for thread in manager._threads) + + +def test_post_terminal_owner_callbacks_use_current_charged_batch(runtime): + manager, dispatch = runtime + old = manager.submit(lambda record: 3) + assert old.result(5) == 3 + calls = [] + def handler(record): + for i in range(20): + old.add_done_callback( + lambda f, i=i: calls.append((i, execution_context()))) + assert calls == [] + current = manager.submit(handler) + current.result(5) + assert manager._credits == 1 + assert len(dispatch.queue) == 1 + dispatch.drain() + assert calls == [(i, 'bridge') for i in range(20)] + assert manager._credits == 0 + + +def test_post_terminal_registration_is_serial_across_threads(runtime): + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + future.result(5) + entered, release = Event(), Event() + order = [] + def first(future): + order.append('first-enter') + entered.set() + assert release.wait(5) + order.append('first-exit') + thread = Thread(target=lambda: future.add_done_callback(first)) + thread.start() + assert entered.wait(5) + future.add_done_callback(lambda f: order.append('second')) + assert order == ['first-enter'] + release.set() + thread.join(5) + assert order == ['first-enter', 'first-exit', 'second'] + + +def test_concurrent_producers_have_one_fifo_owner(monkeypatch): + dispatch = Dispatcher() + manager = ManagementRuntime(dispatch, limit=64) + admitted, handled, owners = [], [], set() + producer_lock = Lock() + start = Event() + admission = Barrier(8) + original_start = manager.start + def synchronized_start(): + # All eight callers have entered the actual submit() frame before any + # reaches its internal admission lock. No producer lock serializes it. + admission.wait(5) + original_start() + monkeypatch.setattr(manager, 'start', synchronized_start) + def producer(): + assert start.wait(5) + for _ in range(8): + future = manager.submit(lambda r: ( + handled.append(r.envelope.operation), owners.add(get_ident()))) + with producer_lock: + admitted.append((future._record.envelope.operation, future)) + threads = [Thread(target=producer) for _ in range(8)] + try: + for thread in threads: + thread.start() + start.set() + for thread in threads: + thread.join(5) + assert not thread.is_alive() + for _, future in admitted: + future.result(5) + assert handled == sorted(number for number, _ in admitted) + assert len(owners) == 1 + finally: + start.set() + admission.abort() + manager.close() + manager.join(5) + + +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +def test_result_reserved_before_close_is_still_disposed(runtime, pool): + manager, dispatch = runtime + worker = Future() + class ControlledExecutor: + def submit(self, function): + return worker + started, blocked, release, disposed = Event(), Event(), Event(), Event() + def initial(record): + started.set() + return manager.work(record, ControlledExecutor(), object, + dispose=lambda value: disposed.set()) + manager.pool_limit = 3 + future = manager.submit(initial, pool=pool) + try: + assert started.wait(5) + blocker = manager.submit( + lambda record: (blocked.set(), release.wait(5)), pool=pool) + assert blocked.wait(5) + queued = manager.submit( + lambda record: pytest.fail('closed ordinary work ran'), + pool=pool) + # The reserved result sits behind ordinary work. + worker.set_result(object()) + manager.close() + for pending in (future, blocker, queued): + with pytest.raises(ManagementClosed): + pending.result(5) + with manager._condition: + queue = manager._cluster if pool is None else manager._pools[pool] + assert len(queue) == 1 and queue[0][2] + if pool is not None: + assert pool in manager._active_pools + assert not disposed.is_set() + finally: + release.set() + assert disposed.wait(5) + + +def test_synchronous_admission_waits_without_allocating_pending_records( + runtime, monkeypatch): + manager, dispatch = runtime + gate = Event() + futures = [manager.submit(lambda record: gate.wait(5)) for _ in range(4)] + result = [] + waiting, done = Event(), Event() + original_wait = manager._condition.wait + + def observe_wait(timeout=None): + if get_ident() == thread.ident: + assert manager._credits == manager.limit + waiting.set() + return original_wait(timeout) + + def caller(): + result.append(manager.submit(lambda record: 8, wait=True).result(5)) + done.set() + thread = Thread(target=caller) + monkeypatch.setattr(manager._condition, 'wait', observe_wait) + thread.start() + try: + assert waiting.wait(5) + with manager._condition: + assert len(manager._records) == 4 + assert manager._credits == 4 + assert not done.is_set() + finally: + gate.set() + thread.join(5) + for future in futures: + future.result(5) + assert done.wait(5) + thread.join(5) + assert result == [8] + + +def test_pool_mailboxes_share_two_runners_without_concurrent_handlers(): + dispatch = Dispatcher() + manager = ManagementRuntime(dispatch, limit=20) + entered = [Event(), Event()] + release = Event() + pools = [object(), object(), object()] + running = set() + lock = Lock() + seen = [] + def handler(index): + def run(record): + manager.assert_owner(pools[index]) + with lock: + assert pools[index] not in running + running.add(pools[index]) + seen.append((index, get_ident())) + if index < 2: + entered[index].set() + assert release.wait(5) + with lock: + running.remove(pools[index]) + return run + try: + first = manager.submit(handler(0), pool=pools[0]) + second = manager.submit(handler(1), pool=pools[1]) + assert all(event.wait(5) for event in entered) + queued_same = manager.submit(handler(0), pool=pools[0]) + third = manager.submit(handler(2), pool=pools[2]) + assert not third.done() + assert not queued_same.done() + release.set() + for future in [first, second, queued_same, third]: + future.result(5) + assert len({thread for _, thread in seen}) == 2 + assert len(manager._threads) == 4 + finally: + release.set() + manager.close() + manager.join(5) + + +def test_worker_close_and_join_guard(runtime): + manager, dispatch = runtime + with ThreadPoolExecutor(1) as executor: + def physical(): + assert execution_context() == 'worker' + manager.close() + with pytest.raises(ManagementContextError): + manager.join() + return object() + future = manager.submit(lambda r: manager.work(r, executor, physical)) + with pytest.raises(ManagementClosed): + future.result(5) + manager.join(5) + + +@pytest.mark.parametrize('entrypoint', [ + 'submit', 'observe_negative', 'register_lifetime', 'close_lifetime', + 'unregister_health', 'retain_cleanup', 'counts', 'release_cleanup']) +def test_inherited_runtime_rejects_before_using_parent_locks( + runtime, entrypoint): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + manager, dispatch = runtime + manager.start() + handle = manager.register_lifetime() + finish_cleanup = manager.retain_cleanup() + locked, release = Event(), Event() + def holder(): + with manager._condition: + locked.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert locked.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + try: + { + 'submit': lambda: manager.submit(lambda record: None), + 'observe_negative': lambda: manager.observe_negative( + object(), None), + 'register_lifetime': manager.register_lifetime, + 'close_lifetime': lambda: manager.close_lifetime( + handle, lambda: None), + 'unregister_health': lambda: manager.unregister_health( + object()), + 'retain_cleanup': manager.retain_cleanup, + 'counts': manager.counts, + 'release_cleanup': finish_cleanup, + }[entrypoint]() + except ManagementClosed: + os.write(write_fd, b'rejected') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'inherited runtime entered a parent-held condition' + assert os.read(read_fd, 8) == b'rejected' + finally: + release.set() + thread.join(5) + finish_cleanup() + # The child is killed on timeout; after writing it exits in finally. + os.waitpid(pid, 0) + os.close(read_fd) + + +def test_one_physical_result_per_current_admitted_record(runtime): + manager, dispatch = runtime + physical = Future() + submissions = [] + disposed = Event() + class Executor: + def submit(self, function): + submissions.append(function) + return physical + def handler(record): + manager.work(record, Executor(), lambda: None, + dispose=lambda value: disposed.set()) + # A second work item would create an uncharged result queue node. + return manager.work(record, Executor(), lambda: None) + future = manager.submit(handler) + try: + with pytest.raises(ManagementContextError): + future.result(5) + assert len(submissions) == 1 + finally: + physical.set_result(object()) + assert disposed.wait(5) + old = future._record + def reuse(record): + with pytest.raises(ManagementContextError): + manager.work(old, Executor(), lambda: None) + manager.submit(reuse).result(5) + assert len(submissions) == 1 + + +def test_inherited_future_rejects_before_parent_held_condition(runtime): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + assert future.result(5) == 1 + locked, release = Event(), Event() + def holder(): + with future._condition: + locked.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert locked.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + calls = [future.done, future.running, future.cancelled, + future.result, future.exception, future.cancel, + lambda: future.add_done_callback(lambda f: None), + lambda: wait([future], timeout=0)] + rejected = 0 + for call in calls: + try: + call() + except ManagementClosed: + rejected += 1 + os.write(write_fd, str(rejected).encode()) + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'inherited Future entered a parent-held condition' + assert os.read(read_fd, 8) == b'8' + finally: + release.set() + thread.join(5) + os.close(read_fd) + os.waitpid(pid, 0) + + +def test_dispatch_rejection_closes_without_stranding_waiters_or_resources(): + from cassandra._management import ManagementDeliveryFailed + attempts, calls = [], [] + def reject(callback): + attempts.append(callback) + raise RuntimeError('reactor rejected callback') + (handler_entered, handler_release, worker_started, worker_release, + disposed) = [Event() for _ in range(5)] + with ThreadPoolExecutor(1) as executor: + manager = ManagementRuntime(reject, limit=2) + def physical(): + worker_started.set() + assert worker_release.wait(5) + return object() + try: + late = manager.submit(lambda r: manager.work( + r, executor, physical, dispose=lambda value: disposed.set())) + assert worker_started.wait(5) + def handler(record): + handler_entered.set() + assert handler_release.wait(5) + return 4 + future = manager.submit(handler) + assert handler_entered.wait(5) + future.add_done_callback( + lambda f: calls.append(execution_context())) + handler_release.set() + assert future.result(5) == 4 + assert manager.closed.wait(5) + assert isinstance( + manager.completion_error, ManagementDeliveryFailed) + assert isinstance(future._delivery_error, ManagementDeliveryFailed) + with pytest.raises(ManagementDeliveryFailed): + future.add_done_callback(lambda f: calls.append('late')) + with pytest.raises(ManagementClosed): + late.result(5) + worker_release.set() + assert disposed.wait(5) + finally: + handler_release.set() + worker_release.set() + manager.close() + manager.join(5) + assert len(attempts) == 1 + assert calls == [] + assert manager._credits == 0 + assert not manager._records + + +def test_fatal_service_loss_abandons_queued_tickets_without_retaining_results( +): + from cassandra._management import ManagementDeliveryFailed + queue = [] + calls = [] + def dispatch(ticket): + if queue: + raise RuntimeError('validated reactor lost') + queue.append(ticket) + manager = ManagementRuntime(dispatch, limit=2) + (first_entered, first_release, second_entered, + second_release) = [Event() for _ in range(4)] + def first_handler(record): + first_entered.set() + assert first_release.wait(5) + return object() + def second_handler(record): + second_entered.set() + assert second_release.wait(5) + try: + first = manager.submit(first_handler) + assert first_entered.wait(5) + first.add_done_callback(lambda f: calls.append('first')) + first_release.set() + first.result(5) + second = manager.submit(second_handler) + assert second_entered.wait(5) + second.add_done_callback(lambda f: calls.append('second')) + second_release.set() + second.result(5) + assert manager.closed.wait(5) + manager.join(5) + finally: + first_release.set() + second_release.set() + manager.close() + for ticket in queue: + ticket() + manager.join(5) + assert isinstance(first._delivery_error, ManagementDeliveryFailed) + assert queue[0]._callback is None + queue[0]() # A late reactor wakeup cannot resurrect an abandoned batch. + assert calls == [] + assert manager._credits == 0 + + +def test_cancellation_racing_settlement_does_not_strand_coalesced_waiters( + runtime, monkeypatch): + manager, dispatch = runtime + entered, release, at_setter, cancel_attempted = [Event() for _ in range(4)] + cancelled_state = Event() + def handler(record): + entered.set() + assert release.wait(5) + return 17 + first = manager.submit(handler, key='shared') + second = manager.submit(handler, key='shared') + assert entered.wait(5) + setter = first.set_result + invoke = first._invoke_callbacks + def observed_cancel(): + cancelled_state.set() + invoke() + def paused_setter(value): + at_setter.set() + assert cancel_attempted.wait(5) + # If settlement owns the future lock, cancellation must wait for its + # terminal update. Otherwise force the competing cancellation to win. + if not first._condition._is_owned(): + assert cancelled_state.wait(5) + setter(value) + monkeypatch.setattr(first, '_invoke_callbacks', observed_cancel) + monkeypatch.setattr(first, 'set_result', paused_setter) + def cancel(): + assert at_setter.wait(5) + cancel_attempted.set() + first.cancel() + thread = Thread(target=cancel) + thread.start() + try: + release.set() + assert second.result(5) == 17 + assert first.done() + finally: + release.set() + thread.join(5) + assert not thread.is_alive() + + +def test_notification_admission_after_close_does_not_retain_callback( + runtime, monkeypatch): + import gc + import weakref + manager, dispatch = runtime + entered, release = Event(), Event() + original = manager.submit + def submit(handler, **kwargs): + def paused(record): + entered.set() + assert release.wait(5) + return handler(record) + return original(paused, **kwargs) + monkeypatch.setattr(manager, 'submit', submit) + class Callback: + def __call__(self): + pytest.fail('closed notification ran') + callback = Callback() + reference = weakref.ref(callback) + future = manager.notify(callback) + assert entered.wait(5) + manager.close() + release.set() + manager.join(5) + del callback + gc.collect() + assert reference() is None + assert not manager._notifications + with pytest.raises(ManagementClosed): + future.result(5) + + +@pytest.mark.parametrize('private_first', [False, True]) +@pytest.mark.parametrize('private_budget', ['reentry', 'aggregate']) +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +def test_coalesced_waiters_keep_independent_budget_and_delivery_charges( + runtime, private_first, private_budget, pool): + from contextlib import nullcontext + manager, dispatch = runtime + physical = Future() + work_started, joined, allow_join = Event(), Event(), Event() + calls, private = [], [] + class Executor: + def submit(self, function): + calls.append(function) + work_started.set() + return physical + def operation(record): + return manager.work(record, Executor(), lambda: 23) + def listener(): + assert allow_join.wait(5) + scope = (manager.synchronous_reentry() if private_budget == 'reentry' + else nullcontext('aggregate')) + with scope as budget: + future = manager.submit(operation, key='shared-budget', + budget=budget, pool=pool) + private.append(future) + with pytest.raises(ManagementContextError): + future.add_done_callback(lambda f: None) + joined.set() + assert future.result(5) == 23 + try: + if not private_first: + ordinary = manager.submit( + operation, key='shared-budget', pool=pool) + assert work_started.wait(5) + notification = manager.notify(listener) + allow_join.set() + assert joined.wait(5) + if private_first: + assert work_started.wait(5) + ordinary = manager.submit( + operation, key='shared-budget', pool=pool) + assert private[0]._record is ordinary._record + ordinary.add_done_callback(lambda f: None) + physical.set_result(23) + assert ordinary.result(5) == 23 + notification.result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: manager._credits == 1, 5) + assert manager._reentry_credits == 0 + assert not manager._reentry_active + if pool is not None: + assert manager._pool_credits[pool] == 1 + assert len(calls) == 1 + assert len(dispatch.queue) == 1 + dispatch.drain() + assert manager.counts()['ordinary'] == manager.counts()['reentry'] == 0 + assert not manager._pool_credits + finally: + allow_join.set() + if not physical.done(): + physical.set_result(23) + + +def test_foreign_callback_uses_ordinary_admission(runtime): + first, dispatch = runtime + other = ManagementRuntime(dispatch, limit=1) + gate, entered = Event(), Event() + try: + occupied = other.submit(lambda record: (entered.set(), gate.wait(5))) + assert entered.wait(5) + def listener(): + with other.synchronous_reentry() as budget: + assert budget == 'ordinary' + with pytest.raises(ManagementOverloaded): + other.submit(lambda record: None, budget=budget) + assert not other._reentry_active + first.notify(listener).result(5) + finally: + gate.set() + other.close() + other.join(5) + + +def test_standard_wait_rejects_owner_self_wait_and_allows_external_wait( + runtime): + manager, dispatch = runtime + following = [] + def handler(record): + future = manager.submit(lambda record: 41) + following.append(future) + with pytest.raises(ManagementContextError): + wait([future], timeout=0) + return 40 + completed = manager.submit(handler) + assert completed.result(5) == 40 + assert not wait([completed, following[0]], timeout=5).not_done + assert following[0].result(5) == 41 + + +def test_external_standard_wait_wakes_when_pending_operation_is_cancelled( + runtime): + manager, dispatch = runtime + entered, release, installed = Event(), Event(), Event() + blocker = manager.submit(lambda record: (entered.set(), release.wait(5))) + assert entered.wait(5) + future = manager.submit(lambda record: 42) + class ObservedWaiters(list): + def append(self, waiter): + super().append(waiter) + installed.set() + future._waiters = ObservedWaiters() + results = [] + thread = Thread(target=lambda: results.append(wait([future], timeout=5))) + thread.start() + try: + assert installed.wait(5) + assert future.cancel() + thread.join(5) + assert not thread.is_alive() + assert results[0].done == {future} + assert not results[0].not_done + finally: + release.set() + thread.join(5) + blocker.result(5) + + +def test_close_handles_retire_history_but_preserve_live_duplicate_joins( + runtime): + import gc + import weakref + manager, dispatch = runtime + references, retired = [], [] + for i in range(128): + handle = manager.register_lifetime() + references.append(weakref.ref(handle)) + done = manager.close_lifetime(handle, lambda: retired.append(i)) + assert done.is_set() + assert manager.close_lifetime( + handle, lambda: pytest.fail('duplicate')) is done + del handle + gc.collect() + assert all(reference() is None for reference in references) + assert retired == list(range(128)) + assert manager.counts()['close'] == 0 + first, second = manager.register_lifetime(), manager.register_lifetime() + manager.close_lifetime(first, lambda: None) + assert not second.closed.is_set() + manager.close() + assert second.closed.is_set() + assert not manager._lifetimes + assert manager.close_lifetime( + second, lambda: pytest.fail('after close')) is second.closed + with pytest.raises(ManagementClosed): + manager.register_lifetime() + + +@pytest.mark.parametrize('terminal_first', [False, True]) +def test_bulk_owner_rejection_unwinds_all_conditions(runtime, terminal_first): + manager, dispatch = runtime + already = manager.submit(lambda record: 1) + assert already.result(5) == 1 + following = [] + def handler(record): + pair = [manager.submit(lambda record: 2) for _ in range(2)] + terminal, pending = sorted(pair, key=id, reverse=not terminal_first) + assert terminal.cancel() + assert wait([already, terminal], timeout=0).done == {already, terminal} + with pytest.raises(ManagementContextError): + wait([terminal, pending], timeout=0) + # Rejection must not contaminate this thread's ordinary status access. + assert not pending.done() + following.extend(pair) + manager.submit(handler).result(5) + assert not wait(following, timeout=5).not_done + for future in following: + acquired = future._condition.acquire(timeout=1) + try: + assert acquired, ( + 'bulk wait rejection retained an earlier condition') + finally: + if acquired: + future._condition.release() + + +@pytest.mark.parametrize( + 'outcome', ['return', 'error', 'base_error', 'rejected_reacquire']) +def test_policy_phase_restores_context_and_only_reacquires_after_success( + runtime, outcome): + from cassandra import _management + manager, dispatch = runtime + class PolicyAbort(BaseException): + pass + @_management.policy_call + def policy(): + assert _management._context.call is None + assert manager.counts()['ordinary'] == 0 + if outcome == 'error': + raise ValueError('policy failed') + if outcome == 'base_error': + raise PolicyAbort() + if outcome == 'rejected_reacquire': + manager.close() + return 13 + with _management.compatibility_call(manager): + call = _management._context.call + original = call.record + if outcome == 'return': + assert policy() == 13 + assert call.record is not original + else: + error = {'error': ValueError, 'base_error': PolicyAbort, + 'rejected_reacquire': ManagementClosed}[outcome] + with pytest.raises(error): + policy() + assert call.record is None + assert _management._context.call is call + assert getattr(_management._context, 'call', None) is None + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +def test_compatibility_work_rejects_owner_before_admission( + runtime, monkeypatch, pool): + from cassandra._management import compatibility_work + manager, dispatch = runtime + submitted = [] + original = manager.submit + def submit(*args, **kwargs): + future = original(*args, **kwargs) + submitted.append(future) + def forbidden_wait(*args, **kwargs): + raise AssertionError('compatibility work attempted an owner wait') + monkeypatch.setattr(future._record.started, 'wait', forbidden_wait) + return future + def handler(record): + monkeypatch.setattr(manager, 'submit', submit) + try: + with pytest.raises(ManagementContextError): + with compatibility_work(manager): + pytest.fail('owner entered physical compatibility work') + assert submitted == [] + finally: + # Keep the regression's old-source failure safe: it admits a + # synthetic physical obligation before reaching the forbidden wait. + for future in submitted: + with manager._condition: + manager._finish(future._record) + future._record.physical -= 1 + manager._release_locked(future._record) + original(handler, pool=pool).result(5) + + +def test_health_registration_crossing_final_close_rejects_without_retention( + runtime, monkeypatch): + manager, dispatch = runtime + started, resume = Event(), Event() + original = manager.start + def start(): + original() + started.set() + assert resume.wait(5) + monkeypatch.setattr(manager, 'start', start) + incarnation = object() + errors = [] + def register(): + try: + manager.register_health(incarnation) + except BaseException as exc: + errors.append(exc) + thread = Thread(target=register) + thread.start() + try: + assert started.wait(5) + manager.close() + manager.join(5) + resume.set() + thread.join(5) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], ManagementClosed) + assert manager.counts()['health'] == 0 + finally: + resume.set() + thread.join(5) + manager.unregister_health(incarnation) + + +def test_cleanup_cannot_join_an_already_started_async_finalization(): + detached = [] + manager = ManagementRuntime(Dispatcher(), on_finalized=detached.append) + early = manager.retain_cleanup() + late = None + try: + manager.close() + assert not detached + early() + assert len(detached) == 1 and manager._finalizing + assert not manager.finalized.is_set() + with pytest.raises(ManagementClosed): + late = manager.retain_cleanup() + assert manager.counts()['close'] == 0 + detached[0]() + manager.join(5) + finally: + early() + if late is not None: + late() + manager.close() + if detached: + detached[0]() + manager.join(5) + + +@pytest.mark.parametrize('entry', ['call', 'work']) +@pytest.mark.parametrize('drain_queued', [False, True]) +def test_interrupted_initial_compatibility_admission_discharges_reservation( + runtime, monkeypatch, entry, drain_queued): + from cassandra._management import compatibility_call, compatibility_work + manager, dispatch = runtime + entered, resume = Event(), Event() + manager.submit(lambda record: (entered.set(), resume.wait(5))) + assert entered.wait(5) + admitted = [] + original = manager.submit + def submit(*args, **kwargs): + future = original(*args, **kwargs) + admitted.append(future) + def interrupt(*args, **kwargs): + assert not future._record.started.is_set() + raise KeyboardInterrupt() + monkeypatch.setattr(future._record.started, 'wait', interrupt) + return future + monkeypatch.setattr(manager, 'submit', submit) + try: + scope = compatibility_call if entry == 'call' else compatibility_work + with pytest.raises(KeyboardInterrupt): + with scope(manager): + pytest.fail('interrupted admission entered its body') + assert len(admitted) == 1 + assert admitted[0]._record.physical == 0 + if drain_queued: + monkeypatch.setattr(manager, 'submit', original) + resume.set() + manager.submit(lambda record: None).result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: not manager._records, 5) + assert admitted[0]._record.released + assert admitted[0]._record.physical == 0 + else: + manager.close() + assert admitted[0]._record.released + finally: + resume.set() + manager.close() + # Unwind the old-source counterexample without leaving daemon threads + # or a synthetic physical reservation behind after the failed + # assertion. + for future in admitted: + with manager._condition: + if future._record.physical: + future._record.physical -= 1 + manager._release_locked(future._record) + manager.join(5) + + +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +@pytest.mark.parametrize('ending', ['normal', 'close', 'fatal']) +def test_owner_callback_handoff_yields_reactor_while_caller_callback_runs( + runtime, monkeypatch, pool, ending): + from cassandra._management import ManagementDeliveryFailed + manager, dispatch = runtime + old = manager.submit(lambda record: 3) + assert old.result(5) == 3 + with manager._condition: + assert manager._condition.wait_for(lambda: old._record.released, 5) + entered, resume, io_progress = Event(), Event(), Event() + calls = [] + def caller_callback(future): + calls.append(('caller-enter', execution_context())) + entered.set() + assert resume.wait(5) + calls.append(('caller-exit', execution_context())) + caller = Thread(target=lambda: old.add_done_callback(caller_callback)) + reactor = Thread(target=dispatch.drain) + caller.start() + try: + assert entered.wait(5) + def owner(record): + old.add_done_callback( + lambda f: calls.append(('owner', execution_context()))) + manager.submit(owner, pool=pool).result(5) + dispatch(io_progress.set) + reactor.start() + assert io_progress.wait(1), ( + 'callback handoff blocked shared reactor progress') + reactor.join(5) + assert not reactor.is_alive() + assert calls == [('caller-enter', None)] + assert manager.counts()['ordinary'] == 1 + if ending == 'close': + manager.close() + assert not manager.finalized.is_set() + elif ending == 'fatal': + def reject(ticket): + raise RuntimeError('validated reactor lost during handoff') + monkeypatch.setattr(manager, '_dispatch', reject) + with manager._condition: + failed_delivery = manager.submit(lambda record: 9) + failed_delivery.add_done_callback( + lambda f: pytest.fail('undeliverable callback ran')) + assert failed_delivery.result(5) == 9 + assert manager.closed.wait(5) + manager.join(5) + assert isinstance(old._delivery_error, ManagementDeliveryFailed) + assert manager.counts()['ordinary'] == 0 + finally: + resume.set() + caller.join(5) + if reactor.ident is not None: + reactor.join(5) + assert not caller.is_alive() and not reactor.is_alive() + dispatch.drain() + expected = [('caller-enter', None), ('caller-exit', None)] + if ending != 'fatal': + expected.append(('owner', 'bridge')) + assert calls == expected + assert manager.counts()['ordinary'] == 0 + assert not old._callback_waiters + + +@pytest.mark.parametrize('one_record', [False, True]) +def test_deferred_callback_handoffs_keep_one_ticket_per_charged_record( + runtime, one_record): + manager, dispatch = runtime + old = [manager.submit(lambda record: 1) for _ in range(3)] + assert all(f.result(5) == 1 for f in old) + with manager._condition: + assert manager._condition.wait_for( + lambda: not manager._records, 5) + count = 1 if one_record else 3 + manager.limit = count + entered = [Event() for _ in old] + resume, io_progress = Event(), Event() + calls = [] + def caller_callback(future, ready): + ready.set() + assert resume.wait(5) + callers = [Thread(target=lambda f=f, ready=ready: + f.add_done_callback(lambda f: caller_callback(f, ready))) + for f, ready in zip(old, entered)] + reactor = Thread(target=dispatch.drain) + for thread in callers: + thread.start() + try: + assert all(ready.wait(5) for ready in entered) + def owner(futures): + for future in futures: + future.add_done_callback( + lambda f: calls.append(execution_context())) + if one_record: + manager.submit(lambda record: owner(old)).result(5) + else: + for future in old: + manager.submit(lambda record, f=future: owner([f])).result(5) + dispatch(io_progress.set) + reactor.start() + assert io_progress.wait(1) + reactor.join(5) + assert not reactor.is_alive() + with manager._condition: + assert len(manager._records) == count + assert all(r.delivery and r.delivery_ticket is None + for r in manager._records) + assert not dispatch.queue + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None) + resume.set() + for thread in callers: + thread.join(5) + assert not thread.is_alive() + assert len(dispatch.queue) == count + dispatch.drain() + assert calls == ['bridge'] * 3 + assert manager.counts()['ordinary'] == 0 + assert all(not f._callback_waiters for f in old) + finally: + resume.set() + for thread in callers: + thread.join(5) + if reactor.ident is not None: + reactor.join(5) + + +def test_callback_handoff_completion_during_active_delivery_cannot_lose_wakeup( + runtime, monkeypatch): + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + assert future.result(5) == 1 + with manager._condition: + assert manager._condition.wait_for(lambda: future._record.released, 5) + entered, resume, deferred, return_delivery = [Event() for _ in range(4)] + calls = [] + def callback(future): + entered.set() + assert resume.wait(5) + caller = Thread(target=lambda: future.add_done_callback(callback)) + reactor = Thread(target=dispatch.drain) + original = future._drain_callbacks + def drain(record=None): + result = original(record) + if record is not None and not result: + deferred.set() + assert return_delivery.wait(5) + return result + monkeypatch.setattr(future, '_drain_callbacks', drain) + caller.start() + try: + assert entered.wait(5) + owner = manager.submit(lambda record: future.add_done_callback( + lambda f: calls.append(execution_context()))) + owner.result(5) + reactor.start() + assert deferred.wait(5) + resume.set() + caller.join(5) + assert not caller.is_alive() + with manager._condition: + assert owner._record.delivery_again + assert owner._record.delivery_ticket is not None + return_delivery.set() + reactor.join(5) + assert not reactor.is_alive() + assert calls == ['bridge'] + assert not dispatch.queue + assert manager.counts()['ordinary'] == 0 + finally: + resume.set() + return_delivery.set() + caller.join(5) + if reactor.ident is not None: + reactor.join(5) + + +@pytest.mark.parametrize('budget', ['reentry', 'aggregate']) +def test_private_owner_record_cannot_borrow_an_old_future_callback_batch( + runtime, budget): + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + assert future.result(5) == 1 + with manager._condition: + assert manager._condition.wait_for(lambda: future._record.released, 5) + def handler(record): + with pytest.raises(ManagementContextError): + future.add_done_callback( + lambda f: pytest.fail('uncharged callback')) + manager.submit(handler, budget=budget).result(5) + assert future._delivery_record is future._record + assert not dispatch.queue + + +@pytest.mark.parametrize('outcome', ['result', 'error']) +def test_coalesced_settlement_and_bulk_wait_use_compatible_lock_order(outcome): + import subprocess + import sys + # The broken lock order also blocks close, so isolate and reap the child. + # Preallocate real futures in reverse ID order rather than assume an + # allocator order. Admission and stdlib bulk waiting remain unchanged. + script = r'''import faulthandler +faulthandler.dump_traceback_later(7) +"""Bounded child probe; parent terminates a genuine bulk-lock deadlock.""" +from concurrent.futures import wait +from threading import Event, Thread, get_ident +import sys + +from cassandra import _management + +print('management-loader', _management.__file__, flush=True) +manager = _management.ManagementRuntime(lambda callback: None) +gate, owner_held, waiter_held = Event(), Event(), Event() +original = _management.OperationFuture +available = [] + + +def factory(record): + if not available: + available.extend(sorted([original(record), original(record)], + key=id, reverse=True)) + return available.pop(0) + + +def handler(record): + assert gate.wait(5) + if sys.argv[1] == 'error': + raise ValueError('controlled failure') + return 17 + + +_management.OperationFuture = factory +try: + first = manager.submit(handler, key='shared') + second = manager.submit(handler, key='shared') +finally: + _management.OperationFuture = original +assert first._record is second._record and id(second) < id(first) +setter = 'set_exception' if sys.argv[1] == 'error' else 'set_result' +original_setter = getattr(first, setter) + + +def publish(value): + owner_held.set() + assert waiter_held.wait(5) + original_setter(value) + + +setattr(first, setter, publish) +original_acquire = second._condition.acquire + + +def acquire(*args, **kwargs): + result = original_acquire(*args, **kwargs) + if get_ident() == waiter.ident: + waiter_held.set() + return result + + +second._condition.acquire = acquire +results = [] + + +def bulk_wait(): + assert owner_held.wait(5) + results.append(wait([first, second], timeout=5)) + + +waiter = Thread(target=bulk_wait) +waiter.start() +gate.set() +waiter.join(6) +assert not waiter.is_alive(), 'coalesced settlement deadlocked bulk wait' +assert results and results[0].done == {first, second} +assert not results[0].not_done +for future in (first, second): + if sys.argv[1] == 'error': + assert isinstance(future.exception(1), ValueError) + else: + assert future.result(1) == 17 +assert manager.submit(lambda record: 18).result(5) == 18 +manager.close() +manager.join(5) +assert manager.finalized.is_set() +print('bulk-wait-and-close-complete', flush=True) + +faulthandler.cancel_dump_traceback_later() +''' + result = subprocess.run( + [sys.executable, '-c', script, outcome], capture_output=True, + text=True, timeout=12) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize('plain', [False, True]) +@pytest.mark.parametrize('inherited_first', [False, True]) +def test_mixed_bulk_fork_rejection_preserves_fresh_locks( + runtime, plain, inherited_first): + import os + import select + import signal + from concurrent.futures import as_completed + from concurrent.futures._base import _AcquireFutures + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + manager, dispatch = runtime + inherited = manager.submit(lambda record: 1) + assert inherited.result(5) == 1 + locked, release = Event(), Event() + + def hold_parent_condition(): + with inherited._condition: + locked.set() + assert release.wait(25) + + holder = Thread(target=hold_parent_condition) + holder.start() + try: + assert locked.wait(5) + read_fd, write_fd = os.pipe() + try: + pid = os.fork() + except BaseException: + os.close(read_fd) + os.close(write_fd) + raise + except BaseException: + release.set() + holder.join(5) + raise + if pid == 0: + os.close(read_fd) + try: + fresh_manager = None + gate = Event() + if plain: + fresh = Future() + else: + fresh_manager = ManagementRuntime(lambda callback: None) + fresh = fresh_manager.submit(lambda record: gate.wait(5)) + + def inherited_lock_access(*args, **kwargs): + raise AssertionError('touched an inherited condition lock') + + inherited._condition._acquire = inherited_lock_access + inherited._condition._release = inherited_lock_access + try: + with inherited._condition: + raise AssertionError('entered inherited condition context') + except ManagementClosed: + pass + fresh._condition.acquire() + for consumer in ( + lambda: wait([fresh, inherited], timeout=0), + lambda: next(as_completed([fresh, inherited], timeout=0))): + try: + consumer() + except ManagementClosed: + pass + else: + raise AssertionError('bulk wait accepted inherited future') + # Exercise both possible ID-sorted acquisition permutations without + # assuming an allocator order or patching any stdlib function. + ordered = _AcquireFutures([fresh, inherited]) + ordered.futures = ([inherited, fresh] if inherited_first + else [fresh, inherited]) + try: + with ordered: + inherited._state + except ManagementClosed: + pass + else: + raise AssertionError('bulk context accepted inherited state') + attempts = [] + + def probe_outer_depth(): + acquired = fresh._condition.acquire(blocking=False) + attempts.append(acquired) + if acquired: + fresh._condition.release() + + probe = Thread(target=probe_outer_depth) + probe.start() + probe.join(5) + assert attempts == [False], 'bulk rejection released outer lock' + fresh._condition.release() + settled = Event() + + def settle(): + fresh.set_result(23) + settled.set() + + worker = Thread(target=settle) + worker.start() + assert settled.wait(5), 'bulk rejection leaked fresh condition' + worker.join(5) + assert fresh.result(0) == 23 + if fresh_manager is not None: + gate.set() + fresh_manager.close() + fresh_manager.join(5) + os.write(write_fd, b'ok') + except BaseException as exc: + os.write(write_fd, repr(exc).encode()[:4000]) + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 20) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'mixed bulk wait entered a parent-held condition' + outcome = os.read(read_fd, 4096) + assert outcome == b'ok', outcome.decode() + finally: + release.set() + holder.join(5) + os.close(read_fd) + os.waitpid(pid, 0) From 193e9c9ff44b3c0c88eb513933319997c648fe10 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Nov 2026 21:47:33 -0400 Subject: [PATCH 2/3] management: preserve notification and result ownership Reject pending same-runtime notification waits from the serial callback lane without blocking independent native reentry. Retry callback batch selection when its original reservation retires during registration. Serialize physical-result adoption with close and supersession. Retain the original executor worker until native acceptance or rejected-result disposal finishes, so a completed Future cannot move cleanup onto an owner. Claim delivery once across execution, cancellation and interrupted callback registration. Keep admission charged until cleanup completes. Cover the races with controlled interleavings, actual disposal thread checks, standard bulk waits and executor failure regressions. --- cassandra/_management.py | 126 +++++++++---- tests/unit/test_management_runtime.py | 256 ++++++++++++++++++++++++-- 2 files changed, 333 insertions(+), 49 deletions(-) diff --git a/cassandra/_management.py b/cassandra/_management.py index 5bdd5f94e3..e425093118 100644 --- a/cassandra/_management.py +++ b/cassandra/_management.py @@ -98,7 +98,8 @@ def _checked_acquire(self, *args, **kwargs): def _checked_release(self): if self._runtime._pid == os.getpid(): - return self._release() + self._release() + return None def __enter__(self): self._runtime._check_pid() @@ -149,7 +150,7 @@ def _state(self): # Bulk waits read state after acquiring ALL future conditions. # Reject before waiter installation, where their context unwinds # every lock. Own nonblocking methods use scoped access above. - assert_can_wait() + self._assert_can_wait() return state @_state.setter @@ -179,16 +180,25 @@ def cancelled(self): def result(self, timeout=None): self._record.runtime._check_pid() if not self.done(): - assert_can_wait() + self._assert_can_wait() return super().result(timeout) @_future_access def exception(self, timeout=None): self._record.runtime._check_pid() if not self.done(): - assert_can_wait() + self._assert_can_wait() return super().exception(timeout) + def _assert_can_wait(self): + assert_can_wait() + current = getattr(_context, 'record', None) + if (execution_context() == 'callback' and current is not None and + current.runtime is self._record.runtime and + self._record.envelope.kind == 'notification'): + raise ManagementContextError( + 'Notification wait would block its own callback lane') + @_future_access def cancel(self): self._record.runtime._check_pid() @@ -235,6 +245,10 @@ def add_done_callback(self, fn): with self._condition: if record is not self._delivery_record: continue + if owner and record.released and target is record: + # Retirement can happen after target selection but + # before acquiring the runtime condition. + continue if self._delivery_error is not None: raise self._delivery_error if record.released and owner: @@ -580,7 +594,13 @@ def supersede(self, future): def work(self, record, executor, function, *, accept=lambda value: value, dispose=lambda value: None): - """Reserve result delivery before physical work, on C/P only.""" + """Reserve result delivery before physical work, on C/P only. + + The executor runs work outside C/P. Its completion retains the worker + until the owner accepts or rejects the value, then disposes a rejected + value there. ``accept`` is a short native state transition, serialized + with shutdown and supersession; it must not block or run user code. + """ self.assert_owner(record.pool) with self._condition: if (getattr(_context, 'record', None) is not record or @@ -590,17 +610,9 @@ def work(self, record, executor, function, *, accept=lambda value: value, 'result reservation') record.physical = 1 self._condition.notify_all() - def physical_work(): - with _entered('worker', record): - return function() - - try: - physical = executor.submit(physical_work) - except BaseException as exc: - with self._condition: - record.physical -= 1 - self._finish(record, error=exc) - return _PENDING + returned_to_owner = Event() + adopted = False + received = False def completed(future): with _entered('worker', record): @@ -611,14 +623,29 @@ def complete(future): value, error = future.result(), None except BaseException as exc: value, error = None, exc + receive(value, error) + + def receive(value, error): + nonlocal received with self._condition: + if received: + return + received = True if not self._closing: self._enqueue_locked( record, lambda: returned(value, error), reserved=True) - return - # Close retains this obligation without owner readmission. + queued = True + else: + queued = False + if queued and error is None: + # Only the short native result transition is awaited. Neither + # application callback delivery nor notification progress is + # part of this handoff. + returned_to_owner.wait() + # Keep physical ownership until adoption or disposal completes, + # including results rejected by a closed owner mailbox. try: - if error is None: + if error is None and not adopted: dispose(value) except BaseException: log.exception('Error disposing late management result') @@ -628,28 +655,53 @@ def complete(future): self._release_locked(record) def returned(value, error): + nonlocal adopted try: - if self._closing or record.handler_done: - if error is None: - dispose(value) - return - if error is not None: - self._finish(record, error=error) - else: - try: - accepted = accept(value) - except BaseException as exc: + with self._condition: + if self._closing or record.handler_done: + return + if error is not None: + self._finish(record, error=error) + else: try: - dispose(value) - finally: + accepted = accept(value) + except BaseException as exc: self._finish(record, error=exc) - else: - self._finish(record, accepted) + else: + adopted = True + self._finish(record, accepted) finally: - with self._condition: - record.physical -= 1 - self._release_locked(record) - physical.add_done_callback(completed) + returned_to_owner.set() + + def physical_work(): + with _entered('worker', record): + try: + value = function() + except BaseException as exc: + receive(None, exc) + raise + else: + # Hand off before completing the executor Future. A late + # callback registration can run inline on C/P, but by then + # this worker already owns adoption/disposal responsibility. + receive(value, None) + return value + + try: + physical = executor.submit(physical_work) + except BaseException as exc: + self._finish(record, error=exc) + receive(None, exc) + return _PENDING + try: + physical.add_done_callback(completed) + except BaseException as exc: + self._finish(record, error=exc) + if physical.cancel(): + # No worker will discharge a cancellation before execution. + # A callback installed before an interrupted registration may + # also deliver it; receive claims the obligation exactly once. + receive(None, exc) return _PENDING def notify(self, callback, *, wait=False, timeout=None, budget='ordinary'): diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index 1f8e829ba6..e8976db440 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -1,5 +1,6 @@ from collections import deque -from concurrent.futures import Future, ThreadPoolExecutor, wait, TimeoutError +from concurrent.futures import ( + Future, ThreadPoolExecutor, as_completed, wait, TimeoutError) from threading import Barrier, Event, Lock, Thread, get_ident import pytest @@ -163,9 +164,28 @@ def test_callback_lane_wait_does_not_block_native_or_reactor_completion( release.set() +@pytest.mark.parametrize('waiting', [ + lambda future: future.result(), + lambda future: future.exception(), + lambda future: wait([future]), + lambda future: next(as_completed([future]))]) +def test_callback_lane_rejects_its_pending_notification_wait(runtime, waiting): + manager, dispatch = runtime + nested = [] + def callback(): + future = manager.notify(lambda: 17) + nested.append(future) + with pytest.raises(ManagementContextError, match='callback lane'): + waiting(future) + # Ordinary owner operations remain available to synchronous reentry. + with manager.synchronous_reentry() as budget: + assert manager.submit(lambda record: 18, budget=budget).result(5) == 18 + manager.notify(callback).result(5) + assert nested[0].result(5) == 17 + + def test_owner_wait_fails_and_pool_handlers_are_serial(runtime): manager, dispatch = runtime - pending = Future() entered, release = Event(), Event() pool = object() def hold(record): @@ -226,10 +246,11 @@ def test_health_and_close_bypass_full_delivery_and_io_capacity(runtime): child_two = manager.register_lifetime() effects = [] one = manager.close_lifetime(child_one, lambda: effects.append(1)) - assert manager.close_lifetime( - child_one, lambda: pytest.fail('duplicate')) is one - assert manager.close_lifetime( - child_two, lambda: effects.append(2)).is_set() + repeated = manager.close_lifetime( + child_one, lambda: pytest.fail('duplicate')) + assert repeated is one + two = manager.close_lifetime(child_two, lambda: effects.append(2)) + assert two.is_set() assert effects == [1, 2] dispatch.drain() assert notified.wait(5) @@ -287,10 +308,156 @@ def physical(): assert disposed.wait(5) +@pytest.mark.parametrize('ending', ['close', 'supersede']) +def test_result_adoption_is_atomic_with_terminal_settlement(runtime, ending): + manager, dispatch = runtime + accepting, finish_accept, ending_started = Event(), Event(), Event() + resource = object() + disposed, errors = [], [] + def accept(value): + assert value is resource + # Acceptance publishes native state. Other terminal paths must take + # this same condition before they can claim the operation's outcome. + assert manager._condition._is_owned() + accepting.set() + assert finish_accept.wait(5) + return value + with ThreadPoolExecutor(1) as executor: + future = manager.submit(lambda record: manager.work( + record, executor, lambda: resource, accept=accept, + dispose=disposed.append)) + def end(): + ending_started.set() + try: + if ending == 'close': + manager.close() + else: + manager.supersede(future) + except BaseException as exc: + errors.append(exc) + ending_thread = Thread(target=end) + try: + assert accepting.wait(5) + ending_thread.start() + assert ending_started.wait(5) + finally: + finish_accept.set() + if ending_thread.ident is not None: + ending_thread.join(5) + assert not ending_thread.is_alive() + assert not errors + assert future.result(5) is resource + assert disposed == [] + + +def test_rejected_acceptance_retains_worker_until_disposal_finishes(runtime): + manager, dispatch = runtime + disposing, finish_dispose = Event(), Event() + resource = object() + physical_threads, disposal_threads = [], [] + def physical(): + physical_threads.append(get_ident()) + return resource + def accept(value): + raise ValueError('cannot adopt resource') + def dispose(value): + assert value is resource + disposal_threads.append(get_ident()) + disposing.set() + assert finish_dispose.wait(5) + with ThreadPoolExecutor(1) as executor: + future = manager.submit(lambda record: manager.work( + record, executor, physical, accept=accept, dispose=dispose)) + try: + with pytest.raises(ValueError, match='cannot adopt resource'): + future.result(5) + assert disposing.wait(5) + assert manager.submit(lambda record: 23).result(5) == 23 + manager.close() + assert not manager.finalized.is_set() + assert manager.counts()['ordinary'] == 1 + finally: + finish_dispose.set() + assert disposal_threads == physical_threads + manager.join(5) + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('cancelled', [False, True]) +def test_executor_returns_terminal_failure_without_owner_self_wait( + runtime, cancelled): + from concurrent.futures import CancelledError + manager, dispatch = runtime + physical = Future() + if cancelled: + physical.cancel() + else: + physical.set_exception(ValueError('executor failed before execution')) + class FailedExecutor: + def submit(self, function): + return physical + future = manager.submit(lambda record: manager.work( + record, FailedExecutor(), lambda: pytest.fail('physical work ran'))) + with pytest.raises(CancelledError if cancelled else ValueError): + future.result(5) + assert manager.submit(lambda record: 24).result(5) == 24 + + +def test_completed_executor_future_does_not_dispose_on_registering_owner( + runtime): + manager, dispatch = runtime + resource = object() + disposal_threads = [] + with ThreadPoolExecutor(1) as executor: + class ClosingExecutor: + def submit(self, function): + manager.close() + physical = executor.submit(function) + # Completion precedes callback registration, so Future will + # invoke that callback on this owner. Cleanup must already + # have happened on the physical worker, exactly once. + assert physical.result(5) is resource + return physical + future = manager.submit(lambda record: manager.work( + record, ClosingExecutor(), lambda: resource, + dispose=lambda value: disposal_threads.append(get_ident()))) + with pytest.raises(ManagementClosed): + future.result(5) + manager.join(5) + assert len(disposal_threads) == 1 + assert disposal_threads[0] not in { + thread.ident for thread in manager._threads} + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('installed', [False, True]) +def test_interrupted_physical_callback_registration_releases_once( + runtime, installed): + manager, dispatch = runtime + physical = Future() + register = physical.add_done_callback + def interrupted(callback): + if installed: + register(callback) + raise KeyboardInterrupt('physical registration interrupted') + physical.add_done_callback = interrupted + class PendingExecutor: + def submit(self, function): + return physical + future = manager.submit(lambda record: manager.work( + record, PendingExecutor(), lambda: pytest.fail('cancelled work ran'))) + with pytest.raises(KeyboardInterrupt, match='registration interrupted'): + future.result(5) + assert manager.submit(lambda record: 25).result(5) == 25 + assert physical.cancelled() + with manager._condition: + assert future._record.physical == 0 + assert manager.counts()['ordinary'] == 0 + + @pytest.mark.parametrize('failed_phase', [1, 2, 3, 4]) def test_start_failure_and_repeated_close_leave_no_threads( monkeypatch, failed_phase): - import cassandra._management as management real_start = Thread.start starts = [] def start(thread): @@ -302,7 +469,7 @@ def start(thread): manager = ManagementRuntime(dispatch) try: with monkeypatch.context() as patcher: - patcher.setattr(management.Thread, 'start', start) + patcher.setattr(Thread, 'start', start) with pytest.raises(RuntimeError, match='thread startup failed'): manager.start() finally: @@ -314,7 +481,8 @@ def start(thread): try: assert manager.submit(lambda record: 1).result(5) == 1 first = manager.close() - assert manager.close() is first + second = manager.close() + assert second is first finally: manager.close() manager.join(5) @@ -341,6 +509,54 @@ def handler(record): assert manager._credits == 0 +def test_owner_callback_reselects_batch_retired_during_registration(runtime): + manager, dispatch = runtime + entered, finish, selected, register = [Event() for _ in range(4)] + armed = Event() + calls = [] + old = manager.submit(lambda record: (entered.set(), finish.wait(5), 1)[2]) + assert entered.wait(5) + old.add_done_callback(lambda future: calls.append('old')) + finish.set() + assert old.result(5) == 1 + condition = manager._condition + + class RegistrationGate: + def __getattr__(self, name): + return getattr(condition, name) + + def __enter__(self): + if execution_context() == 'cluster' and armed.is_set(): + armed.clear() + selected.set() + assert register.wait(5) + return condition.__enter__() + + def __exit__(self, *args): + return condition.__exit__(*args) + + manager._condition = RegistrationGate() + current = None + try: + def handler(record): + armed.set() + old.add_done_callback(lambda future: calls.append('new')) + assert old._delivery_record is record + current = manager.submit(handler) + assert selected.wait(5) + dispatch.drain() + assert old._record.released + register.set() + current.result(5) + dispatch.drain() + assert calls == ['old', 'new'] + finally: + register.set() + if current is not None: + current.result(5) + manager._condition = condition + + def test_post_terminal_registration_is_serial_across_threads(runtime): manager, dispatch = runtime future = manager.submit(lambda record: 1) @@ -410,12 +626,17 @@ class ControlledExecutor: def submit(self, function): return worker started, blocked, release, disposed = Event(), Event(), Event(), Event() + disposal_context = [] + def dispose(value): + disposal_context.append((execution_context(), get_ident())) + disposed.set() def initial(record): started.set() return manager.work(record, ControlledExecutor(), object, - dispose=lambda value: disposed.set()) + dispose=dispose) manager.pool_limit = 3 future = manager.submit(initial, pool=pool) + completion = Thread(target=lambda: worker.set_result(object())) try: assert started.wait(5) blocker = manager.submit( @@ -425,7 +646,13 @@ def initial(record): lambda record: pytest.fail('closed ordinary work ran'), pool=pool) # The reserved result sits behind ordinary work. - worker.set_result(object()) + completion.start() + with manager._condition: + def result_queued(): + queue = (manager._cluster if pool is None + else manager._pools[pool]) + return any(item[2] for item in queue) + assert manager._condition.wait_for(result_queued, 5) manager.close() for pending in (future, blocker, queued): with pytest.raises(ManagementClosed): @@ -438,7 +665,12 @@ def initial(record): assert not disposed.is_set() finally: release.set() + if completion.ident is not None: + completion.join(5) + assert not completion.is_alive() assert disposed.wait(5) + assert disposal_context == [('worker', completion.ident)] + assert completion.ident not in {thread.ident for thread in manager._threads} def test_synchronous_admission_waits_without_allocating_pending_records( @@ -913,7 +1145,7 @@ def test_foreign_callback_uses_ordinary_admission(runtime): other = ManagementRuntime(dispatch, limit=1) gate, entered = Event(), Event() try: - occupied = other.submit(lambda record: (entered.set(), gate.wait(5))) + other.submit(lambda record: (entered.set(), gate.wait(5))) assert entered.wait(5) def listener(): with other.synchronous_reentry() as budget: From 48750ff3de969b6a8b618d83a59ebe5c62ec0c12 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Nov 2026 23:47:09 -0400 Subject: [PATCH 3/3] management: retain reservations across interrupted handoffs Serialize physical wrapper entry with executor rejection so queued work cannot outlive a retired reservation and running results remain owned until disposal. Track launched runners when startup is interrupted before Thread.start returns, including before bootstrap publishes an identity. Keep compatibility work charged until it finishes if callback registration is interrupted, and retire its reservation once. Reject saturated blocking ordinary admissions from same-runtime callbacks. Bound controlled result completion and failure cleanup. Require all 64 concurrent submissions to succeed, and observe rejection-credit release without racing terminal publication. Cover the ownership and startup interruption schedules with controlled regressions. --- cassandra/_management.py | 70 ++++++-- tests/unit/test_management_runtime.py | 246 ++++++++++++++++++++++++-- 2 files changed, 286 insertions(+), 30 deletions(-) diff --git a/cassandra/_management.py b/cassandra/_management.py index e425093118..7bb9e245de 100644 --- a/cassandra/_management.py +++ b/cassandra/_management.py @@ -24,7 +24,9 @@ from itertools import chain import logging import os -from threading import Condition, Event, Lock, Thread, current_thread, local +from threading import ( + Condition, Event, Lock, Thread, current_thread, enumerate as active_threads, + local) import time from cassandra import DriverException @@ -448,7 +450,14 @@ def start(self): thread = Thread(target=self._run, args=(kind, target), name='cassandra-management-' + kind, daemon=True) - thread.start() + try: + thread.start() + except BaseException: + # A start interrupted after native launch still owns a + # runner, even before bootstrap publishes its ident. + if thread in active_threads(): + self._threads.append(thread) + raise self._threads.append(thread) except BaseException: self._closing = True @@ -497,6 +506,12 @@ def submit(self, handler, *, pool=None, key=None, identity=(), if not wait: raise ManagementOverloaded( 'Management admission capacity exhausted') + current = getattr(_context, 'record', None) + if (execution_context() == 'callback' and + current is not None and current.runtime is self and + budget == 'ordinary'): + raise ManagementContextError( + 'Callback admission cannot wait behind its own credit') remaining = (None if deadline is None else deadline - time.monotonic()) if remaining is not None and remaining <= 0: @@ -613,17 +628,16 @@ def work(self, record, executor, function, *, accept=lambda value: value, returned_to_owner = Event() adopted = False received = False + work_started = False + submission_failed = False def completed(future): with _entered('worker', record): - complete(future) - - def complete(future): - try: - value, error = future.result(), None - except BaseException as exc: - value, error = None, exc - receive(value, error) + try: + value, error = future.result(), None + except BaseException as exc: + value, error = None, exc + receive(value, error) def receive(value, error): nonlocal received @@ -674,6 +688,13 @@ def returned(value, error): returned_to_owner.set() def physical_work(): + nonlocal work_started + with self._condition: + if submission_failed: + # submit() can queue a callable before raising. Its + # reservation was retired only if execution never began. + return None + work_started = True with _entered('worker', record): try: value = function() @@ -690,8 +711,11 @@ def physical_work(): try: physical = executor.submit(physical_work) except BaseException as exc: - self._finish(record, error=exc) - receive(None, exc) + with self._condition: + submission_failed = True + self._finish(record, error=exc) + if not work_started: + receive(None, exc) return _PENDING try: physical.add_done_callback(completed) @@ -1302,9 +1326,14 @@ def compatibility_work(runtime): budget=budget, reserve=1) record = future._record attached = False + completed = False def complete(work): + nonlocal completed with runtime._condition: + if completed: + return + completed = True runtime._finish(record) record.physical -= 1 runtime._release_locked(record) @@ -1313,8 +1342,21 @@ def attach(work): nonlocal attached if (work is not None and callable(getattr(work, 'add_done_callback', None))): - attached = True - work.add_done_callback(complete) + try: + attached = True + work.add_done_callback(complete) + except BaseException: + # Registration may have been interrupted before or after + # installation. Keep this synchronous caller responsible for + # the physical work until it actually settles; the callback + # and this fallback share the same once-only retirement. + while not work.done(): + try: + work.result() + except BaseException: + pass + complete(work) + raise try: record.started.wait() if future.done(): diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index e8976db440..6853bd1989 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -28,6 +28,24 @@ def drain(self): callback() +def complete_physical(manager, future, value): + """Bound a controlled Future's synchronous owner-handoff callback.""" + errors = [] + def complete(): + try: + future.set_result(value) + except BaseException as exc: + errors.append(exc) + thread = Thread(target=complete, daemon=True) + thread.start() + thread.join(5) + if thread.is_alive(): + manager.close() + thread.join(5) + assert not thread.is_alive(), 'physical result handoff did not finish' + assert not errors + + @pytest.fixture def runtime(): dispatch = Dispatcher() @@ -217,7 +235,83 @@ def submit(self, function): manager.submit( lambda r: manager.work(r, Rejected(), lambda: None)).result(5) with manager._condition: - assert manager._credits == 0 + assert manager._condition.wait_for(lambda: manager._credits == 0, 5) + + +@pytest.mark.parametrize('already_running', [False, True]) +def test_executor_failure_after_queueing_preserves_physical_ownership( + runtime, already_running): + manager, dispatch = runtime + entered, release = Event(), Event() + calls, disposed, queued, errors = [], [], [], [] + resource = object() + def physical(): + calls.append(get_ident()) + entered.set() + assert release.wait(5) + return resource + def run(function): + try: + function() + except BaseException as exc: + errors.append(exc) + class QueuedThenFailed: + def submit(self, function): + queued.append(Thread(target=run, args=(function,), daemon=True)) + if already_running: + queued[0].start() + assert entered.wait(5) + raise RuntimeError('failed after queueing') + future = manager.submit(lambda record: manager.work( + record, QueuedThenFailed(), physical, + dispose=lambda value: disposed.append(value))) + try: + with pytest.raises(RuntimeError, match='after queueing'): + future.result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: future._record.physical == int(already_running), 5) + if already_running: + manager.close() + assert not manager.finalized.is_set() + else: + queued[0].start() + finally: + release.set() + for thread in queued: + if thread.ident is not None: + thread.join(5) + assert not thread.is_alive() + assert not errors + assert bool(calls) == already_running + assert disposed == ([resource] if already_running else []) + with manager._condition: + assert manager._condition.wait_for(lambda: not manager._records, 5) + + +@pytest.mark.parametrize('notification', [False, True]) +def test_callback_rejects_saturated_ordinary_admission(notification): + manager = ManagementRuntime(Dispatcher(), limit=1) + finished = Event() + errors = [] + def callback(): + try: + if notification: + manager.notify(lambda: None, wait=True) + else: + manager.submit(lambda record: None, wait=True) + except BaseException as exc: + errors.append(exc) + finally: + finished.set() + try: + manager.notify(callback) + assert finished.wait(1), 'callback waited behind its own credit' + assert len(errors) == 1 + assert isinstance(errors[0], ManagementContextError) + finally: + manager.close() + manager.join(5) def test_health_and_close_bypass_full_delivery_and_io_capacity(runtime): @@ -456,21 +550,22 @@ def submit(self, function): @pytest.mark.parametrize('failed_phase', [1, 2, 3, 4]) +@pytest.mark.parametrize('startup_error', [RuntimeError, KeyboardInterrupt]) def test_start_failure_and_repeated_close_leave_no_threads( - monkeypatch, failed_phase): + monkeypatch, failed_phase, startup_error): real_start = Thread.start starts = [] def start(thread): starts.append(thread) if len(starts) == failed_phase: - raise RuntimeError('thread startup failed') + raise startup_error('thread startup failed') return real_start(thread) dispatch = Dispatcher() manager = ManagementRuntime(dispatch) try: with monkeypatch.context() as patcher: patcher.setattr(Thread, 'start', start) - with pytest.raises(RuntimeError, match='thread startup failed'): + with pytest.raises(startup_error, match='thread startup failed'): manager.start() finally: manager.close() @@ -490,6 +585,55 @@ def start(thread): assert all(not thread.is_alive() for thread in manager._threads) +@pytest.mark.parametrize('bootstrap_pending', [False, True]) +def test_start_interruption_after_launch_tracks_the_runner( + monkeypatch, bootstrap_pending): + manager = ManagementRuntime(Dispatcher()) + launched, release = Event(), Event() + threads = [] + real_start = Thread.start + real_bootstrap = Thread._bootstrap_inner + def bootstrap(thread): + if bootstrap_pending: + launched.set() + assert release.wait(5) + real_bootstrap(thread) + def start(thread): + threads.append(thread) + if not bootstrap_pending: + real_start(thread) + raise KeyboardInterrupt('interrupted after launch') + original_wait = thread._started.wait + def interrupted_wait(*args, **kwargs): + assert launched.wait(5) + assert thread.ident is None + raise KeyboardInterrupt('interrupted before bootstrap') + thread._started.wait = interrupted_wait + try: + real_start(thread) + finally: + thread._started.wait = original_wait + try: + with monkeypatch.context() as patcher: + patcher.setattr(Thread, '_bootstrap_inner', bootstrap) + patcher.setattr(Thread, 'start', start) + with pytest.raises(KeyboardInterrupt): + manager.start() + manager.close() + if bootstrap_pending: + assert not manager.finalized.is_set() + assert manager._threads == threads + finally: + release.set() + for thread in threads: + assert thread._started.wait(5) + thread.join(5) + assert not thread.is_alive() + manager.close() + manager.join(5) + assert manager._exited == len(threads) == 1 + + def test_post_terminal_owner_callbacks_use_current_charged_batch(runtime): manager, dispatch = runtime old = manager.submit(lambda record: 3) @@ -581,7 +725,7 @@ def first(future): def test_concurrent_producers_have_one_fifo_owner(monkeypatch): dispatch = Dispatcher() manager = ManagementRuntime(dispatch, limit=64) - admitted, handled, owners = [], [], set() + admitted, handled, owners, errors = [], [], set(), [] producer_lock = Lock() start = Event() admission = Barrier(8) @@ -593,12 +737,16 @@ def synchronized_start(): original_start() monkeypatch.setattr(manager, 'start', synchronized_start) def producer(): - assert start.wait(5) - for _ in range(8): - future = manager.submit(lambda r: ( - handled.append(r.envelope.operation), owners.add(get_ident()))) + try: + assert start.wait(5) + for _ in range(8): + future = manager.submit(lambda r: ( + handled.append(r.envelope.operation), owners.add(get_ident()))) + with producer_lock: + admitted.append((future._record.envelope.operation, future)) + except BaseException as exc: with producer_lock: - admitted.append((future._record.envelope.operation, future)) + errors.append(exc) threads = [Thread(target=producer) for _ in range(8)] try: for thread in threads: @@ -607,9 +755,12 @@ def producer(): for thread in threads: thread.join(5) assert not thread.is_alive() + assert not errors + assert len(admitted) == 64 for _, future in admitted: future.result(5) assert handled == sorted(number for number, _ in admitted) + assert len(handled) == 64 assert len(owners) == 1 finally: start.set() @@ -636,7 +787,7 @@ def initial(record): dispose=dispose) manager.pool_limit = 3 future = manager.submit(initial, pool=pool) - completion = Thread(target=lambda: worker.set_result(object())) + completion = Thread(target=lambda: worker.set_result(object()), daemon=True) try: assert started.wait(5) blocker = manager.submit( @@ -665,9 +816,14 @@ def result_queued(): assert not disposed.is_set() finally: release.set() - if completion.ident is not None: + if completion.ident is None: + manager.close() + completion.start() + completion.join(5) + if completion.is_alive(): + manager.close() completion.join(5) - assert not completion.is_alive() + assert not completion.is_alive() assert disposed.wait(5) assert disposal_context == [('worker', completion.ident)] assert completion.ident not in {thread.ident for thread in manager._threads} @@ -847,7 +1003,7 @@ def handler(record): future.result(5) assert len(submissions) == 1 finally: - physical.set_result(object()) + complete_physical(manager, physical, object()) assert disposed.wait(5) old = future._record def reuse(record): @@ -1119,7 +1275,7 @@ def listener(): operation, key='shared-budget', pool=pool) assert private[0]._record is ordinary._record ordinary.add_done_callback(lambda f: None) - physical.set_result(23) + complete_physical(manager, physical, 23) assert ordinary.result(5) == 23 notification.result(5) with manager._condition: @@ -1137,7 +1293,7 @@ def listener(): finally: allow_join.set() if not physical.done(): - physical.set_result(23) + complete_physical(manager, physical, 23) def test_foreign_callback_uses_ordinary_admission(runtime): @@ -1443,6 +1599,64 @@ def interrupt(*args, **kwargs): manager.join(5) +@pytest.mark.parametrize('installed', [False, True]) +@pytest.mark.parametrize('outcome', ['result', 'exception']) +def test_interrupted_compatibility_attachment_retains_work_until_completion( + runtime, installed, outcome): + from cassandra._management import compatibility_work + manager, dispatch = runtime + physical = Future() + assert physical.set_running_or_notify_cancel() + register = physical.add_done_callback + attaching, finished = Event(), Event() + errors, records = [], [] + def interrupted(callback): + if installed: + register(callback) + attaching.set() + raise KeyboardInterrupt('compatibility attachment interrupted') + physical.add_done_callback = interrupted + def caller(): + try: + with compatibility_work(manager) as attach: + with manager._condition: + records.extend(manager._records) + attach(physical) + except BaseException as exc: + errors.append(exc) + finally: + finished.set() + thread = Thread(target=caller, daemon=True) + thread.start() + try: + assert attaching.wait(5) + with manager._condition: + assert len(records) == 1 + assert records[0].physical == 1 + assert not finished.wait(0.05) + finally: + if outcome == 'result': + physical.set_result(23) + else: + physical.set_exception(ValueError('physical work failed')) + thread.join(5) + assert not thread.is_alive() + # Keep the original-source failure bounded without masking the defect. + with manager._condition: + debt_after_completion = sum(record.physical for record in records) + credits_after_completion = manager.counts()['ordinary'] + for record in records: + if record.physical: + manager._finish(record) + record.physical -= 1 + manager._release_locked(record) + assert len(errors) == 1 + assert isinstance(errors[0], KeyboardInterrupt) + assert debt_after_completion == credits_after_completion == 0 + assert records[0].physical == 0 + assert manager.counts()['ordinary'] == 0 + + @pytest.mark.parametrize('pool', [None, 'logical-pool']) @pytest.mark.parametrize('ending', ['normal', 'close', 'fatal']) def test_owner_callback_handoff_yields_reactor_while_caller_callback_runs(