diff --git a/cassandra/_management.py b/cassandra/_management.py new file mode 100644 index 0000000000..7bb9e245de --- /dev/null +++ b/cassandra/_management.py @@ -0,0 +1,1369 @@ +# Copyright ScyllaDB, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0 + +"""Bounded management execution contexts. + +No query-path work belongs here. + +Owners handle short, nonblocking messages. Physical work belongs to +an existing executor and lifecycle notifications to the serial callback +lane. Charged future callback batches run on the selected reactor. After +a batch retires, external registrations may drain through the same +serialized adapter on their caller. C/P attach to a current charged +batch. State not yet migrated still requires its original publication +locks. +""" + +from collections import deque +from concurrent.futures import Future, TimeoutError +from contextlib import contextmanager +from dataclasses import dataclass +from functools import wraps +from itertools import chain +import logging +import os +from threading import ( + Condition, Event, Lock, Thread, current_thread, enumerate as active_threads, + local) +import time + +from cassandra import DriverException + +log = logging.getLogger(__name__) +_context = local() +_PENDING = object() + + +class ManagementClosed(DriverException): + """The original management lifetime has ended.""" + + +class ManagementSuperseded(DriverException): + """The operation was superseded without losing its waiters.""" + + +class ManagementOverloaded(DriverException): + """No reservation is available for a nonblocking submission.""" + + +class ManagementDeliveryFailed(DriverException): + """Delivery failed while native outcomes remain terminal.""" + + +class ManagementContextError(DriverException): + """A blocking management call would wait on its own context.""" + + +def execution_context(): + return getattr(_context, 'kind', None) + + +def assert_can_wait(): + if execution_context() in ('cluster', 'pool', 'bridge'): + raise ManagementContextError( + 'Blocking management call on an owner or reactor') + + +def _future_access(method): + @wraps(method) + def access(self, *args, **kwargs): + self._record.runtime._check_pid() + previous = getattr(_context, 'future', None) + _context.future = self + try: + return method(self, *args, **kwargs) + finally: + _context.future = previous + return access + + +class _FutureCondition(Condition): + """Keep bulk-wait rejection inside its cleanup context.""" + + def __init__(self, runtime): + super().__init__() + self._runtime = runtime + self._acquire = self.acquire + self._release = self.release + self.acquire = self._checked_acquire + self.release = self._checked_release + + def _checked_acquire(self, *args, **kwargs): + # Stdlib bulk acquisition does not unwind an acquisition failure. + # Skip inherited locks; the Future state guard rejects inside the + # bulk context, whose exit then releases earlier fresh locks. + if self._runtime._pid != os.getpid(): + return False + return self._acquire(*args, **kwargs) + + def _checked_release(self): + if self._runtime._pid == os.getpid(): + self._release() + return None + + def __enter__(self): + self._runtime._check_pid() + return super().__enter__() + + +@contextmanager +def _entered(kind, record=None): + previous = (getattr(_context, 'kind', None), + getattr(_context, 'record', None)) + _context.kind, _context.record = kind, record + try: + yield + finally: + _context.kind, _context.record = previous + + +@dataclass(frozen=True) +class Envelope: + operation: int + kind: str + origin: object + lifetime: object + identity: tuple + + +class OperationFuture(Future): + """Native futures complete without running callbacks on owners.""" + + def __init__(self, record): + super().__init__() + self._record = record + self._condition = _FutureCondition(record.runtime) + self._budget = 'ordinary' + self._delivery_error = None + self._callback_cursor = 0 + self._callbacks_running = False + self._callback_waiters = set() + self._delivery_record = record + self._publishing = False + + @property + def _state(self): + self._record.runtime._check_pid() + state = self.__state + if (state not in ('CANCELLED_AND_NOTIFIED', 'FINISHED') and + getattr(_context, 'future', None) is not self): + # Bulk waits read state after acquiring ALL future conditions. + # Reject before waiter installation, where their context unwinds + # every lock. Own nonblocking methods use scoped access above. + self._assert_can_wait() + return state + + @_state.setter + def _state(self, value): + self.__state = value + + @_future_access + def __repr__(self): + return super().__repr__() + + @_future_access + def done(self): + self._record.runtime._check_pid() + return super().done() + + @_future_access + def running(self): + self._record.runtime._check_pid() + return super().running() + + @_future_access + def cancelled(self): + self._record.runtime._check_pid() + return super().cancelled() + + @_future_access + def result(self, timeout=None): + self._record.runtime._check_pid() + if not self.done(): + self._assert_can_wait() + return super().result(timeout) + + @_future_access + def exception(self, timeout=None): + self._record.runtime._check_pid() + if not self.done(): + self._assert_can_wait() + return super().exception(timeout) + + def _assert_can_wait(self): + assert_can_wait() + current = getattr(_context, 'record', None) + if (execution_context() == 'callback' and current is not None and + current.runtime is self._record.runtime and + self._record.envelope.kind == 'notification'): + raise ManagementContextError( + 'Notification wait would block its own callback lane') + + @_future_access + def cancel(self): + self._record.runtime._check_pid() + cancelled = super().cancel() + if cancelled: + # Future.cancel() alone leaves concurrent.futures.wait() asleep. + with self._condition: + if self._state == 'CANCELLED': + self.set_running_or_notify_cancel() + return cancelled + + @_future_access + def set_result(self, result): + return super().set_result(result) + + @_future_access + def set_exception(self, exception): + return super().set_exception(exception) + + @_future_access + def set_running_or_notify_cancel(self): + return super().set_running_or_notify_cancel() + + def _invoke_callbacks(self): + if not self._publishing: + self._record.runtime._schedule_callbacks(self) + + def add_done_callback(self, fn): + self._record.runtime._check_pid() + if self._delivery_error is not None: + raise self._delivery_error + if self._budget in ('reentry', 'aggregate'): + raise ManagementContextError( + 'Private synchronous waiters have no callback batch') + while True: + record = self._delivery_record + owner = execution_context() in ('cluster', 'pool') + target = (getattr(_context, 'record', None) + if record.released and owner else record) + if target is None: + raise ManagementContextError( + 'Owner callback has no reservation') + with target.runtime._condition: + with self._condition: + if record is not self._delivery_record: + continue + if owner and record.released and target is record: + # Retirement can happen after target selection but + # before acquiring the runtime condition. + continue + if self._delivery_error is not None: + raise self._delivery_error + if record.released and owner: + if target.released or not target.budgets['ordinary']: + raise ManagementContextError( + 'Owner callback has no reservation') + self._delivery_record = target + target.foreign_futures.add(self) + self._done_callbacks.append(fn) + terminal = self.done() + if not terminal: + return + if not target.released: + target.runtime._queue_delivery_locked(target) + return + # Released callbacks share a caller-side drainer; a concurrent + # registrant may leave its callback to the active drainer. + self._drain_callbacks() + return + + def _drain_callbacks(self, record=None): + """Drain the adapter's serialized callback stream outside C/P.""" + with self._condition: + if self._callbacks_running: + if record is not None: + self._callback_waiters.add(record) + return False + self._callbacks_running = True + original = self._delivery_record + while True: + with self._condition: + if (original is not self._delivery_record or + self._callback_cursor == len(self._done_callbacks)): + self._callbacks_running = False + resume, self._callback_waiters = ( + self._callback_waiters, set()) + if original is not self._delivery_record: + # An owner took responsibility for callbacks appended + # during a caller-side drain. Hand them back to its + # reactor batch after this callback has returned. + resume.add(self._delivery_record) + break + callback = self._done_callbacks[self._callback_cursor] + self._callback_cursor += 1 + try: + callback(self) + except BaseException: + log.exception('Error in management completion callback') + for waiting in resume: + waiting.runtime._resume_delivery(waiting) + return True + + +class _DeliveryTicket: + """One cancellable completion ticket per charged record.""" + + def __init__(self, callback): + self._lock = Lock() + self._callback = callback + self._active = False + + def __call__(self): + with self._lock: + callback = self._callback + if callback is None: + return + self._active = True + try: + callback() + finally: + with self._lock: + self._callback = None + self._active = False + + def cancel(self): + with self._lock: + if self._active: + return False + self._callback = None + return True + + +class _Record: + def __init__(self, runtime, envelope, handler, pool, key): + self.runtime, self.envelope, self.handler = runtime, envelope, handler + self.pool, self.key = pool, key + self.budgets = dict.fromkeys(('ordinary', 'reentry', 'aggregate'), 0) + self.private_released = False + self.futures = [] + self.foreign_futures = set() + self.physical = 0 + self.mailbox_pending = 0 + self.started = Event() + self.handler_done = False + self.delivery = False + self.delivery_ticket = None + self.delivery_again = False + self.notification = False + self.released = False + self.dispatched = False + self.result = None + self.error = None + + +class _LifetimeHandle: + def __init__(self, runtime): + self.runtime = runtime + self.closed = Event() + + +class ManagementRuntime: + """One C owner, two P runners and one callback lane per Cluster. + + ``dispatch`` schedules a callable on the shared reactor, never + inline on C/P. It must remain usable until ``finalized``. A + controlled dispatcher may be supplied by unit tests; production + obtains it from its reactor adapter. + + Catastrophic loss of that validated shared service closes + admission and records ``completion_error``. Native terminal + outcomes remain immutable; undeliverable application callbacks + are explicitly marked failed while physical cleanup continues. + Normal startup, idle and close use the adapter's lifetime + guarantee rather than this failure policy. + """ + + def __init__(self, dispatch, limit=1024, pool_limit=64, reentry_window=64, + on_finalized=None): + self._dispatch = dispatch + self.limit, self.pool_limit = limit, pool_limit + self.reentry_window = reentry_window + self.lifetime = object() + self._pid = os.getpid() + self._condition = Condition() + self._cluster = deque() + self._pools = {} + self._ready = deque() + self._active_pools = set() + self._notifications = deque() + self._records = set() + self._keys = {} + self._pool_credits = {} + self._credits = 0 + self._reentry_credits = 0 + self._reentry_active = False + self._sequence = 0 + self._started = False + self._closing = False + self.completion_error = None + self.cleanup_error = None + self._threads = [] + self._exited = 0 + self._cleanup_obligations = 0 + self._finalizing = False + self._on_finalized = on_finalized + self._health = {} + self._unknown_refresh = None + self._lifetimes = set() + self.closed = Event() + self.finalized = Event() + self.high_water = 0 + self.high_waters = dict.fromkeys( + ('ordinary', 'reentry', 'aggregate', 'health', 'close', 'delivery', + 'mailbox'), 0) + + def _observe_counts_locked(self): + counts = { + 'ordinary': self._credits, 'reentry': self._reentry_credits, + 'aggregate': int(self._reentry_active), + 'health': len(self._health), + 'close': len(self._lifetimes) + self._cleanup_obligations, + 'delivery': sum(record.delivery for record in self._records), + 'mailbox': sum(record.mailbox_pending for record in self._records)} + for kind, count in counts.items(): + self.high_waters[kind] = max(self.high_waters[kind], count) + return counts + + def counts(self): + self._check_pid() + with self._condition: + return self._observe_counts_locked() + + def start(self): + self._check_pid() + with self._condition: + self._check_pid() + if self._closing: + raise ManagementClosed('Management lifetime is closed') + if self._started: + return + self._started = True + targets = [('cluster', self._run_cluster), + ('pool', self._run_pool), ('pool', self._run_pool), + ('callback', self._run_notifications)] + try: + for kind, target in targets: + thread = Thread(target=self._run, args=(kind, target), + name='cassandra-management-' + kind, + daemon=True) + try: + thread.start() + except BaseException: + # A start interrupted after native launch still owns a + # runner, even before bootstrap publishes its ident. + if thread in active_threads(): + self._threads.append(thread) + raise + self._threads.append(thread) + except BaseException: + self._closing = True + self._condition.notify_all() + # Started threads exit once this startup lock is released. + raise + + def _check_pid(self): + if self._pid != os.getpid(): + raise ManagementClosed( + 'Inherited management runtime cannot run after fork') + + def assert_owner(self, pool=None): + self._check_pid() + record = getattr(_context, 'record', None) + if (execution_context() != ('cluster' if pool is None else 'pool') or + record is None or record.runtime is not self or + record.pool is not pool): + raise ManagementContextError( + 'Management state accessed outside its owner') + + def _capacity(self, pool, budget): + if budget == 'aggregate': + return not self._reentry_active + if budget == 'reentry': + return self._reentry_credits < self.reentry_window + return (self._credits < self.limit and + (pool is None or + self._pool_credits.get(pool, 0) < self.pool_limit)) + + def submit(self, handler, *, pool=None, key=None, identity=(), + kind='operation', origin=None, wait=False, timeout=None, + budget='ordinary', reserve=0): + if budget not in ('ordinary', 'reentry', 'aggregate'): + raise ManagementContextError('Unknown management admission budget') + if reserve not in (0, 1) or (reserve and key is not None): + raise ManagementContextError( + 'Physical fanout requires separate admitted records') + if wait: + assert_can_wait() + self.start() + deadline = None if timeout is None else time.monotonic() + timeout + with self._condition: + self._check_pid() + while not self._closing and not self._capacity(pool, budget): + if not wait: + raise ManagementOverloaded( + 'Management admission capacity exhausted') + current = getattr(_context, 'record', None) + if (execution_context() == 'callback' and + current is not None and current.runtime is self and + budget == 'ordinary'): + raise ManagementContextError( + 'Callback admission cannot wait behind its own credit') + remaining = (None if deadline is None + else deadline - time.monotonic()) + if remaining is not None and remaining <= 0: + raise TimeoutError() + self._condition.wait(remaining) + if self._closing: + raise ManagementClosed('Management lifetime is closed') + if budget == 'aggregate': + self._reentry_active = True + elif budget == 'reentry': + self._reentry_credits += 1 + else: + self._credits += 1 + if pool is not None: + self._pool_credits[pool] = ( + self._pool_credits.get(pool, 0) + 1) + self.high_water = max(self.high_water, self._credits) + coalesce = (pool, key) if key is not None else None + record = self._keys.get(coalesce) if coalesce is not None else None + if record is None: + self._sequence += 1 + envelope = Envelope(self._sequence, kind, origin, + self.lifetime, tuple(identity)) + record = _Record(self, envelope, handler, pool, coalesce) + record.physical = reserve + self._records.add(record) + if coalesce is not None: + self._keys[coalesce] = record + self._enqueue_locked(record, lambda: self._handle(record)) + future = OperationFuture(record) + future._budget = budget + record.futures.append(future) + record.budgets[budget] += 1 + self._observe_counts_locked() + self._condition.notify_all() + return future + + def _enqueue_locked(self, record, callback, reserved=False): + record.mailbox_pending += 1 + if record.pool is None: + self._cluster.append((record, callback, reserved)) + else: + queue = self._pools.setdefault(record.pool, deque()) + queue.append((record, callback, reserved)) + if (record.pool not in self._active_pools and + record.pool not in self._ready): + self._ready.append(record.pool) + self._condition.notify_all() + + def _handle(self, record): + if record.handler_done: + return + record.dispatched = True + record.started.set() + try: + outcome = record.handler(record) + if outcome is _PENDING: + return + except BaseException as exc: + self._finish(record, error=exc) + else: + self._finish(record, outcome) + + def _finish(self, record, result=None, error=None): + with self._condition: + if record.handler_done: + return + record.handler_done, record.result, record.error = ( + True, result, error) + record.started.set() + if record.key is not None: + self._keys.pop(record.key, None) + for future in record.futures: + with future._condition: + if not future.done(): + # Keep cancellation and terminal publication atomic, + # but defer the cross-future delivery scan until this + # condition is released. Stdlib bulk waits acquire + # these conditions in a different order. + future._publishing = True + try: + if error is None: + future.set_result(result) + else: + future.set_exception(error) + finally: + future._publishing = False + self._queue_delivery_locked(record) + self._release_locked(record) + + def supersede(self, future): + self._check_pid() + self._finish(future._record, + error=ManagementSuperseded('Operation superseded')) + + def work(self, record, executor, function, *, accept=lambda value: value, + dispose=lambda value: None): + """Reserve result delivery before physical work, on C/P only. + + The executor runs work outside C/P. Its completion retains the worker + until the owner accepts or rejects the value, then disposes a rejected + value there. ``accept`` is a short native state transition, serialized + with shutdown and supersession; it must not block or run user code. + """ + self.assert_owner(record.pool) + with self._condition: + if (getattr(_context, 'record', None) is not record or + record.released or record.handler_done or record.physical): + raise ManagementContextError( + 'Physical work requires its current unused ' + 'result reservation') + record.physical = 1 + self._condition.notify_all() + returned_to_owner = Event() + adopted = False + received = False + work_started = False + submission_failed = False + + def completed(future): + with _entered('worker', record): + try: + value, error = future.result(), None + except BaseException as exc: + value, error = None, exc + receive(value, error) + + def receive(value, error): + nonlocal received + with self._condition: + if received: + return + received = True + if not self._closing: + self._enqueue_locked( + record, lambda: returned(value, error), reserved=True) + queued = True + else: + queued = False + if queued and error is None: + # Only the short native result transition is awaited. Neither + # application callback delivery nor notification progress is + # part of this handoff. + returned_to_owner.wait() + # Keep physical ownership until adoption or disposal completes, + # including results rejected by a closed owner mailbox. + try: + if error is None and not adopted: + dispose(value) + except BaseException: + log.exception('Error disposing late management result') + finally: + with self._condition: + record.physical -= 1 + self._release_locked(record) + + def returned(value, error): + nonlocal adopted + try: + with self._condition: + if self._closing or record.handler_done: + return + if error is not None: + self._finish(record, error=error) + else: + try: + accepted = accept(value) + except BaseException as exc: + self._finish(record, error=exc) + else: + adopted = True + self._finish(record, accepted) + finally: + returned_to_owner.set() + + def physical_work(): + nonlocal work_started + with self._condition: + if submission_failed: + # submit() can queue a callable before raising. Its + # reservation was retired only if execution never began. + return None + work_started = True + with _entered('worker', record): + try: + value = function() + except BaseException as exc: + receive(None, exc) + raise + else: + # Hand off before completing the executor Future. A late + # callback registration can run inline on C/P, but by then + # this worker already owns adoption/disposal responsibility. + receive(value, None) + return value + + try: + physical = executor.submit(physical_work) + except BaseException as exc: + with self._condition: + submission_failed = True + self._finish(record, error=exc) + if not work_started: + receive(None, exc) + return _PENDING + try: + physical.add_done_callback(completed) + except BaseException as exc: + self._finish(record, error=exc) + if physical.cancel(): + # No worker will discharge a cancellation before execution. + # A callback installed before an interrupted registration may + # also deliver it; receive claims the obligation exactly once. + receive(None, exc) + return _PENDING + + def notify(self, callback, *, wait=False, timeout=None, budget='ordinary'): + self._check_pid() + def admit(record): + with self._condition: + if self._closing or record.handler_done or record.released: + return _PENDING + record.notification = True + self._notifications.append((record, callback)) + self._condition.notify_all() + return _PENDING + return self.submit(admit, wait=wait, timeout=timeout, budget=budget, + kind='notification') + + @contextmanager + def synchronous_reentry(self): + """Reserve one L aggregate and a bounded prerequisite window.""" + self._check_pid() + assert_can_wait() + current = getattr(_context, 'record', None) + if (execution_context() != 'callback' or current is None or + current.runtime is not self): + yield 'ordinary' + return + with self._condition: + if self._reentry_active: + raise ManagementContextError( + 'Recursive native reentry aggregate') + if self._closing: + raise ManagementClosed('Management lifetime is closed') + self._reentry_active = True + try: + yield 'reentry' + finally: + self._check_pid() + with self._condition: + self._reentry_active = False + self._condition.notify_all() + + def register_health(self, incarnation): + self.start() + with self._condition: + if self._closing: + raise ManagementClosed('Management lifetime is closed') + self._health.setdefault( + id(incarnation), (incarnation, [None, None, False])) + self._observe_counts_locked() + + def observe_negative(self, incarnation, apply, notification=None, + removed=False): + """Apply incarnation health despite ordinary admission load.""" + self._check_pid() + with self._condition: + if self._closing: + return False + entry = self._health.get(id(incarnation)) + slot = (entry[1] if entry is not None and entry[0] is incarnation + else None) + if slot is None: + self._unknown_refresh = True + return False + # An already pending removal is terminal for this incarnation. + if slot[2] and not removed: + return True + slot[:] = [apply, notification, removed] + self._condition.notify_all() + return True + + def take_refresh_demand(self): + self.assert_owner() + with self._condition: + demand, self._unknown_refresh = self._unknown_refresh, None + return bool(demand) + + def unregister_health(self, incarnation): + self._check_pid() + with self._condition: + self._health.pop(id(incarnation), None) + + def register_lifetime(self): + """Return a caller-owned handle for an active lifetime.""" + self._check_pid() + with self._condition: + if self._closing: + raise ManagementClosed('Management lifetime is closed') + handle = _LifetimeHandle(self) + self._lifetimes.add(handle) + self._observe_counts_locked() + return handle + + def close_lifetime(self, handle, retire): + self._check_pid() + if (not isinstance(handle, _LifetimeHandle) or + handle.runtime is not self): + raise ManagementContextError( + 'Close handle belongs to another runtime') + with self._condition: + closed = handle.closed + if closed.is_set(): + return closed + # Retirement is native, short and must not run application code. + retire() + closed.set() + self._lifetimes.discard(handle) + return closed + + def _schedule_callbacks(self, future): + with self._condition: + record = future._record + self._queue_delivery_locked(record) + + def _delivery_failed_locked(self, record, error): + for future in chain(record.futures, record.foreign_futures): + with future._condition: + future._delivery_error = error + future._done_callbacks.clear() + future._callback_cursor = 0 + + def _queue_delivery_locked(self, record): + if record.delivery: + return + if not any( + future.done() and + future._callback_cursor < len(future._done_callbacks) + for future in chain(record.futures, record.foreign_futures)): + return + if self.completion_error is not None: + self._delivery_failed_locked(record, self.completion_error) + return + record.delivery = True + self._observe_counts_locked() + self._dispatch_delivery_locked(record) + + def _resume_delivery(self, record): + """Resume a deferred batch when its drainer returns.""" + self._check_pid() + with self._condition: + if record.released or not record.delivery: + return + if record.delivery_ticket is not None: + record.delivery_again = True + else: + self._dispatch_delivery_locked(record) + + def _dispatch_delivery_locked(self, record): + if self.completion_error is not None: + self._delivery_failed_locked(record, self.completion_error) + record.delivery = False + self._release_locked(record) + return + record.delivery_ticket = _DeliveryTicket(lambda: self._deliver(record)) + try: + self._dispatch(record.delivery_ticket) + except BaseException as exc: + record.delivery = False + error = ManagementDeliveryFailed( + 'Completion reactor rejected management delivery') + error.__cause__ = exc + self.completion_error = error + for pending in tuple(self._records): + self._delivery_failed_locked(pending, error) + ticket = pending.delivery_ticket + if ticket is None or ticket.cancel(): + pending.delivery = False + pending.delivery_ticket = None + self._release_locked(pending) + # Infrastructure failure closes admission. Native outcomes already + # published are immutable; remaining waiters receive shutdown. + # Inactive queued tickets are abandoned; an active delivery keeps + # its credit until it returns. Undeliverable callbacks are never + # retried or executed as application code on an owner. + log.exception('Management reactor completion delivery failed') + self.close() + + def _deliver(self, record): + with _entered('bridge', record): + with self._condition: + record.delivery_again = False + while True: + with self._condition: + futures = tuple(chain(record.futures, + record.foreign_futures)) + stalled = False + for future in futures: + if future.done(): + if not future._drain_callbacks(record): + stalled = True + with self._condition: + if any( + future.done() and future._callback_cursor < + len(future._done_callbacks) + for future in chain(record.futures, + record.foreign_futures)): + if stalled: + # Another caller is executing a callback. Keep the + # charged batch and yield I/O until its handoff. + record.delivery_ticket = None + if record.delivery_again: + record.delivery_again = False + self._dispatch_delivery_locked(record) + return + continue + record.delivery = False + record.delivery_ticket = None + record.delivery_again = False + self._release_locked(record) + return + + def _release_locked(self, record): + if (record.released or not record.handler_done or record.physical or + record.mailbox_pending): + return + # Coalesced waiters share physical work, but private native completion + # must release its own window independently of ordinary callback debt. + if not record.private_released: + if record.budgets['aggregate']: + self._reentry_active = False + self._reentry_credits -= record.budgets['reentry'] + record.private_released = True + self._condition.notify_all() + if record.delivery or record.notification: + return + record.released = True + count = record.budgets['ordinary'] + if count: + self._credits -= count + if record.pool is not None: + remaining = self._pool_credits[record.pool] - count + if remaining: + self._pool_credits[record.pool] = remaining + else: + del self._pool_credits[record.pool] + self._records.discard(record) + record.handler = None + self._condition.notify_all() + self._maybe_finalized_locked() + + def _run(self, kind, target): + with _entered(kind): + try: + target() + finally: + with self._condition: + self._exited += 1 + self._maybe_finalized_locked() + + def _health_ready(self): + return any(slot[0] is not None or + (slot[1] is not None and self._capacity(None, 'ordinary')) + for _, slot in self._health.values()) + + def _run_cluster(self): + while True: + with self._condition: + while (not self._cluster and not self._closing and + not self._health_ready()): + self._condition.wait() + if self._closing and not self._cluster: + return + health = next((slot for _, slot in self._health.values() + if slot[0] is not None), None) + if health is not None: + callback, health[0] = health[0], None + self._sequence += 1 + record = _Record( + self, Envelope(self._sequence, 'health', None, + self.lifetime, ()), None, None, None) + self._records.add(record) + native_health = True + else: + pending = next((slot for _, slot in self._health.values() + if slot[1] is not None and + self._capacity(None, 'ordinary')), None) + if pending is not None: + notification, pending[1] = pending[1], None + self.notify(notification) + notification = pending = None + continue + record, callback, reserved = self._cluster.popleft() + record.mailbox_pending -= 1 + native_health = False + self._execute(record, callback, 'cluster') + if native_health: + self._finish(record) + with self._condition: + self._release_locked(record) + record = callback = health = None + + def _run_pool(self): + while True: + with self._condition: + while not self._ready and not self._closing: + self._condition.wait() + if self._closing and not self._ready: + return + pool = self._ready.popleft() + self._active_pools.add(pool) + record, callback, reserved = self._pools[pool].popleft() + record.mailbox_pending -= 1 + self._execute(record, callback, 'pool') + with self._condition: + self._active_pools.remove(pool) + self._release_locked(record) + if self._pools[pool]: + self._ready.append(pool) + else: + del self._pools[pool] + self._condition.notify_all() + record = callback = pool = None + + def _execute(self, record, callback, kind): + with _entered(kind, record): + try: + callback() + except BaseException as exc: + self._finish(record, error=exc) + + def _run_notifications(self): + while True: + with self._condition: + while not self._notifications and not self._closing: + self._condition.wait() + if self._closing: + return + record, callback = self._notifications.popleft() + with _entered('callback', record): + try: + value = callback() + except BaseException as exc: + self._finish(record, error=exc) + else: + self._finish(record, value) + finally: + # Drop a bound adapter before publishing released credits; + # an idle lane must not root its last coordinator. + callback = None + with self._condition: + record.notification = False + self._release_locked(record) + record = callback = value = None + + def retain_cleanup(self): + """Retain one cleanup obligation independently of load.""" + self._check_pid() + with self._condition: + if self._finalizing: + raise ManagementClosed('Management cleanup lifetime has ended') + self._cleanup_obligations += 1 + self._observe_counts_locked() + completed = Event() + def release(): + self._check_pid() + with self._condition: + if not completed.is_set(): + completed.set() + self._cleanup_obligations -= 1 + self._maybe_finalized_locked() + return release + + def close(self, wait=False): + self._check_pid() + if wait: + assert_can_wait() + with self._condition: + self._check_pid() + if not self._closing: + self._closing = True + for record in tuple(self._records): + self._finish(record, error=ManagementClosed( + 'Management lifetime ended')) + for record, _ in self._notifications: + record.notification = False + self._release_locked(record) + self._notifications.clear() + def retained_results(queue): + retained = deque() + for item in queue: + if item[2]: + retained.append(item) + else: + item[0].mailbox_pending -= 1 + self._release_locked(item[0]) + return retained + self._cluster = retained_results(self._cluster) + for pool, queue in tuple(self._pools.items()): + retained = retained_results(queue) + if retained or pool in self._active_pools: + self._pools[pool] = retained + else: + del self._pools[pool] + self._ready = deque(pool for pool in self._ready + if pool in self._pools) + self._keys.clear() + self._health.clear() + for handle in self._lifetimes: + handle.closed.set() + self._lifetimes.clear() + self._condition.notify_all() + self.closed.set() + self._maybe_finalized_locked() + if wait and execution_context() not in ('callback', 'worker'): + self.closed.wait() + return self.closed + + def _maybe_finalized_locked(self): + if (self._closing and not self._records and + not self._cleanup_obligations + and self._exited == len(self._threads) + and not self._finalizing): + self._finalizing = True + if self._on_finalized is None: + self.finalized.set() + else: + try: + self._on_finalized(self.finalized.set) + except BaseException: + log.exception( + 'Error detaching management completion channel') + self.finalized.set() + + def join(self, timeout=None): + self._check_pid() + assert_can_wait() + if execution_context() in ('callback', 'worker'): + raise ManagementContextError('Cannot join own management resource') + if not self.finalized.wait(timeout): + raise TimeoutError() + for thread in self._threads: + if thread is not current_thread(): + thread.join(timeout) + + +class ReactorChannel: + """Borrow dispatch without owning the process-global I/O service. + + Libev holds its I/O runner live until the final admitted delivery + drains; thread startup failures consequently precede runtime + admission. Closing one channel releases only its borrow, never + another Cluster's reactor. + """ + + def __init__(self, connection_class): + dispatch = getattr(connection_class, '_management_dispatch', None) + in_context = getattr(connection_class, '_management_in_context', None) + if dispatch is None or in_context is None: + raise ManagementContextError( + 'Connection class has no management completion adapter') + self._closed = False + self._dispatch = dispatch + self._release = None + connection_class.initialize_reactor() + borrow = getattr(connection_class, '_management_borrow', None) + if borrow is not None: + self._release = borrow() + elif not in_context(): + ready = Event() + dispatch(ready.set) + if not ready.wait(5): + raise ManagementContextError( + 'Management reactor channel startup did not complete') + + def dispatch(self, callback): + if self._closed: + raise ManagementClosed('Management completion channel detached') + self._dispatch(callback) + + def close(self, completed=lambda: None): + if self._closed: + completed() + return + self._closed = True + release, self._release = self._release, None + self._dispatch = None + if release is None: + completed() + else: + release(completed) + + +def assert_synchronous_api(connection_class): + assert_can_wait() + in_context = getattr(connection_class, '_management_in_context', None) + if in_context is not None and in_context(): + raise ManagementContextError( + 'Blocking management call on a reactor thread') + + +@contextmanager +def physical_work_context(cleanup=False): + """Mark physical work; cleanup may finish child closes inline.""" + previous = getattr(_context, 'cleanup', False) + _context.cleanup = cleanup + try: + with _entered('worker'): + yield + finally: + _context.cleanup = previous + + +def performing_cleanup(): + return getattr(_context, 'cleanup', False) + + +class _CompatibilityCall: + """Admit legacy phases while their state writers still use locks. + + The calling stack performs existing blocking work. A reserved + physical obligation holds its credit; C never waits for that + stack. Policy wrappers release a completed phase before arbitrary + policy reentry, then acquire the next phase after the policy + returns. Later migrations replace these bodies with native + asynchronous owner continuations. + """ + + def __init__(self, runtime): + self.runtime = runtime + current = getattr(_context, 'record', None) + self.budget = ('aggregate' if execution_context() == 'callback' and + current is not None and current.runtime is runtime + else 'ordinary') + self.record = None + + def acquire(self): + if self.record is not None: + return + if self.budget == 'aggregate' and self.runtime._reentry_active: + raise ManagementContextError('Recursive native reentry aggregate') + future = self.runtime.submit(lambda record: _PENDING, wait=True, + budget=self.budget, reserve=1) + self.record = future._record + try: + self.record.started.wait() + if future.done(): + future.result() + except BaseException: + self.release() + raise + + def release(self): + record, self.record = self.record, None + if record is not None: + with self.runtime._condition: + self.runtime._finish(record) + record.physical -= 1 + self.runtime._release_locked(record) + + +@contextmanager +def compatibility_call(runtime): + previous = getattr(_context, 'call', None) + call = _CompatibilityCall(runtime) + call.acquire() + _context.call = call + try: + yield + finally: + call.release() + _context.call = previous + + +def policy_call(method): + """Run user policy between released compatibility phases.""" + @wraps(method) + def call_policy(*args, **kwargs): + call = getattr(_context, 'call', None) + if call is None: + return method(*args, **kwargs) + call.release() + _context.call = None + try: + result = method(*args, **kwargs) + finally: + _context.call = call + call.acquire() + return result + return call_policy + + +@contextmanager +def compatibility_work(runtime): + """Charge physical work; L fanout uses its independent window.""" + if runtime is None: + # Internal tests construct bare Sessions with __new__, without a + # registered production lifetime or management admission contract. + yield lambda work: None + return + assert_can_wait() + call = getattr(_context, 'call', None) + same_call = call is not None and call.runtime is runtime + budget = ('reentry' if same_call and call.budget == 'aggregate' + else 'ordinary') + if same_call and budget == 'ordinary': + # Admission cannot retain a phase's credit while waiting for a child + # record in a full window, or retain a Session lock needed by workers. + call.release() + future = runtime.submit(lambda record: _PENDING, wait=same_call, + budget=budget, reserve=1) + record = future._record + attached = False + completed = False + + def complete(work): + nonlocal completed + with runtime._condition: + if completed: + return + completed = True + runtime._finish(record) + record.physical -= 1 + runtime._release_locked(record) + + def attach(work): + nonlocal attached + if (work is not None and + callable(getattr(work, 'add_done_callback', None))): + try: + attached = True + work.add_done_callback(complete) + except BaseException: + # Registration may have been interrupted before or after + # installation. Keep this synchronous caller responsible for + # the physical work until it actually settles; the callback + # and this fallback share the same once-only retirement. + while not work.done(): + try: + work.result() + except BaseException: + pass + complete(work) + raise + try: + record.started.wait() + if future.done(): + future.result() + yield attach + finally: + if not attached: + complete(None) + if same_call and budget == 'ordinary': + call.acquire() diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py new file mode 100644 index 0000000000..6853bd1989 --- /dev/null +++ b/tests/unit/test_management_runtime.py @@ -0,0 +1,2079 @@ +from collections import deque +from concurrent.futures import ( + Future, ThreadPoolExecutor, as_completed, wait, TimeoutError) +from threading import Barrier, Event, Lock, Thread, get_ident + +import pytest + +from cassandra._management import ( + ManagementClosed, ManagementContextError, ManagementOverloaded, + ManagementRuntime, execution_context) + + +class Dispatcher: + def __init__(self): + self.queue = deque() + self.lock = Lock() + + def __call__(self, callback): + with self.lock: + self.queue.append(callback) + + def drain(self): + while True: + with self.lock: + if not self.queue: + return + callback = self.queue.popleft() + callback() + + +def complete_physical(manager, future, value): + """Bound a controlled Future's synchronous owner-handoff callback.""" + errors = [] + def complete(): + try: + future.set_result(value) + except BaseException as exc: + errors.append(exc) + thread = Thread(target=complete, daemon=True) + thread.start() + thread.join(5) + if thread.is_alive(): + manager.close() + thread.join(5) + assert not thread.is_alive(), 'physical result handoff did not finish' + assert not errors + + +@pytest.fixture +def runtime(): + dispatch = Dispatcher() + runtime = ManagementRuntime(dispatch, limit=4, pool_limit=2) + yield runtime, dispatch + runtime.close() + dispatch.drain() + runtime.join(5) + + +def test_fifo_one_owner_and_exception_progress(runtime): + manager, dispatch = runtime + observed = [] + owner = [] + def handler(record): + manager.assert_owner() + observed.append(record.envelope.operation) + owner.append(get_ident()) + return record.envelope.operation + futures = [manager.submit(handler) for _ in range(4)] + assert [f.result(5) for f in futures] == observed == [1, 2, 3, 4] + assert len(set(owner)) == 1 + def fail(record): + raise ValueError('bad operation') + with pytest.raises(ValueError): + manager.submit(fail).result(5) + assert manager.submit(handler).result(5) == 6 + + +def test_native_completion_preserves_delivery_capacity(runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + def handler(record): + entered.set() + assert release.wait(5) + return 7 + futures = [manager.submit(handler) for _ in range(4)] + order = [] + for i, future in enumerate(futures): + future.add_done_callback( + lambda f, i=i: order.append((i, execution_context()))) + assert entered.wait(5) + release.set() + assert all(f.result(5) == 7 for f in futures) + assert wait(futures, timeout=5).not_done == set() + with manager._condition: + assert manager._credits == 4 + assert len(dispatch.queue) == 4 + for _ in range(20): + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None) + futures[0].add_done_callback( + lambda f: order.append(('extra', execution_context()))) + assert len(dispatch.queue) == 4 + dispatch.drain() + assert order == [(0, 'bridge'), ('extra', 'bridge'), (1, 'bridge'), + (2, 'bridge'), (3, 'bridge')] + assert manager._credits == 0 + + +def test_coalesced_cancel_waiters_and_physical_work(runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + attempts = [] + with ThreadPoolExecutor(1) as executor: + def physical(): + attempts.append(True) + entered.set() + assert release.wait(5) + return 42 + first = manager.submit( + lambda r: manager.work(r, executor, physical), key='same') + assert entered.wait(5) + second = manager.submit( + lambda r: pytest.fail('duplicate handler'), key='same') + assert first is not second + assert first.cancel() + assert wait([first], timeout=1).not_done == set() + with pytest.raises(TimeoutError): + second.result(0) + release.set() + assert second.result(5) == 42 + assert attempts == [True] + + +def test_late_result_disposal_does_not_need_admission(runtime): + manager, dispatch = runtime + entered, release, disposed = Event(), Event(), Event() + resource = object() + with ThreadPoolExecutor(1) as executor: + def physical(): + entered.set() + assert release.wait(5) + return resource + future = manager.submit(lambda r: manager.work( + r, executor, physical, + dispose=lambda value: disposed.set() + if value is resource else None)) + assert entered.wait(5) + manager.close() + with pytest.raises(ManagementClosed): + future.result(5) + assert not manager.finalized.is_set() + release.set() + assert disposed.wait(5) + manager.join(5) + + +def test_callback_lane_wait_does_not_block_native_or_reactor_completion( + runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + complete = Event() + future = manager.notify(lambda: (entered.set(), release.wait(5))) + try: + assert entered.wait(5) + result = manager.submit(lambda record: (complete.wait(5), 9)[1]) + seen = [] + result.add_done_callback( + lambda f: seen.append((f.result(), execution_context()))) + complete.set() + assert result.result(5) == 9 + # Native publication precedes enqueue; fence that owner phase before + # observing the independently locked dispatcher. + with manager._condition: + assert len(dispatch.queue) == 1 + dispatch.drain() + assert seen == [(9, 'bridge')] + assert not future.done() + manager.close() + assert future.done() + finally: + complete.set() + release.set() + + +@pytest.mark.parametrize('waiting', [ + lambda future: future.result(), + lambda future: future.exception(), + lambda future: wait([future]), + lambda future: next(as_completed([future]))]) +def test_callback_lane_rejects_its_pending_notification_wait(runtime, waiting): + manager, dispatch = runtime + nested = [] + def callback(): + future = manager.notify(lambda: 17) + nested.append(future) + with pytest.raises(ManagementContextError, match='callback lane'): + waiting(future) + # Ordinary owner operations remain available to synchronous reentry. + with manager.synchronous_reentry() as budget: + assert manager.submit(lambda record: 18, budget=budget).result(5) == 18 + manager.notify(callback).result(5) + assert nested[0].result(5) == 17 + + +def test_owner_wait_fails_and_pool_handlers_are_serial(runtime): + manager, dispatch = runtime + entered, release = Event(), Event() + pool = object() + def hold(record): + manager.assert_owner(pool) + entered.set() + assert release.wait(5) + first = manager.submit(hold, pool=pool) + assert entered.wait(5) + second = manager.submit(lambda record: None, pool=pool) + assert not second.done() + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None, pool=pool) + release.set() + first.result(5) + second.result(5) + def self_wait(record): + other = manager.submit(lambda r: 1) + with pytest.raises(ManagementContextError): + other.result() + manager.submit(self_wait).result(5) + + +def test_executor_rejection_releases_reservation(runtime): + manager, dispatch = runtime + class Rejected: + def submit(self, function): + raise RuntimeError('executor closed') + with pytest.raises(RuntimeError, match='executor closed'): + manager.submit( + lambda r: manager.work(r, Rejected(), lambda: None)).result(5) + with manager._condition: + assert manager._condition.wait_for(lambda: manager._credits == 0, 5) + + +@pytest.mark.parametrize('already_running', [False, True]) +def test_executor_failure_after_queueing_preserves_physical_ownership( + runtime, already_running): + manager, dispatch = runtime + entered, release = Event(), Event() + calls, disposed, queued, errors = [], [], [], [] + resource = object() + def physical(): + calls.append(get_ident()) + entered.set() + assert release.wait(5) + return resource + def run(function): + try: + function() + except BaseException as exc: + errors.append(exc) + class QueuedThenFailed: + def submit(self, function): + queued.append(Thread(target=run, args=(function,), daemon=True)) + if already_running: + queued[0].start() + assert entered.wait(5) + raise RuntimeError('failed after queueing') + future = manager.submit(lambda record: manager.work( + record, QueuedThenFailed(), physical, + dispose=lambda value: disposed.append(value))) + try: + with pytest.raises(RuntimeError, match='after queueing'): + future.result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: future._record.physical == int(already_running), 5) + if already_running: + manager.close() + assert not manager.finalized.is_set() + else: + queued[0].start() + finally: + release.set() + for thread in queued: + if thread.ident is not None: + thread.join(5) + assert not thread.is_alive() + assert not errors + assert bool(calls) == already_running + assert disposed == ([resource] if already_running else []) + with manager._condition: + assert manager._condition.wait_for(lambda: not manager._records, 5) + + +@pytest.mark.parametrize('notification', [False, True]) +def test_callback_rejects_saturated_ordinary_admission(notification): + manager = ManagementRuntime(Dispatcher(), limit=1) + finished = Event() + errors = [] + def callback(): + try: + if notification: + manager.notify(lambda: None, wait=True) + else: + manager.submit(lambda record: None, wait=True) + except BaseException as exc: + errors.append(exc) + finally: + finished.set() + try: + manager.notify(callback) + assert finished.wait(1), 'callback waited behind its own credit' + assert len(errors) == 1 + assert isinstance(errors[0], ManagementContextError) + finally: + manager.close() + manager.join(5) + + +def test_health_and_close_bypass_full_delivery_and_io_capacity(runtime): + manager, dispatch = runtime + # Fill ordinary credits with terminal records awaiting callback delivery. + gate = Event() + futures = [manager.submit(lambda record: gate.wait(5)) for _ in range(4)] + for future in futures: + future.add_done_callback(lambda f: None) + gate.set() + for future in futures: + future.result(5) + assert manager._credits == 4 + incarnation = object() + manager.register_health(incarnation) + native, notified = Event(), Event() + assert manager.observe_negative( + incarnation, native.set, notified.set, removed=True) + assert native.wait(5) + assert not notified.is_set() + # A pending removal cannot be overwritten by a positive/less terminal + # event. + assert manager.observe_negative( + incarnation, lambda: pytest.fail('old health')) + child_one = manager.register_lifetime() + child_two = manager.register_lifetime() + effects = [] + one = manager.close_lifetime(child_one, lambda: effects.append(1)) + repeated = manager.close_lifetime( + child_one, lambda: pytest.fail('duplicate')) + assert repeated is one + two = manager.close_lifetime(child_two, lambda: effects.append(2)) + assert two.is_set() + assert effects == [1, 2] + dispatch.drain() + assert notified.wait(5) + + +def test_callback_reserved_window_progresses_at_full_delivery_capacity( + runtime): + manager, dispatch = runtime + gate = Event() + entered, release, completed = Event(), Event(), Event() + def listener(): + entered.set() + assert release.wait(5) + with manager.synchronous_reentry() as budget: + assert budget == 'reentry' + with ThreadPoolExecutor(1) as executor: + for _ in range(100): + future = manager.submit( + lambda r: manager.work(r, executor, lambda: 11), + budget=budget, wait=True) + assert future.result(5) == 11 + completed.set() + notification = manager.notify(listener) + assert entered.wait(5) + futures = [manager.submit(lambda record: gate.wait(5)) for _ in range(3)] + for future in futures: + future.add_done_callback(lambda f: None) + gate.set() + for future in futures: + future.result(5) + assert manager._credits == 4 + release.set() + assert completed.wait(5) + notification.result(5) + assert manager._reentry_credits == 0 + + +def test_superseded_physical_result_is_disposed(runtime): + from cassandra._management import ManagementSuperseded + manager, dispatch = runtime + entered, release, disposed = Event(), Event(), Event() + with ThreadPoolExecutor(1) as executor: + def physical(): + entered.set() + assert release.wait(5) + return object() + future = manager.submit( + lambda r: manager.work( + r, executor, physical, dispose=lambda value: disposed.set())) + assert entered.wait(5) + manager.supersede(future) + with pytest.raises(ManagementSuperseded): + future.result(5) + release.set() + assert disposed.wait(5) + + +@pytest.mark.parametrize('ending', ['close', 'supersede']) +def test_result_adoption_is_atomic_with_terminal_settlement(runtime, ending): + manager, dispatch = runtime + accepting, finish_accept, ending_started = Event(), Event(), Event() + resource = object() + disposed, errors = [], [] + def accept(value): + assert value is resource + # Acceptance publishes native state. Other terminal paths must take + # this same condition before they can claim the operation's outcome. + assert manager._condition._is_owned() + accepting.set() + assert finish_accept.wait(5) + return value + with ThreadPoolExecutor(1) as executor: + future = manager.submit(lambda record: manager.work( + record, executor, lambda: resource, accept=accept, + dispose=disposed.append)) + def end(): + ending_started.set() + try: + if ending == 'close': + manager.close() + else: + manager.supersede(future) + except BaseException as exc: + errors.append(exc) + ending_thread = Thread(target=end) + try: + assert accepting.wait(5) + ending_thread.start() + assert ending_started.wait(5) + finally: + finish_accept.set() + if ending_thread.ident is not None: + ending_thread.join(5) + assert not ending_thread.is_alive() + assert not errors + assert future.result(5) is resource + assert disposed == [] + + +def test_rejected_acceptance_retains_worker_until_disposal_finishes(runtime): + manager, dispatch = runtime + disposing, finish_dispose = Event(), Event() + resource = object() + physical_threads, disposal_threads = [], [] + def physical(): + physical_threads.append(get_ident()) + return resource + def accept(value): + raise ValueError('cannot adopt resource') + def dispose(value): + assert value is resource + disposal_threads.append(get_ident()) + disposing.set() + assert finish_dispose.wait(5) + with ThreadPoolExecutor(1) as executor: + future = manager.submit(lambda record: manager.work( + record, executor, physical, accept=accept, dispose=dispose)) + try: + with pytest.raises(ValueError, match='cannot adopt resource'): + future.result(5) + assert disposing.wait(5) + assert manager.submit(lambda record: 23).result(5) == 23 + manager.close() + assert not manager.finalized.is_set() + assert manager.counts()['ordinary'] == 1 + finally: + finish_dispose.set() + assert disposal_threads == physical_threads + manager.join(5) + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('cancelled', [False, True]) +def test_executor_returns_terminal_failure_without_owner_self_wait( + runtime, cancelled): + from concurrent.futures import CancelledError + manager, dispatch = runtime + physical = Future() + if cancelled: + physical.cancel() + else: + physical.set_exception(ValueError('executor failed before execution')) + class FailedExecutor: + def submit(self, function): + return physical + future = manager.submit(lambda record: manager.work( + record, FailedExecutor(), lambda: pytest.fail('physical work ran'))) + with pytest.raises(CancelledError if cancelled else ValueError): + future.result(5) + assert manager.submit(lambda record: 24).result(5) == 24 + + +def test_completed_executor_future_does_not_dispose_on_registering_owner( + runtime): + manager, dispatch = runtime + resource = object() + disposal_threads = [] + with ThreadPoolExecutor(1) as executor: + class ClosingExecutor: + def submit(self, function): + manager.close() + physical = executor.submit(function) + # Completion precedes callback registration, so Future will + # invoke that callback on this owner. Cleanup must already + # have happened on the physical worker, exactly once. + assert physical.result(5) is resource + return physical + future = manager.submit(lambda record: manager.work( + record, ClosingExecutor(), lambda: resource, + dispose=lambda value: disposal_threads.append(get_ident()))) + with pytest.raises(ManagementClosed): + future.result(5) + manager.join(5) + assert len(disposal_threads) == 1 + assert disposal_threads[0] not in { + thread.ident for thread in manager._threads} + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('installed', [False, True]) +def test_interrupted_physical_callback_registration_releases_once( + runtime, installed): + manager, dispatch = runtime + physical = Future() + register = physical.add_done_callback + def interrupted(callback): + if installed: + register(callback) + raise KeyboardInterrupt('physical registration interrupted') + physical.add_done_callback = interrupted + class PendingExecutor: + def submit(self, function): + return physical + future = manager.submit(lambda record: manager.work( + record, PendingExecutor(), lambda: pytest.fail('cancelled work ran'))) + with pytest.raises(KeyboardInterrupt, match='registration interrupted'): + future.result(5) + assert manager.submit(lambda record: 25).result(5) == 25 + assert physical.cancelled() + with manager._condition: + assert future._record.physical == 0 + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('failed_phase', [1, 2, 3, 4]) +@pytest.mark.parametrize('startup_error', [RuntimeError, KeyboardInterrupt]) +def test_start_failure_and_repeated_close_leave_no_threads( + monkeypatch, failed_phase, startup_error): + real_start = Thread.start + starts = [] + def start(thread): + starts.append(thread) + if len(starts) == failed_phase: + raise startup_error('thread startup failed') + return real_start(thread) + dispatch = Dispatcher() + manager = ManagementRuntime(dispatch) + try: + with monkeypatch.context() as patcher: + patcher.setattr(Thread, 'start', start) + with pytest.raises(startup_error, match='thread startup failed'): + manager.start() + finally: + manager.close() + manager.join(5) + assert all(not thread.is_alive() for thread in manager._threads) + for _ in range(100): + manager = ManagementRuntime(dispatch) + try: + assert manager.submit(lambda record: 1).result(5) == 1 + first = manager.close() + second = manager.close() + assert second is first + finally: + manager.close() + manager.join(5) + assert not manager._records + assert all(not thread.is_alive() for thread in manager._threads) + + +@pytest.mark.parametrize('bootstrap_pending', [False, True]) +def test_start_interruption_after_launch_tracks_the_runner( + monkeypatch, bootstrap_pending): + manager = ManagementRuntime(Dispatcher()) + launched, release = Event(), Event() + threads = [] + real_start = Thread.start + real_bootstrap = Thread._bootstrap_inner + def bootstrap(thread): + if bootstrap_pending: + launched.set() + assert release.wait(5) + real_bootstrap(thread) + def start(thread): + threads.append(thread) + if not bootstrap_pending: + real_start(thread) + raise KeyboardInterrupt('interrupted after launch') + original_wait = thread._started.wait + def interrupted_wait(*args, **kwargs): + assert launched.wait(5) + assert thread.ident is None + raise KeyboardInterrupt('interrupted before bootstrap') + thread._started.wait = interrupted_wait + try: + real_start(thread) + finally: + thread._started.wait = original_wait + try: + with monkeypatch.context() as patcher: + patcher.setattr(Thread, '_bootstrap_inner', bootstrap) + patcher.setattr(Thread, 'start', start) + with pytest.raises(KeyboardInterrupt): + manager.start() + manager.close() + if bootstrap_pending: + assert not manager.finalized.is_set() + assert manager._threads == threads + finally: + release.set() + for thread in threads: + assert thread._started.wait(5) + thread.join(5) + assert not thread.is_alive() + manager.close() + manager.join(5) + assert manager._exited == len(threads) == 1 + + +def test_post_terminal_owner_callbacks_use_current_charged_batch(runtime): + manager, dispatch = runtime + old = manager.submit(lambda record: 3) + assert old.result(5) == 3 + calls = [] + def handler(record): + for i in range(20): + old.add_done_callback( + lambda f, i=i: calls.append((i, execution_context()))) + assert calls == [] + current = manager.submit(handler) + current.result(5) + assert manager._credits == 1 + assert len(dispatch.queue) == 1 + dispatch.drain() + assert calls == [(i, 'bridge') for i in range(20)] + assert manager._credits == 0 + + +def test_owner_callback_reselects_batch_retired_during_registration(runtime): + manager, dispatch = runtime + entered, finish, selected, register = [Event() for _ in range(4)] + armed = Event() + calls = [] + old = manager.submit(lambda record: (entered.set(), finish.wait(5), 1)[2]) + assert entered.wait(5) + old.add_done_callback(lambda future: calls.append('old')) + finish.set() + assert old.result(5) == 1 + condition = manager._condition + + class RegistrationGate: + def __getattr__(self, name): + return getattr(condition, name) + + def __enter__(self): + if execution_context() == 'cluster' and armed.is_set(): + armed.clear() + selected.set() + assert register.wait(5) + return condition.__enter__() + + def __exit__(self, *args): + return condition.__exit__(*args) + + manager._condition = RegistrationGate() + current = None + try: + def handler(record): + armed.set() + old.add_done_callback(lambda future: calls.append('new')) + assert old._delivery_record is record + current = manager.submit(handler) + assert selected.wait(5) + dispatch.drain() + assert old._record.released + register.set() + current.result(5) + dispatch.drain() + assert calls == ['old', 'new'] + finally: + register.set() + if current is not None: + current.result(5) + manager._condition = condition + + +def test_post_terminal_registration_is_serial_across_threads(runtime): + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + future.result(5) + entered, release = Event(), Event() + order = [] + def first(future): + order.append('first-enter') + entered.set() + assert release.wait(5) + order.append('first-exit') + thread = Thread(target=lambda: future.add_done_callback(first)) + thread.start() + assert entered.wait(5) + future.add_done_callback(lambda f: order.append('second')) + assert order == ['first-enter'] + release.set() + thread.join(5) + assert order == ['first-enter', 'first-exit', 'second'] + + +def test_concurrent_producers_have_one_fifo_owner(monkeypatch): + dispatch = Dispatcher() + manager = ManagementRuntime(dispatch, limit=64) + admitted, handled, owners, errors = [], [], set(), [] + producer_lock = Lock() + start = Event() + admission = Barrier(8) + original_start = manager.start + def synchronized_start(): + # All eight callers have entered the actual submit() frame before any + # reaches its internal admission lock. No producer lock serializes it. + admission.wait(5) + original_start() + monkeypatch.setattr(manager, 'start', synchronized_start) + def producer(): + try: + assert start.wait(5) + for _ in range(8): + future = manager.submit(lambda r: ( + handled.append(r.envelope.operation), owners.add(get_ident()))) + with producer_lock: + admitted.append((future._record.envelope.operation, future)) + except BaseException as exc: + with producer_lock: + errors.append(exc) + threads = [Thread(target=producer) for _ in range(8)] + try: + for thread in threads: + thread.start() + start.set() + for thread in threads: + thread.join(5) + assert not thread.is_alive() + assert not errors + assert len(admitted) == 64 + for _, future in admitted: + future.result(5) + assert handled == sorted(number for number, _ in admitted) + assert len(handled) == 64 + assert len(owners) == 1 + finally: + start.set() + admission.abort() + manager.close() + manager.join(5) + + +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +def test_result_reserved_before_close_is_still_disposed(runtime, pool): + manager, dispatch = runtime + worker = Future() + class ControlledExecutor: + def submit(self, function): + return worker + started, blocked, release, disposed = Event(), Event(), Event(), Event() + disposal_context = [] + def dispose(value): + disposal_context.append((execution_context(), get_ident())) + disposed.set() + def initial(record): + started.set() + return manager.work(record, ControlledExecutor(), object, + dispose=dispose) + manager.pool_limit = 3 + future = manager.submit(initial, pool=pool) + completion = Thread(target=lambda: worker.set_result(object()), daemon=True) + try: + assert started.wait(5) + blocker = manager.submit( + lambda record: (blocked.set(), release.wait(5)), pool=pool) + assert blocked.wait(5) + queued = manager.submit( + lambda record: pytest.fail('closed ordinary work ran'), + pool=pool) + # The reserved result sits behind ordinary work. + completion.start() + with manager._condition: + def result_queued(): + queue = (manager._cluster if pool is None + else manager._pools[pool]) + return any(item[2] for item in queue) + assert manager._condition.wait_for(result_queued, 5) + manager.close() + for pending in (future, blocker, queued): + with pytest.raises(ManagementClosed): + pending.result(5) + with manager._condition: + queue = manager._cluster if pool is None else manager._pools[pool] + assert len(queue) == 1 and queue[0][2] + if pool is not None: + assert pool in manager._active_pools + assert not disposed.is_set() + finally: + release.set() + if completion.ident is None: + manager.close() + completion.start() + completion.join(5) + if completion.is_alive(): + manager.close() + completion.join(5) + assert not completion.is_alive() + assert disposed.wait(5) + assert disposal_context == [('worker', completion.ident)] + assert completion.ident not in {thread.ident for thread in manager._threads} + + +def test_synchronous_admission_waits_without_allocating_pending_records( + runtime, monkeypatch): + manager, dispatch = runtime + gate = Event() + futures = [manager.submit(lambda record: gate.wait(5)) for _ in range(4)] + result = [] + waiting, done = Event(), Event() + original_wait = manager._condition.wait + + def observe_wait(timeout=None): + if get_ident() == thread.ident: + assert manager._credits == manager.limit + waiting.set() + return original_wait(timeout) + + def caller(): + result.append(manager.submit(lambda record: 8, wait=True).result(5)) + done.set() + thread = Thread(target=caller) + monkeypatch.setattr(manager._condition, 'wait', observe_wait) + thread.start() + try: + assert waiting.wait(5) + with manager._condition: + assert len(manager._records) == 4 + assert manager._credits == 4 + assert not done.is_set() + finally: + gate.set() + thread.join(5) + for future in futures: + future.result(5) + assert done.wait(5) + thread.join(5) + assert result == [8] + + +def test_pool_mailboxes_share_two_runners_without_concurrent_handlers(): + dispatch = Dispatcher() + manager = ManagementRuntime(dispatch, limit=20) + entered = [Event(), Event()] + release = Event() + pools = [object(), object(), object()] + running = set() + lock = Lock() + seen = [] + def handler(index): + def run(record): + manager.assert_owner(pools[index]) + with lock: + assert pools[index] not in running + running.add(pools[index]) + seen.append((index, get_ident())) + if index < 2: + entered[index].set() + assert release.wait(5) + with lock: + running.remove(pools[index]) + return run + try: + first = manager.submit(handler(0), pool=pools[0]) + second = manager.submit(handler(1), pool=pools[1]) + assert all(event.wait(5) for event in entered) + queued_same = manager.submit(handler(0), pool=pools[0]) + third = manager.submit(handler(2), pool=pools[2]) + assert not third.done() + assert not queued_same.done() + release.set() + for future in [first, second, queued_same, third]: + future.result(5) + assert len({thread for _, thread in seen}) == 2 + assert len(manager._threads) == 4 + finally: + release.set() + manager.close() + manager.join(5) + + +def test_worker_close_and_join_guard(runtime): + manager, dispatch = runtime + with ThreadPoolExecutor(1) as executor: + def physical(): + assert execution_context() == 'worker' + manager.close() + with pytest.raises(ManagementContextError): + manager.join() + return object() + future = manager.submit(lambda r: manager.work(r, executor, physical)) + with pytest.raises(ManagementClosed): + future.result(5) + manager.join(5) + + +@pytest.mark.parametrize('entrypoint', [ + 'submit', 'observe_negative', 'register_lifetime', 'close_lifetime', + 'unregister_health', 'retain_cleanup', 'counts', 'release_cleanup']) +def test_inherited_runtime_rejects_before_using_parent_locks( + runtime, entrypoint): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + manager, dispatch = runtime + manager.start() + handle = manager.register_lifetime() + finish_cleanup = manager.retain_cleanup() + locked, release = Event(), Event() + def holder(): + with manager._condition: + locked.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert locked.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + try: + { + 'submit': lambda: manager.submit(lambda record: None), + 'observe_negative': lambda: manager.observe_negative( + object(), None), + 'register_lifetime': manager.register_lifetime, + 'close_lifetime': lambda: manager.close_lifetime( + handle, lambda: None), + 'unregister_health': lambda: manager.unregister_health( + object()), + 'retain_cleanup': manager.retain_cleanup, + 'counts': manager.counts, + 'release_cleanup': finish_cleanup, + }[entrypoint]() + except ManagementClosed: + os.write(write_fd, b'rejected') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'inherited runtime entered a parent-held condition' + assert os.read(read_fd, 8) == b'rejected' + finally: + release.set() + thread.join(5) + finish_cleanup() + # The child is killed on timeout; after writing it exits in finally. + os.waitpid(pid, 0) + os.close(read_fd) + + +def test_one_physical_result_per_current_admitted_record(runtime): + manager, dispatch = runtime + physical = Future() + submissions = [] + disposed = Event() + class Executor: + def submit(self, function): + submissions.append(function) + return physical + def handler(record): + manager.work(record, Executor(), lambda: None, + dispose=lambda value: disposed.set()) + # A second work item would create an uncharged result queue node. + return manager.work(record, Executor(), lambda: None) + future = manager.submit(handler) + try: + with pytest.raises(ManagementContextError): + future.result(5) + assert len(submissions) == 1 + finally: + complete_physical(manager, physical, object()) + assert disposed.wait(5) + old = future._record + def reuse(record): + with pytest.raises(ManagementContextError): + manager.work(old, Executor(), lambda: None) + manager.submit(reuse).result(5) + assert len(submissions) == 1 + + +def test_inherited_future_rejects_before_parent_held_condition(runtime): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + assert future.result(5) == 1 + locked, release = Event(), Event() + def holder(): + with future._condition: + locked.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert locked.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + calls = [future.done, future.running, future.cancelled, + future.result, future.exception, future.cancel, + lambda: future.add_done_callback(lambda f: None), + lambda: wait([future], timeout=0)] + rejected = 0 + for call in calls: + try: + call() + except ManagementClosed: + rejected += 1 + os.write(write_fd, str(rejected).encode()) + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'inherited Future entered a parent-held condition' + assert os.read(read_fd, 8) == b'8' + finally: + release.set() + thread.join(5) + os.close(read_fd) + os.waitpid(pid, 0) + + +def test_dispatch_rejection_closes_without_stranding_waiters_or_resources(): + from cassandra._management import ManagementDeliveryFailed + attempts, calls = [], [] + def reject(callback): + attempts.append(callback) + raise RuntimeError('reactor rejected callback') + (handler_entered, handler_release, worker_started, worker_release, + disposed) = [Event() for _ in range(5)] + with ThreadPoolExecutor(1) as executor: + manager = ManagementRuntime(reject, limit=2) + def physical(): + worker_started.set() + assert worker_release.wait(5) + return object() + try: + late = manager.submit(lambda r: manager.work( + r, executor, physical, dispose=lambda value: disposed.set())) + assert worker_started.wait(5) + def handler(record): + handler_entered.set() + assert handler_release.wait(5) + return 4 + future = manager.submit(handler) + assert handler_entered.wait(5) + future.add_done_callback( + lambda f: calls.append(execution_context())) + handler_release.set() + assert future.result(5) == 4 + assert manager.closed.wait(5) + assert isinstance( + manager.completion_error, ManagementDeliveryFailed) + assert isinstance(future._delivery_error, ManagementDeliveryFailed) + with pytest.raises(ManagementDeliveryFailed): + future.add_done_callback(lambda f: calls.append('late')) + with pytest.raises(ManagementClosed): + late.result(5) + worker_release.set() + assert disposed.wait(5) + finally: + handler_release.set() + worker_release.set() + manager.close() + manager.join(5) + assert len(attempts) == 1 + assert calls == [] + assert manager._credits == 0 + assert not manager._records + + +def test_fatal_service_loss_abandons_queued_tickets_without_retaining_results( +): + from cassandra._management import ManagementDeliveryFailed + queue = [] + calls = [] + def dispatch(ticket): + if queue: + raise RuntimeError('validated reactor lost') + queue.append(ticket) + manager = ManagementRuntime(dispatch, limit=2) + (first_entered, first_release, second_entered, + second_release) = [Event() for _ in range(4)] + def first_handler(record): + first_entered.set() + assert first_release.wait(5) + return object() + def second_handler(record): + second_entered.set() + assert second_release.wait(5) + try: + first = manager.submit(first_handler) + assert first_entered.wait(5) + first.add_done_callback(lambda f: calls.append('first')) + first_release.set() + first.result(5) + second = manager.submit(second_handler) + assert second_entered.wait(5) + second.add_done_callback(lambda f: calls.append('second')) + second_release.set() + second.result(5) + assert manager.closed.wait(5) + manager.join(5) + finally: + first_release.set() + second_release.set() + manager.close() + for ticket in queue: + ticket() + manager.join(5) + assert isinstance(first._delivery_error, ManagementDeliveryFailed) + assert queue[0]._callback is None + queue[0]() # A late reactor wakeup cannot resurrect an abandoned batch. + assert calls == [] + assert manager._credits == 0 + + +def test_cancellation_racing_settlement_does_not_strand_coalesced_waiters( + runtime, monkeypatch): + manager, dispatch = runtime + entered, release, at_setter, cancel_attempted = [Event() for _ in range(4)] + cancelled_state = Event() + def handler(record): + entered.set() + assert release.wait(5) + return 17 + first = manager.submit(handler, key='shared') + second = manager.submit(handler, key='shared') + assert entered.wait(5) + setter = first.set_result + invoke = first._invoke_callbacks + def observed_cancel(): + cancelled_state.set() + invoke() + def paused_setter(value): + at_setter.set() + assert cancel_attempted.wait(5) + # If settlement owns the future lock, cancellation must wait for its + # terminal update. Otherwise force the competing cancellation to win. + if not first._condition._is_owned(): + assert cancelled_state.wait(5) + setter(value) + monkeypatch.setattr(first, '_invoke_callbacks', observed_cancel) + monkeypatch.setattr(first, 'set_result', paused_setter) + def cancel(): + assert at_setter.wait(5) + cancel_attempted.set() + first.cancel() + thread = Thread(target=cancel) + thread.start() + try: + release.set() + assert second.result(5) == 17 + assert first.done() + finally: + release.set() + thread.join(5) + assert not thread.is_alive() + + +def test_notification_admission_after_close_does_not_retain_callback( + runtime, monkeypatch): + import gc + import weakref + manager, dispatch = runtime + entered, release = Event(), Event() + original = manager.submit + def submit(handler, **kwargs): + def paused(record): + entered.set() + assert release.wait(5) + return handler(record) + return original(paused, **kwargs) + monkeypatch.setattr(manager, 'submit', submit) + class Callback: + def __call__(self): + pytest.fail('closed notification ran') + callback = Callback() + reference = weakref.ref(callback) + future = manager.notify(callback) + assert entered.wait(5) + manager.close() + release.set() + manager.join(5) + del callback + gc.collect() + assert reference() is None + assert not manager._notifications + with pytest.raises(ManagementClosed): + future.result(5) + + +@pytest.mark.parametrize('private_first', [False, True]) +@pytest.mark.parametrize('private_budget', ['reentry', 'aggregate']) +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +def test_coalesced_waiters_keep_independent_budget_and_delivery_charges( + runtime, private_first, private_budget, pool): + from contextlib import nullcontext + manager, dispatch = runtime + physical = Future() + work_started, joined, allow_join = Event(), Event(), Event() + calls, private = [], [] + class Executor: + def submit(self, function): + calls.append(function) + work_started.set() + return physical + def operation(record): + return manager.work(record, Executor(), lambda: 23) + def listener(): + assert allow_join.wait(5) + scope = (manager.synchronous_reentry() if private_budget == 'reentry' + else nullcontext('aggregate')) + with scope as budget: + future = manager.submit(operation, key='shared-budget', + budget=budget, pool=pool) + private.append(future) + with pytest.raises(ManagementContextError): + future.add_done_callback(lambda f: None) + joined.set() + assert future.result(5) == 23 + try: + if not private_first: + ordinary = manager.submit( + operation, key='shared-budget', pool=pool) + assert work_started.wait(5) + notification = manager.notify(listener) + allow_join.set() + assert joined.wait(5) + if private_first: + assert work_started.wait(5) + ordinary = manager.submit( + operation, key='shared-budget', pool=pool) + assert private[0]._record is ordinary._record + ordinary.add_done_callback(lambda f: None) + complete_physical(manager, physical, 23) + assert ordinary.result(5) == 23 + notification.result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: manager._credits == 1, 5) + assert manager._reentry_credits == 0 + assert not manager._reentry_active + if pool is not None: + assert manager._pool_credits[pool] == 1 + assert len(calls) == 1 + assert len(dispatch.queue) == 1 + dispatch.drain() + assert manager.counts()['ordinary'] == manager.counts()['reentry'] == 0 + assert not manager._pool_credits + finally: + allow_join.set() + if not physical.done(): + complete_physical(manager, physical, 23) + + +def test_foreign_callback_uses_ordinary_admission(runtime): + first, dispatch = runtime + other = ManagementRuntime(dispatch, limit=1) + gate, entered = Event(), Event() + try: + other.submit(lambda record: (entered.set(), gate.wait(5))) + assert entered.wait(5) + def listener(): + with other.synchronous_reentry() as budget: + assert budget == 'ordinary' + with pytest.raises(ManagementOverloaded): + other.submit(lambda record: None, budget=budget) + assert not other._reentry_active + first.notify(listener).result(5) + finally: + gate.set() + other.close() + other.join(5) + + +def test_standard_wait_rejects_owner_self_wait_and_allows_external_wait( + runtime): + manager, dispatch = runtime + following = [] + def handler(record): + future = manager.submit(lambda record: 41) + following.append(future) + with pytest.raises(ManagementContextError): + wait([future], timeout=0) + return 40 + completed = manager.submit(handler) + assert completed.result(5) == 40 + assert not wait([completed, following[0]], timeout=5).not_done + assert following[0].result(5) == 41 + + +def test_external_standard_wait_wakes_when_pending_operation_is_cancelled( + runtime): + manager, dispatch = runtime + entered, release, installed = Event(), Event(), Event() + blocker = manager.submit(lambda record: (entered.set(), release.wait(5))) + assert entered.wait(5) + future = manager.submit(lambda record: 42) + class ObservedWaiters(list): + def append(self, waiter): + super().append(waiter) + installed.set() + future._waiters = ObservedWaiters() + results = [] + thread = Thread(target=lambda: results.append(wait([future], timeout=5))) + thread.start() + try: + assert installed.wait(5) + assert future.cancel() + thread.join(5) + assert not thread.is_alive() + assert results[0].done == {future} + assert not results[0].not_done + finally: + release.set() + thread.join(5) + blocker.result(5) + + +def test_close_handles_retire_history_but_preserve_live_duplicate_joins( + runtime): + import gc + import weakref + manager, dispatch = runtime + references, retired = [], [] + for i in range(128): + handle = manager.register_lifetime() + references.append(weakref.ref(handle)) + done = manager.close_lifetime(handle, lambda: retired.append(i)) + assert done.is_set() + assert manager.close_lifetime( + handle, lambda: pytest.fail('duplicate')) is done + del handle + gc.collect() + assert all(reference() is None for reference in references) + assert retired == list(range(128)) + assert manager.counts()['close'] == 0 + first, second = manager.register_lifetime(), manager.register_lifetime() + manager.close_lifetime(first, lambda: None) + assert not second.closed.is_set() + manager.close() + assert second.closed.is_set() + assert not manager._lifetimes + assert manager.close_lifetime( + second, lambda: pytest.fail('after close')) is second.closed + with pytest.raises(ManagementClosed): + manager.register_lifetime() + + +@pytest.mark.parametrize('terminal_first', [False, True]) +def test_bulk_owner_rejection_unwinds_all_conditions(runtime, terminal_first): + manager, dispatch = runtime + already = manager.submit(lambda record: 1) + assert already.result(5) == 1 + following = [] + def handler(record): + pair = [manager.submit(lambda record: 2) for _ in range(2)] + terminal, pending = sorted(pair, key=id, reverse=not terminal_first) + assert terminal.cancel() + assert wait([already, terminal], timeout=0).done == {already, terminal} + with pytest.raises(ManagementContextError): + wait([terminal, pending], timeout=0) + # Rejection must not contaminate this thread's ordinary status access. + assert not pending.done() + following.extend(pair) + manager.submit(handler).result(5) + assert not wait(following, timeout=5).not_done + for future in following: + acquired = future._condition.acquire(timeout=1) + try: + assert acquired, ( + 'bulk wait rejection retained an earlier condition') + finally: + if acquired: + future._condition.release() + + +@pytest.mark.parametrize( + 'outcome', ['return', 'error', 'base_error', 'rejected_reacquire']) +def test_policy_phase_restores_context_and_only_reacquires_after_success( + runtime, outcome): + from cassandra import _management + manager, dispatch = runtime + class PolicyAbort(BaseException): + pass + @_management.policy_call + def policy(): + assert _management._context.call is None + assert manager.counts()['ordinary'] == 0 + if outcome == 'error': + raise ValueError('policy failed') + if outcome == 'base_error': + raise PolicyAbort() + if outcome == 'rejected_reacquire': + manager.close() + return 13 + with _management.compatibility_call(manager): + call = _management._context.call + original = call.record + if outcome == 'return': + assert policy() == 13 + assert call.record is not original + else: + error = {'error': ValueError, 'base_error': PolicyAbort, + 'rejected_reacquire': ManagementClosed}[outcome] + with pytest.raises(error): + policy() + assert call.record is None + assert _management._context.call is call + assert getattr(_management._context, 'call', None) is None + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +def test_compatibility_work_rejects_owner_before_admission( + runtime, monkeypatch, pool): + from cassandra._management import compatibility_work + manager, dispatch = runtime + submitted = [] + original = manager.submit + def submit(*args, **kwargs): + future = original(*args, **kwargs) + submitted.append(future) + def forbidden_wait(*args, **kwargs): + raise AssertionError('compatibility work attempted an owner wait') + monkeypatch.setattr(future._record.started, 'wait', forbidden_wait) + return future + def handler(record): + monkeypatch.setattr(manager, 'submit', submit) + try: + with pytest.raises(ManagementContextError): + with compatibility_work(manager): + pytest.fail('owner entered physical compatibility work') + assert submitted == [] + finally: + # Keep the regression's old-source failure safe: it admits a + # synthetic physical obligation before reaching the forbidden wait. + for future in submitted: + with manager._condition: + manager._finish(future._record) + future._record.physical -= 1 + manager._release_locked(future._record) + original(handler, pool=pool).result(5) + + +def test_health_registration_crossing_final_close_rejects_without_retention( + runtime, monkeypatch): + manager, dispatch = runtime + started, resume = Event(), Event() + original = manager.start + def start(): + original() + started.set() + assert resume.wait(5) + monkeypatch.setattr(manager, 'start', start) + incarnation = object() + errors = [] + def register(): + try: + manager.register_health(incarnation) + except BaseException as exc: + errors.append(exc) + thread = Thread(target=register) + thread.start() + try: + assert started.wait(5) + manager.close() + manager.join(5) + resume.set() + thread.join(5) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], ManagementClosed) + assert manager.counts()['health'] == 0 + finally: + resume.set() + thread.join(5) + manager.unregister_health(incarnation) + + +def test_cleanup_cannot_join_an_already_started_async_finalization(): + detached = [] + manager = ManagementRuntime(Dispatcher(), on_finalized=detached.append) + early = manager.retain_cleanup() + late = None + try: + manager.close() + assert not detached + early() + assert len(detached) == 1 and manager._finalizing + assert not manager.finalized.is_set() + with pytest.raises(ManagementClosed): + late = manager.retain_cleanup() + assert manager.counts()['close'] == 0 + detached[0]() + manager.join(5) + finally: + early() + if late is not None: + late() + manager.close() + if detached: + detached[0]() + manager.join(5) + + +@pytest.mark.parametrize('entry', ['call', 'work']) +@pytest.mark.parametrize('drain_queued', [False, True]) +def test_interrupted_initial_compatibility_admission_discharges_reservation( + runtime, monkeypatch, entry, drain_queued): + from cassandra._management import compatibility_call, compatibility_work + manager, dispatch = runtime + entered, resume = Event(), Event() + manager.submit(lambda record: (entered.set(), resume.wait(5))) + assert entered.wait(5) + admitted = [] + original = manager.submit + def submit(*args, **kwargs): + future = original(*args, **kwargs) + admitted.append(future) + def interrupt(*args, **kwargs): + assert not future._record.started.is_set() + raise KeyboardInterrupt() + monkeypatch.setattr(future._record.started, 'wait', interrupt) + return future + monkeypatch.setattr(manager, 'submit', submit) + try: + scope = compatibility_call if entry == 'call' else compatibility_work + with pytest.raises(KeyboardInterrupt): + with scope(manager): + pytest.fail('interrupted admission entered its body') + assert len(admitted) == 1 + assert admitted[0]._record.physical == 0 + if drain_queued: + monkeypatch.setattr(manager, 'submit', original) + resume.set() + manager.submit(lambda record: None).result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: not manager._records, 5) + assert admitted[0]._record.released + assert admitted[0]._record.physical == 0 + else: + manager.close() + assert admitted[0]._record.released + finally: + resume.set() + manager.close() + # Unwind the old-source counterexample without leaving daemon threads + # or a synthetic physical reservation behind after the failed + # assertion. + for future in admitted: + with manager._condition: + if future._record.physical: + future._record.physical -= 1 + manager._release_locked(future._record) + manager.join(5) + + +@pytest.mark.parametrize('installed', [False, True]) +@pytest.mark.parametrize('outcome', ['result', 'exception']) +def test_interrupted_compatibility_attachment_retains_work_until_completion( + runtime, installed, outcome): + from cassandra._management import compatibility_work + manager, dispatch = runtime + physical = Future() + assert physical.set_running_or_notify_cancel() + register = physical.add_done_callback + attaching, finished = Event(), Event() + errors, records = [], [] + def interrupted(callback): + if installed: + register(callback) + attaching.set() + raise KeyboardInterrupt('compatibility attachment interrupted') + physical.add_done_callback = interrupted + def caller(): + try: + with compatibility_work(manager) as attach: + with manager._condition: + records.extend(manager._records) + attach(physical) + except BaseException as exc: + errors.append(exc) + finally: + finished.set() + thread = Thread(target=caller, daemon=True) + thread.start() + try: + assert attaching.wait(5) + with manager._condition: + assert len(records) == 1 + assert records[0].physical == 1 + assert not finished.wait(0.05) + finally: + if outcome == 'result': + physical.set_result(23) + else: + physical.set_exception(ValueError('physical work failed')) + thread.join(5) + assert not thread.is_alive() + # Keep the original-source failure bounded without masking the defect. + with manager._condition: + debt_after_completion = sum(record.physical for record in records) + credits_after_completion = manager.counts()['ordinary'] + for record in records: + if record.physical: + manager._finish(record) + record.physical -= 1 + manager._release_locked(record) + assert len(errors) == 1 + assert isinstance(errors[0], KeyboardInterrupt) + assert debt_after_completion == credits_after_completion == 0 + assert records[0].physical == 0 + assert manager.counts()['ordinary'] == 0 + + +@pytest.mark.parametrize('pool', [None, 'logical-pool']) +@pytest.mark.parametrize('ending', ['normal', 'close', 'fatal']) +def test_owner_callback_handoff_yields_reactor_while_caller_callback_runs( + runtime, monkeypatch, pool, ending): + from cassandra._management import ManagementDeliveryFailed + manager, dispatch = runtime + old = manager.submit(lambda record: 3) + assert old.result(5) == 3 + with manager._condition: + assert manager._condition.wait_for(lambda: old._record.released, 5) + entered, resume, io_progress = Event(), Event(), Event() + calls = [] + def caller_callback(future): + calls.append(('caller-enter', execution_context())) + entered.set() + assert resume.wait(5) + calls.append(('caller-exit', execution_context())) + caller = Thread(target=lambda: old.add_done_callback(caller_callback)) + reactor = Thread(target=dispatch.drain) + caller.start() + try: + assert entered.wait(5) + def owner(record): + old.add_done_callback( + lambda f: calls.append(('owner', execution_context()))) + manager.submit(owner, pool=pool).result(5) + dispatch(io_progress.set) + reactor.start() + assert io_progress.wait(1), ( + 'callback handoff blocked shared reactor progress') + reactor.join(5) + assert not reactor.is_alive() + assert calls == [('caller-enter', None)] + assert manager.counts()['ordinary'] == 1 + if ending == 'close': + manager.close() + assert not manager.finalized.is_set() + elif ending == 'fatal': + def reject(ticket): + raise RuntimeError('validated reactor lost during handoff') + monkeypatch.setattr(manager, '_dispatch', reject) + with manager._condition: + failed_delivery = manager.submit(lambda record: 9) + failed_delivery.add_done_callback( + lambda f: pytest.fail('undeliverable callback ran')) + assert failed_delivery.result(5) == 9 + assert manager.closed.wait(5) + manager.join(5) + assert isinstance(old._delivery_error, ManagementDeliveryFailed) + assert manager.counts()['ordinary'] == 0 + finally: + resume.set() + caller.join(5) + if reactor.ident is not None: + reactor.join(5) + assert not caller.is_alive() and not reactor.is_alive() + dispatch.drain() + expected = [('caller-enter', None), ('caller-exit', None)] + if ending != 'fatal': + expected.append(('owner', 'bridge')) + assert calls == expected + assert manager.counts()['ordinary'] == 0 + assert not old._callback_waiters + + +@pytest.mark.parametrize('one_record', [False, True]) +def test_deferred_callback_handoffs_keep_one_ticket_per_charged_record( + runtime, one_record): + manager, dispatch = runtime + old = [manager.submit(lambda record: 1) for _ in range(3)] + assert all(f.result(5) == 1 for f in old) + with manager._condition: + assert manager._condition.wait_for( + lambda: not manager._records, 5) + count = 1 if one_record else 3 + manager.limit = count + entered = [Event() for _ in old] + resume, io_progress = Event(), Event() + calls = [] + def caller_callback(future, ready): + ready.set() + assert resume.wait(5) + callers = [Thread(target=lambda f=f, ready=ready: + f.add_done_callback(lambda f: caller_callback(f, ready))) + for f, ready in zip(old, entered)] + reactor = Thread(target=dispatch.drain) + for thread in callers: + thread.start() + try: + assert all(ready.wait(5) for ready in entered) + def owner(futures): + for future in futures: + future.add_done_callback( + lambda f: calls.append(execution_context())) + if one_record: + manager.submit(lambda record: owner(old)).result(5) + else: + for future in old: + manager.submit(lambda record, f=future: owner([f])).result(5) + dispatch(io_progress.set) + reactor.start() + assert io_progress.wait(1) + reactor.join(5) + assert not reactor.is_alive() + with manager._condition: + assert len(manager._records) == count + assert all(r.delivery and r.delivery_ticket is None + for r in manager._records) + assert not dispatch.queue + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None) + resume.set() + for thread in callers: + thread.join(5) + assert not thread.is_alive() + assert len(dispatch.queue) == count + dispatch.drain() + assert calls == ['bridge'] * 3 + assert manager.counts()['ordinary'] == 0 + assert all(not f._callback_waiters for f in old) + finally: + resume.set() + for thread in callers: + thread.join(5) + if reactor.ident is not None: + reactor.join(5) + + +def test_callback_handoff_completion_during_active_delivery_cannot_lose_wakeup( + runtime, monkeypatch): + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + assert future.result(5) == 1 + with manager._condition: + assert manager._condition.wait_for(lambda: future._record.released, 5) + entered, resume, deferred, return_delivery = [Event() for _ in range(4)] + calls = [] + def callback(future): + entered.set() + assert resume.wait(5) + caller = Thread(target=lambda: future.add_done_callback(callback)) + reactor = Thread(target=dispatch.drain) + original = future._drain_callbacks + def drain(record=None): + result = original(record) + if record is not None and not result: + deferred.set() + assert return_delivery.wait(5) + return result + monkeypatch.setattr(future, '_drain_callbacks', drain) + caller.start() + try: + assert entered.wait(5) + owner = manager.submit(lambda record: future.add_done_callback( + lambda f: calls.append(execution_context()))) + owner.result(5) + reactor.start() + assert deferred.wait(5) + resume.set() + caller.join(5) + assert not caller.is_alive() + with manager._condition: + assert owner._record.delivery_again + assert owner._record.delivery_ticket is not None + return_delivery.set() + reactor.join(5) + assert not reactor.is_alive() + assert calls == ['bridge'] + assert not dispatch.queue + assert manager.counts()['ordinary'] == 0 + finally: + resume.set() + return_delivery.set() + caller.join(5) + if reactor.ident is not None: + reactor.join(5) + + +@pytest.mark.parametrize('budget', ['reentry', 'aggregate']) +def test_private_owner_record_cannot_borrow_an_old_future_callback_batch( + runtime, budget): + manager, dispatch = runtime + future = manager.submit(lambda record: 1) + assert future.result(5) == 1 + with manager._condition: + assert manager._condition.wait_for(lambda: future._record.released, 5) + def handler(record): + with pytest.raises(ManagementContextError): + future.add_done_callback( + lambda f: pytest.fail('uncharged callback')) + manager.submit(handler, budget=budget).result(5) + assert future._delivery_record is future._record + assert not dispatch.queue + + +@pytest.mark.parametrize('outcome', ['result', 'error']) +def test_coalesced_settlement_and_bulk_wait_use_compatible_lock_order(outcome): + import subprocess + import sys + # The broken lock order also blocks close, so isolate and reap the child. + # Preallocate real futures in reverse ID order rather than assume an + # allocator order. Admission and stdlib bulk waiting remain unchanged. + script = r'''import faulthandler +faulthandler.dump_traceback_later(7) +"""Bounded child probe; parent terminates a genuine bulk-lock deadlock.""" +from concurrent.futures import wait +from threading import Event, Thread, get_ident +import sys + +from cassandra import _management + +print('management-loader', _management.__file__, flush=True) +manager = _management.ManagementRuntime(lambda callback: None) +gate, owner_held, waiter_held = Event(), Event(), Event() +original = _management.OperationFuture +available = [] + + +def factory(record): + if not available: + available.extend(sorted([original(record), original(record)], + key=id, reverse=True)) + return available.pop(0) + + +def handler(record): + assert gate.wait(5) + if sys.argv[1] == 'error': + raise ValueError('controlled failure') + return 17 + + +_management.OperationFuture = factory +try: + first = manager.submit(handler, key='shared') + second = manager.submit(handler, key='shared') +finally: + _management.OperationFuture = original +assert first._record is second._record and id(second) < id(first) +setter = 'set_exception' if sys.argv[1] == 'error' else 'set_result' +original_setter = getattr(first, setter) + + +def publish(value): + owner_held.set() + assert waiter_held.wait(5) + original_setter(value) + + +setattr(first, setter, publish) +original_acquire = second._condition.acquire + + +def acquire(*args, **kwargs): + result = original_acquire(*args, **kwargs) + if get_ident() == waiter.ident: + waiter_held.set() + return result + + +second._condition.acquire = acquire +results = [] + + +def bulk_wait(): + assert owner_held.wait(5) + results.append(wait([first, second], timeout=5)) + + +waiter = Thread(target=bulk_wait) +waiter.start() +gate.set() +waiter.join(6) +assert not waiter.is_alive(), 'coalesced settlement deadlocked bulk wait' +assert results and results[0].done == {first, second} +assert not results[0].not_done +for future in (first, second): + if sys.argv[1] == 'error': + assert isinstance(future.exception(1), ValueError) + else: + assert future.result(1) == 17 +assert manager.submit(lambda record: 18).result(5) == 18 +manager.close() +manager.join(5) +assert manager.finalized.is_set() +print('bulk-wait-and-close-complete', flush=True) + +faulthandler.cancel_dump_traceback_later() +''' + result = subprocess.run( + [sys.executable, '-c', script, outcome], capture_output=True, + text=True, timeout=12) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize('plain', [False, True]) +@pytest.mark.parametrize('inherited_first', [False, True]) +def test_mixed_bulk_fork_rejection_preserves_fresh_locks( + runtime, plain, inherited_first): + import os + import select + import signal + from concurrent.futures import as_completed + from concurrent.futures._base import _AcquireFutures + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + manager, dispatch = runtime + inherited = manager.submit(lambda record: 1) + assert inherited.result(5) == 1 + locked, release = Event(), Event() + + def hold_parent_condition(): + with inherited._condition: + locked.set() + assert release.wait(25) + + holder = Thread(target=hold_parent_condition) + holder.start() + try: + assert locked.wait(5) + read_fd, write_fd = os.pipe() + try: + pid = os.fork() + except BaseException: + os.close(read_fd) + os.close(write_fd) + raise + except BaseException: + release.set() + holder.join(5) + raise + if pid == 0: + os.close(read_fd) + try: + fresh_manager = None + gate = Event() + if plain: + fresh = Future() + else: + fresh_manager = ManagementRuntime(lambda callback: None) + fresh = fresh_manager.submit(lambda record: gate.wait(5)) + + def inherited_lock_access(*args, **kwargs): + raise AssertionError('touched an inherited condition lock') + + inherited._condition._acquire = inherited_lock_access + inherited._condition._release = inherited_lock_access + try: + with inherited._condition: + raise AssertionError('entered inherited condition context') + except ManagementClosed: + pass + fresh._condition.acquire() + for consumer in ( + lambda: wait([fresh, inherited], timeout=0), + lambda: next(as_completed([fresh, inherited], timeout=0))): + try: + consumer() + except ManagementClosed: + pass + else: + raise AssertionError('bulk wait accepted inherited future') + # Exercise both possible ID-sorted acquisition permutations without + # assuming an allocator order or patching any stdlib function. + ordered = _AcquireFutures([fresh, inherited]) + ordered.futures = ([inherited, fresh] if inherited_first + else [fresh, inherited]) + try: + with ordered: + inherited._state + except ManagementClosed: + pass + else: + raise AssertionError('bulk context accepted inherited state') + attempts = [] + + def probe_outer_depth(): + acquired = fresh._condition.acquire(blocking=False) + attempts.append(acquired) + if acquired: + fresh._condition.release() + + probe = Thread(target=probe_outer_depth) + probe.start() + probe.join(5) + assert attempts == [False], 'bulk rejection released outer lock' + fresh._condition.release() + settled = Event() + + def settle(): + fresh.set_result(23) + settled.set() + + worker = Thread(target=settle) + worker.start() + assert settled.wait(5), 'bulk rejection leaked fresh condition' + worker.join(5) + assert fresh.result(0) == 23 + if fresh_manager is not None: + gate.set() + fresh_manager.close() + fresh_manager.join(5) + os.write(write_fd, b'ok') + except BaseException as exc: + os.write(write_fd, repr(exc).encode()[:4000]) + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 20) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'mixed bulk wait entered a parent-held condition' + outcome = os.read(read_fd, 4096) + assert outcome == b'ok', outcome.decode() + finally: + release.set() + holder.join(5) + os.close(read_fd) + os.waitpid(pid, 0)