diff --git a/cassandra/_lifecycle.py b/cassandra/_lifecycle.py index b16819456c..5fa97c1157 100644 --- a/cassandra/_lifecycle.py +++ b/cassandra/_lifecycle.py @@ -23,11 +23,14 @@ from contextlib import contextmanager from functools import partial, wraps import logging -from threading import local, RLock +from threading import Event, Lock, local, RLock import weakref from weakref import WeakValueDictionary from cassandra import DriverException +from cassandra._management import (ManagementRuntime, ReactorChannel, + ManagementClosed, ManagementOverloaded, assert_synchronous_api, execution_context, + physical_work_context, performing_cleanup, compatibility_call, compatibility_work, policy_call) __all__ = () @@ -181,12 +184,19 @@ class LifecycleCoordinator: join the queue instead of recursively overtaking the current lifecycle. """ - def __init__(self, cluster=None): + def __init__(self, cluster=None, runtime=None): self._cluster = weakref.proxy(cluster) if cluster is not None else None + self._cluster_ref = weakref.ref(cluster) if cluster is not None else lambda: None self._lock = RLock() self._local = local() self._events = deque() + self._pending_observations = {} + self._runtime = runtime + self._runtime_lock = RLock() self._draining = False + self._active_event = False + self._idle = Event() + self._idle.set() self._requested = 0 self._published = 0 self._force_revisions = deque() @@ -209,7 +219,12 @@ def close(self): if self._closed: return self._closed = True + for event in self._events: + event[1].set() self._events.clear() + self._pending_observations.clear() + if not self._active_event: + self._idle.set() self._force_revisions.clear() hosts = list(self._retired.values()) if self._cluster is not None: @@ -231,6 +246,12 @@ def close(self): handler = host.get_and_set_reconnection_handler(None) if handler is not None: reconnectors.append(handler) + # Startup does not hold the publication lock while it handshakes with + # a reactor. Join its installation boundary only after releasing that + # lock, so a first-use race cannot install a runtime behind shutdown. + with self._runtime_lock: + if self._runtime is not None: + self._runtime.close() for handler in reconnectors: try: handler.cancel() @@ -269,7 +290,7 @@ def submit_readiness(self, host, callback): with self.transaction(): with host.lock: generation = getattr(host, '_readiness_generation', 0) - self.submit(callback, host, generation) + self.submit_observation(host, 'readiness', callback, host, generation) def readiness_is_current(self, host, generation=None): return (self.is_current(host) and @@ -396,7 +417,7 @@ def transaction(self, drain=True): finally: self._local.depth -= 1 finally: - if drain and not getattr(self._local, 'depth', 0): + if (drain or self._draining) and not getattr(self._local, 'depth', 0): self.drain() def reserve_refresh(self, force_token_rebuild=False): @@ -421,41 +442,204 @@ def accept_refresh(self, revision): self._published = revision return True - def submit(self, callback, *args, **kwargs): - """Queue a notification; delivery never has a synchronous result.""" + def _ensure_runtime(self): + # First use occurs after Cluster dependencies have been constructed. + # Never wait for a reactor handshake while holding publication locks. + with self._runtime_lock: + if self.closed: + raise ManagementClosed('Lifecycle coordinator is closed') + if self._runtime is None: + if self._cluster is None: + from cassandra.io.asyncioreactor import AsyncioConnection + connection_class = AsyncioConnection + else: + connection_class = self._cluster.connection_class + runtime = ManagementRuntime(None) + try: + runtime.start() + channel = ReactorChannel(connection_class) + runtime._dispatch = channel.dispatch + runtime._on_finalized = channel.close + if self.closed: + raise ManagementClosed('Lifecycle coordinator closed during startup') + except BaseException: + runtime.close() + runtime.join(5) + raise + self._runtime = runtime + weakref.finalize(self, runtime.close) + if self._cluster is not None: + # The coordinator stores only a proxy; the Cluster can be + # collected even while this runtime is idle. + cluster = self._cluster_ref() + if cluster is not None: + weakref.finalize(cluster, runtime.close) + return self._runtime + + def _queue_event(self, callback): + if self.closed: + return None + try: + runtime = self._ensure_runtime() + except ManagementClosed: + return None + ready = Event() + event = [callback, ready, None] + + def deliver(): + with self._lock: + if self.closed or event not in self._events: + return + self._events.remove(event) + self._active_event = True + try: + self._local.delivering = True + callback() + except Exception: + log.exception("Error delivering host lifecycle notification") + finally: + self._local.delivering = False + with self._lock: + self._active_event = False + if not self._events: + self._draining = False + self._idle.set() + with self._lock: if self.closed: - return - self._events.append(partial(callback, *args, **kwargs)) + return None + try: + event[2] = runtime.notify(deliver, ready=ready) + except ManagementClosed: + return None + self._events.append(event) + if self._draining and not getattr(self._local, 'depth', 0): + # An active drainer already owns delivery. In particular an + # asynchronous producer inside its callback must not depend + # on a separate executor draining again after that callback. + runtime.release_notifications([ready]) + return event + + def submit(self, callback, *args, **kwargs): + """Admit an ordered notification on the serial compatibility lane.""" + self._queue_event(partial(callback, *args, **kwargs)) if not getattr(self._local, 'depth', 0): self.drain() - def submit_async(self, executor, callback, *args, **kwargs): - """Queue before executor handoff so later notifications cannot overtake.""" - event = partial(callback, *args, **kwargs) + def _defer_observation(self, host, family, callback, removed=False): + """One pending batch per known incarnation; no per-call overflow list. + + These are unadmitted, waiter-free compatibility observations. Accepted + batches keep their original credits and generation checks. Removal + replaces pending positive demand; families retain only their latest + generation, not a history of every producer call. + """ + if family not in ('readiness', 'location', 'pool-results', 'down', 'remove'): + raise ValueError('Unknown lifecycle observation family') + if self.closed: + return False + try: + runtime = self._ensure_runtime() + except ManagementClosed: + return False with self._lock: if self.closed: - return None - self._events.append(event) + return False + known = (self.is_current(host) or self._retired.get(id(host)) is host) + if not known: + # There is no lifecycle owner for an unknown incarnation. + # Report rejection to the producer instead of pretending an + # unconsumed refresh-demand bit will recover its completion. + return False + entry = self._pending_observations.setdefault(id(host), (host, {})) + pending = entry[1] + if removed: + pending.clear() + elif 'remove' in pending: + return True + pending.pop(family, None) + pending[family] = callback + runtime.register_health(host) + + def deliver(): + with self._lock: + entry = self._pending_observations.pop(id(host), None) + callbacks = tuple(entry[1].values()) if entry is not None else () + self._active_event = self._draining = True + self._idle.clear() + try: + self._local.delivering = True + for callback in callbacks: + try: + callback() + except Exception: + log.exception('Error delivering deferred lifecycle observation') + finally: + self._local.delivering = False + with self._lock: + self._active_event = False + if not self._events: + self._draining = False + self._idle.set() + if host._is_removed: + runtime.unregister_health(host) + + return runtime.observe_negative(host, None, deliver, removed=removed) + + def submit_observation(self, host, family, callback, *args, **kwargs): + notification = partial(callback, *args, **kwargs) + try: + self.submit(notification) + except ManagementOverloaded: + if (not self._defer_observation(host, family, notification) and + not self.closed and not host._is_removed): + raise + + def _retain_negative(self, host, callback, removed=False): + return self._defer_observation(host, 'remove' if removed else 'down', + callback, removed=removed) + + def submit_negative(self, host, callback, *args, removed=False, **kwargs): + """Retain a native negative transition when ordinary delivery is full.""" + notification = partial(callback, *args, **kwargs) + try: + self.submit(notification) + except ManagementOverloaded: + self._retain_negative(host, notification, removed) + + def submit_async(self, executor, callback, *args, **kwargs): + """Retain admission-before-handoff behavior for existing producers.""" + try: + event = self._queue_event(partial(callback, *args, **kwargs)) + except ManagementOverloaded: + if getattr(callback, '_negative_lifecycle', False): + host = args[1] + if self._retain_negative(host, partial(callback, *args, **kwargs)): + completed = Future() + completed.set_result(None) + return completed + raise + if event is None: + return None try: return executor.submit(self.drain) except Exception: log.exception("Failed to schedule host lifecycle notifications") - # If no drainer has claimed this event, failed handoff cancels - # admission. Otherwise a concurrent submit already delivered it. with self._lock: - try: + if event in self._events: self._events.remove(event) - except ValueError: - if self.closed: - return None - else: + self._runtime.release_notifications([event[1]]) + if not self._events and not self._active_event: + self._draining = False + self._idle.set() + return None + if self.closed: return None completed = Future() completed.set_result(None) return completed - def collect_futures(self, futures, callback): + def collect_futures(self, futures, callback, host=None): """Deliver all results once, with aggregation owned by the FIFO. Register only after the caller has scheduled every operation. Worker @@ -464,6 +648,31 @@ def collect_futures(self, futures, callback): delivered as results so the caller can apply its recovery policy. Closing the coordinator discards outstanding completion notifications. """ + if host is not None: + # Physical outcomes remain in their original futures; retain only + # one completion observation for this existing Host incarnation. + pending = set(futures) + lock = Lock() + results = [] + + def completed(future): + with lock: + if future not in pending: + return + pending.remove(future) + try: + results.append(future.result()) + except Exception as exc: + results.append(exc) + finished = not pending + if finished: + self.submit_observation(host, 'pool-results', callback, results) + + if not pending: + self.submit_observation(host, 'pool-results', callback, results) + for future in tuple(pending): + future.add_done_callback(completed) + return pending = set(futures) results = [] @@ -632,8 +841,9 @@ def retire(self, host, trigger_reconciliation=True, refresh_nodes=True): with self.transaction(): if not self._retire(host): return - self.submit(self._cluster._notify_host_removed, host, - trigger_reconciliation, True, refresh_nodes) + self.submit_negative(host, self._cluster._notify_host_removed, host, + trigger_reconciliation, True, refresh_nodes, + removed=True) def remove(self, host, trigger_reconciliation=True, refresh_nodes=True): with self.transaction(): @@ -642,8 +852,9 @@ def remove(self, host, trigger_reconciliation=True, refresh_nodes=True): return self._retire(host) log.info("Cassandra host %s removed", host) - self.submit(self._cluster._notify_host_removed, host, - trigger_reconciliation, True, refresh_nodes) + self.submit_negative(host, self._cluster._notify_host_removed, host, + trigger_reconciliation, True, refresh_nodes, + removed=True) def publish_host(self, candidate, signal, refresh_nodes, reconcile_pools_on_failure, replace_existing_id): @@ -734,8 +945,8 @@ def _queue_location(self, host, datacenter, rack): token = object() host._pending_location = desired host._pending_location_token = token - self.submit( - self._notify_location_changed, host, datacenter, rack, token) + self.submit_observation( + host, 'location', self._notify_location_changed, host, datacenter, rack, token) return True def _notify_location_changed(self, host, datacenter, rack, token): @@ -760,29 +971,94 @@ def _notify_location_changed(self, host, datacenter, rack, token): host._pending_location_token = None def drain(self): + """Release transaction batches; preserve idle synchronous observations. + + An already active lane is never joined by another producer or a + reentrant listener. Native operation results do not depend on this lane. + """ with self._lock: - if self._draining or self.closed: + if self.closed: return - self._draining = True - try: - while True: - with self._lock: - if self.closed or not self._events: + events = tuple(self._events) + should_wait = not self._draining and not self.delivering + if events: + self._draining = True + self._idle.clear() + if events: + self._runtime.release_notifications(event[1] for event in events) + reactor_context = (self._cluster is not None and + getattr(self._cluster.connection_class, + '_management_in_context', lambda: False)()) + if (should_wait and not reactor_context and + execution_context() not in ('cluster', 'pool', 'bridge', 'worker')): + while not self._idle.wait(0.05): + # Runtime infrastructure failure may discard accepted + # notifications without running our delivery closure. Never + # leave a caller waiting for callbacks that cannot now run. + runtime = self._runtime + if runtime is not None and runtime.closed.is_set(): + with self._lock: self._events.clear() - self._draining = False - return - callback = self._events.popleft() - try: - self._local.delivering = True - callback() - except Exception: - log.exception("Error delivering host lifecycle notification") - finally: - self._local.delivering = False - except BaseException: - with self._lock: - self._draining = False - raise + self._pending_observations.clear() + if not self._active_event: + self._draining = False + self._idle.set() + + +def managed_context(cluster): + if cluster is None: + return False # Bare Session.__new__ tests have no runtime lifetime. + return (execution_context() is not None or + getattr(getattr(cluster, 'connection_class', None), + '_management_in_context', lambda: False)() is True) + + +def defer_cleanup(cluster, callback): + runtime = cluster._lifecycle._runtime + release = runtime.retain_cleanup() if runtime is not None else lambda: None + def cleanup(): + try: + with physical_work_context(cleanup=True): + callback() + finally: + release() + try: + return cluster.executor.submit(cleanup) + except BaseException as exc: + if runtime is not None: + runtime.cleanup_error = exc + release() + log.exception('Failed to submit management lifetime cleanup') + raise + + +def synchronous_api(method): + @wraps(method) + def invoke(self, *args, **kwargs): + cluster = self if hasattr(self, '_lifecycle') else self.cluster + assert_synchronous_api(cluster.connection_class) + if cluster.is_shutdown or not cluster._management_start_allowed(): + return method(self, *args, **kwargs) + coordinator = cluster._lifecycle + runtime = coordinator._ensure_runtime() + with compatibility_call(runtime): + return method(self, *args, **kwargs) + return invoke + + +def observation_event(family): + def decorate(method): + @wraps(method) + def dispatch(self, host, *args, **kwargs): + return self._lifecycle.submit_observation( + host, family, method, self, host, *args, **kwargs) + return dispatch + return decorate + + +def negative_event(method): + method._negative_lifecycle = True + return event(method) def event(method): diff --git a/cassandra/_management.py b/cassandra/_management.py index 7bb9e245de..241b49b3a7 100644 --- a/cassandra/_management.py +++ b/cassandra/_management.py @@ -728,19 +728,32 @@ def physical_work(): receive(None, exc) return _PENDING - def notify(self, callback, *, wait=False, timeout=None, budget='ordinary'): + def notify(self, callback, *, wait=False, timeout=None, budget='ordinary', + ready=None): + """Admit FIFO delivery, optionally gated by a caller publication batch. + + A gate is scheduler state: handlers never wait on the submitting + thread. Release it through ``release_notifications`` after publication. + """ self._check_pid() def admit(record): with self._condition: if self._closing or record.handler_done or record.released: return _PENDING record.notification = True - self._notifications.append((record, callback)) + self._notifications.append((record, callback, ready)) self._condition.notify_all() return _PENDING return self.submit(admit, wait=wait, timeout=timeout, budget=budget, kind='notification') + def release_notifications(self, gates): + self._check_pid() + with self._condition: + for gate in gates: + gate.set() + self._condition.notify_all() + @contextmanager def synchronous_reentry(self): """Reserve one L aggregate and a bounded prerequisite window.""" @@ -1047,11 +1060,14 @@ def _execute(self, record, callback, kind): def _run_notifications(self): while True: with self._condition: - while not self._notifications and not self._closing: + while (not self._closing and + (not self._notifications or + (self._notifications[0][2] is not None and + not self._notifications[0][2].is_set()))): self._condition.wait() if self._closing: return - record, callback = self._notifications.popleft() + record, callback, ready = self._notifications.popleft() with _entered('callback', record): try: value = callback() @@ -1066,7 +1082,7 @@ def _run_notifications(self): with self._condition: record.notification = False self._release_locked(record) - record = callback = value = None + record = callback = ready = value = None def retain_cleanup(self): """Retain one cleanup obligation independently of load.""" @@ -1097,7 +1113,7 @@ def close(self, wait=False): for record in tuple(self._records): self._finish(record, error=ManagementClosed( 'Management lifetime ended')) - for record, _ in self._notifications: + for record, _, _ in self._notifications: record.notification = False self._release_locked(record) self._notifications.clear() diff --git a/cassandra/cluster.py b/cassandra/cluster.py index cd779725ed..3f6918e088 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -690,6 +690,7 @@ def _profiles_without_explicit_lbps(self): for n in names ) + @_lifecycle.policy_call def distance(self, host): if not self.pools_allowed: return HostDistance.IGNORED @@ -699,14 +700,17 @@ def distance(self, host): HostDistance.REMOTE if HostDistance.REMOTE in distances else \ HostDistance.IGNORED + @_lifecycle.policy_call def populate(self, cluster, hosts): for p in self.profiles.values(): p.load_balancing_policy.populate(cluster, hosts) + @_lifecycle.policy_call def check_supported(self): for p in self.profiles.values(): p.load_balancing_policy.check_supported() + @_lifecycle.policy_call def on_up(self, host): if not self.pools_allowed: return @@ -715,6 +719,7 @@ def on_up(self, host): return p.load_balancing_policy.on_up(host) + @_lifecycle.policy_call def on_down(self, host): if not self.pools_allowed: return @@ -723,12 +728,14 @@ def on_down(self, host): if host._is_removed: return + @_lifecycle.policy_call def on_add(self, host): for p in self.profiles.values(): if host._is_removed: return p.load_balancing_policy.on_add(host) + @_lifecycle.policy_call def on_remove(self, host): for p in self.profiles.values(): p.load_balancing_policy.on_remove(host) @@ -1764,6 +1771,10 @@ def __init__(self, schema_metadata_enabled, token_metadata_enabled, schema_meta_page_size=schema_metadata_page_size) + @staticmethod + def _management_start_allowed(): + return not _cluster_scheduler_shutdown_started + def _resolve_hostnames(self): raw_contact_points = [] for cp in [cp for cp in self.contact_points if not isinstance(cp, EndPoint)]: @@ -2084,6 +2095,7 @@ def protocol_downgrade(self, host_endpoint, previous_version): "https://python-driver.docs.scylladb.com/stable/api/cassandra/cluster.html#cassandra.cluster.Cluster.protocol_version", self.protocol_version, new_version, host_endpoint) self.protocol_version = new_version + @_lifecycle.policy_call def _populate_hosts(self): self.profile_manager.populate( weakref.proxy(self), self.metadata.all_hosts()) @@ -2091,6 +2103,7 @@ def _populate_hosts(self): weakref.proxy(self), self.metadata.all_hosts() ) + @_lifecycle.synchronous_api def connect(self, keyspace=None, wait_for_all_pools=False): """ Creates and returns a new :class:`~.Session` object. @@ -2102,6 +2115,7 @@ def connect(self, keyspace=None, wait_for_all_pools=False): established or attempted. Default is `False`, which means it will return when the first successful connection is established. Remaining pools are added asynchronously. """ + _lifecycle.assert_synchronous_api(self.connection_class) reject_late_connect = False shutdown_rejected_cluster = False with self._lock: @@ -2207,6 +2221,14 @@ def shutdown(self): else: self.is_shutdown = True + if _lifecycle.managed_context(self): + try: + self._shutdown_future = _lifecycle.defer_cleanup( + self, partial(self._shutdown, wait_for_executor=False)) + finally: + self._lifecycle.close() + return + # Once interpreter scheduler cleanup has begun, executor teardown is # imminent and this call may itself be running on an executor worker. self._shutdown( @@ -2216,7 +2238,7 @@ def _shutdown(self, wait_for_executor): """Finish cleanup after shutdown has been claimed under ``self._lock``.""" self._lifecycle.close() - if self._lifecycle.delivering: + if self._lifecycle.delivering or _lifecycle.execution_context() is not None: wait_for_executor = False if self._idle_heartbeat: @@ -2385,12 +2407,12 @@ def _on_up_prepared(self, host, generation): if futures: self._lifecycle.collect_futures( futures, partial(self._on_up_pools_completed, host, - generation=generation)) + generation=generation), host=host) except Exception: log.exception("Unexpected failure handling node %s being marked up:", host) for future in futures: future.cancel() - + self._on_up_preparation_failed(host, generation) raise else: if not futures: @@ -2471,7 +2493,7 @@ def _start_reconnector(self, host, is_host_addition, return reconnector @run_in_executor - @_lifecycle.event + @_lifecycle.negative_event def on_down_potentially_blocking(self, host, is_host_addition, down_event_generation, on_add_reconnection=None): @@ -2843,9 +2865,9 @@ def pools_completed(futures_results): addition_token=addition_token) return - self._lifecycle.collect_futures(futures, pools_completed) + self._lifecycle.collect_futures(futures, pools_completed, host=host) - @_lifecycle.event + @_lifecycle.observation_event('readiness') def _finalize_add(self, host, set_up=True, generation=None, addition_token=None, advance_health=True): if not self._lifecycle.mark_ready( @@ -3135,6 +3157,7 @@ def get_control_connection_host(self): connection = self.control_connection._connection return self.control_connection._get_host_for_connection(connection) + @_lifecycle.synchronous_api def refresh_schema_metadata(self, max_schema_agreement_wait=None): """ Synchronously refresh all schema metadata. @@ -3148,9 +3171,11 @@ def refresh_schema_metadata(self, max_schema_agreement_wait=None): An Exception is raised if schema refresh fails for any reason. """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("Schema metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_keyspace_metadata(self, keyspace, max_schema_agreement_wait=None): """ Synchronously refresh keyspace metadata. This applies to keyspace-level information such as replication @@ -3158,10 +3183,12 @@ def refresh_keyspace_metadata(self, keyspace, max_schema_agreement_wait=None): See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.KEYSPACE, keyspace=keyspace, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("Keyspace metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None): """ Synchronously refresh table metadata. This applies to a table, and any triggers or indexes attached @@ -3169,30 +3196,36 @@ def refresh_table_metadata(self, keyspace, table, max_schema_agreement_wait=None See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TABLE, keyspace=keyspace, table=table, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("Table metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_materialized_view_metadata(self, keyspace, view, max_schema_agreement_wait=None): """ Synchronously refresh materialized view metadata. See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TABLE, keyspace=keyspace, table=view, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("View metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_user_type_metadata(self, keyspace, user_type, max_schema_agreement_wait=None): """ Synchronously refresh user defined type metadata. See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.TYPE, keyspace=keyspace, type=user_type, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("User Type metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_user_function_metadata(self, keyspace, function, max_schema_agreement_wait=None): """ Synchronously refresh user defined function metadata. @@ -3201,10 +3234,12 @@ def refresh_user_function_metadata(self, keyspace, function, max_schema_agreemen See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.FUNCTION, keyspace=keyspace, function=function, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("User Function metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreement_wait=None): """ Synchronously refresh user defined aggregate metadata. @@ -3213,10 +3248,12 @@ def refresh_user_aggregate_metadata(self, keyspace, aggregate, max_schema_agreem See :meth:`~.Cluster.refresh_schema_metadata` for description of ``max_schema_agreement_wait`` behavior """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_schema(target_type=SchemaTargetType.AGGREGATE, keyspace=keyspace, aggregate=aggregate, schema_agreement_wait=max_schema_agreement_wait, force=True): raise DriverException("User Aggregate metadata was not refreshed. See log for details.") + @_lifecycle.synchronous_api def refresh_nodes(self, force_token_rebuild=False): """ Synchronously publish refreshed node-list and token metadata. @@ -3231,6 +3268,7 @@ def refresh_nodes(self, force_token_rebuild=False): An Exception is raised if node refresh fails for any reason. """ + _lifecycle.assert_synchronous_api(self.connection_class) if not self.control_connection.refresh_node_list_and_token_map(force_token_rebuild): raise DriverException("Node list was not refreshed. See log for details.") @@ -3561,12 +3599,15 @@ def default_serial_consistency_level(self, cl): _pool_generations = None _pool_intents = None _pool_creation_work = None + # Bare Session.__new__ fixtures have no admitted runtime lifetime. + _management_runtime = None _profile_manager = None _metrics = None _request_init_callbacks = None def __init__(self, cluster, hosts, keyspace=None): self.cluster = cluster + self._management_runtime = cluster._lifecycle._ensure_runtime() self.hosts = hosts self.keyspace = keyspace @@ -4233,11 +4274,20 @@ def shutdown(self): for future in initial_connect_futures if future in pool_creation_work) + cleanup = partial(self._finish_shutdown, initial_connect_work, + initial_connect_futures) + if (_lifecycle.managed_context(self.cluster) and + not _lifecycle.performing_cleanup()): + self._shutdown_future = _lifecycle.defer_cleanup(self.cluster, cleanup) + return + cleanup() + + def _finish_shutdown(self, initial_connect_work, initial_connect_futures): # PYTHON-673. If shutdown was called shortly after session init, avoid # a race by cancelling any initial connection attempts haven't started, # then blocking on any that have. for future in initial_connect_work: - future.cancel() + _cancel_and_notify(future) wait_futures(initial_connect_work) for future in initial_connect_futures: @@ -4284,21 +4334,25 @@ def _pool_intents_locked(self): return pool_intents def _submit_pool_creation_work(self, completion, fn, *args, **kwargs): - """Submit pool creation and retain its work behind its public proxy.""" - with self._lock: - work_future = self.submit(fn, *args, **kwargs) - add_done_callback = ( - getattr(work_future, 'add_done_callback', None) - if work_future is not None else None) + """Retain one charged physical result behind each legacy pool proxy.""" + def work(): + with _lifecycle.physical_work_context(): + return fn(*args, **kwargs) + + with _lifecycle.compatibility_work(self._management_runtime) as attach: + with self._lock: + work_future = self.submit(work) + add_done_callback = ( + getattr(work_future, 'add_done_callback', None) + if work_future is not None else None) + if callable(add_done_callback): + work_items = self._pool_creation_work + if work_items is None: + work_items = self._pool_creation_work = {} + work_items[completion] = work_future + attach(work_future) if callable(add_done_callback): - work = self._pool_creation_work - if work is None: - work = self._pool_creation_work = {} - work[completion] = work_future - - if callable(add_done_callback): - add_done_callback(partial( - self._pool_creation_work_done, completion)) + add_done_callback(partial(self._pool_creation_work_done, completion)) return work_future def _pool_creation_work_done(self, completion, work_future): @@ -5064,6 +5118,7 @@ def on_remove(self, host, trigger_reconciliation=True): else: self.remove_pool(host) + @_lifecycle.synchronous_api def set_keyspace(self, keyspace): """ Set the default keyspace for all queries made through this Session. @@ -5073,6 +5128,7 @@ def set_keyspace(self, keyspace): :class:`.InvalidRequest` if another Session is using the shared connection or another keyspace change is in progress. """ + _lifecycle.assert_synchronous_api(self.cluster.connection_class) self.execute('USE %s' % (protect_name(keyspace),)) def _set_keyspace_for_all_pools(self, keyspace, callback): diff --git a/cassandra/io/asyncioreactor.py b/cassandra/io/asyncioreactor.py index 20fe79b851..03745f5ab8 100644 --- a/cassandra/io/asyncioreactor.py +++ b/cassandra/io/asyncioreactor.py @@ -1,4 +1,5 @@ import threading +from collections import deque from cassandra.connection import Connection, ConnectionShutdown import sys @@ -7,7 +8,7 @@ import os import socket import ssl -from threading import Lock, Thread +from threading import Event, Lock, Thread, current_thread log = logging.getLogger(__name__) @@ -133,6 +134,7 @@ class AsyncioConnection(Connection): _lock = Lock() _loop_thread = None + _management_state = None _write_queue = None _write_queue_lock = None @@ -258,13 +260,72 @@ def initialize_reactor(cls): cls._pid = os.getpid() if cls._loop is None: assert cls._loop_thread is None - cls._loop = asyncio.new_event_loop() - # daemonize so the loop will be shut down on interpreter - # shutdown - cls._loop_thread = Thread( - target=cls._loop.run_forever, daemon=True, name="asyncio_thread" - ) - cls._loop_thread.start() + loop = asyncio.new_event_loop() + pending, management_lock = deque(), Lock() + accepting = [True] + + def drain(final=False): + count = 0 + while final or count < 64: + with management_lock: + if not pending: + return + callback = pending.popleft() + count += 1 + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + with management_lock: + if pending and accepting[0]: + loop.call_soon(drain) + + def run(): + try: + loop.run_forever() + finally: + with management_lock: + accepting[0] = False + drain(final=True) + + thread = Thread(target=run, daemon=True, + name="asyncio_thread") + ready = Event() + try: + loop.call_soon(ready.set) + thread.start() + if not ready.wait(5): + raise RuntimeError("Asyncio reactor startup did not complete") + except BaseException: + if thread.is_alive(): + loop.call_soon_threadsafe(loop.stop) + thread.join(5) + if not thread.is_alive(): + loop.close() + raise + cls._loop, cls._loop_thread = loop, thread + cls._management_state = (pending, management_lock, accepting, drain) + + @classmethod + def _management_dispatch(cls, callback): + loop = cls._loop + if cls._management_state is None: + raise RuntimeError("Asyncio reactor is unavailable") + pending, lock, accepting, drain = cls._management_state + with lock: + if (not accepting[0] or loop is None or loop.is_closed() or + not cls._loop_thread.is_alive()): + raise RuntimeError("Asyncio reactor is unavailable") + pending.append(callback) + try: + loop.call_soon_threadsafe(drain) + except BaseException: + pending.remove(callback) + raise + + @classmethod + def _management_in_context(cls): + return current_thread() is cls._loop_thread @classmethod def create_timer(cls, timeout, callback): @@ -381,3 +442,12 @@ async def handle_read(self): log.debug("Connection %s closed by server", self) self.close() return + + +def _reset_management_startup_lock_after_fork(): + # A parent thread may own this lock when another thread forks. + AsyncioConnection._lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index 4d19bb9849..79dbfeef2e 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -18,7 +18,7 @@ import os import socket import sys -from threading import Lock, Thread, Event +from threading import Lock, RLock, Thread, Event, current_thread import time import sys import ssl @@ -137,6 +137,14 @@ def notify_loop(self): self._notified = True os.write(self.write_fd, b'x') + def close(self): + try: + asyncore.dispatcher.close(self) + finally: + if self.write_fd is not None: + os.close(self.write_fd) + self.write_fd = None + class _AsyncoreUDPDispatcher(_AsyncoreDispatcher): """ @@ -210,8 +218,12 @@ class AsyncoreLoop(object): def __init__(self): self._pid = os.getpid() self._loop_lock = Lock() + self._state_lock = RLock() + self._management_callbacks = deque() + self._management_failed = False self._started = False self._shutdown = False + self._cleanup_started = False self._thread = None @@ -228,21 +240,46 @@ def __init__(self): self._loop_dispatcher = dispatcher def maybe_start(self): - should_start = False - did_acquire = False - try: - did_acquire = self._loop_lock.acquire(False) - if did_acquire and not self._started: - self._started = True - should_start = True - finally: - if did_acquire: - self._loop_lock.release() + with self._state_lock: + if self._shutdown or self._started: + return + self._started = True + self._thread = Thread(target=self._run_loop, + name="asyncore_cassandra_driver_event_loop", daemon=True) + try: + self._thread.start() + except BaseException: + if not self._thread.is_alive(): + self._started = False + self._thread = None + raise + + def call_soon(self, callback): + with self._state_lock: + if self._shutdown or self._management_failed: + raise RuntimeError("Asyncore reactor is shutting down") + self._management_callbacks.append(callback) + try: + self.maybe_start() + self.wake_loop() + except BaseException: + try: + self._management_callbacks.remove(callback) + except ValueError: + return + raise - if should_start: - self._thread = Thread(target=self._run_loop, name="asyncore_cassandra_driver_event_loop") - self._thread.daemon = True - self._thread.start() + def _dispatch_management(self): + for _ in range(64): + with self._state_lock: + if not self._management_callbacks: + return + callback = self._management_callbacks.popleft() + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + self.wake_loop() def wake_loop(self): self._loop_dispatcher.notify_loop() @@ -250,14 +287,23 @@ def wake_loop(self): def _run_loop(self): log.debug("Starting asyncore event loop") with self._loop_lock: - while not self._shutdown: - try: + try: + while not self._shutdown: self._loop_dispatcher.loop(self.timer_resolution) self._timers.service_timeouts() - except Exception as exc: - self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) - break - self._started = False + self._dispatch_management() + except BaseException as exc: + self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc) + finally: + # Management service loss is permanent for existing channels. + # Ordinary I/O can restart after this runner drains accepted + # completions and relinquishes ownership. + with self._state_lock: + self._management_failed = True + while self._management_callbacks: + self._dispatch_management() + with self._state_lock: + self._started = False self._maybe_log_debug("Asyncore event loop ended") @@ -281,13 +327,17 @@ def add_timer(self, timer): def _cleanup(self): global _dispatcher_map - self._shutdown = True - if not self._thread: - return + with self._state_lock: + if self._cleanup_started: + return + self._cleanup_started = True + self._shutdown = True + self.wake_loop() log.debug("Waiting for event loop thread to join...") - self._thread.join(timeout=1.0) - if self._thread.is_alive(): + if self._thread is not None and self._thread is not current_thread(): + self._thread.join(timeout=1.0) + if self._thread is not None and self._thread.is_alive(): log.warning( "Event loop thread could not be joined, so shutdown may not be clean. " "Please call Cluster.shutdown() to avoid this.") @@ -306,6 +356,7 @@ def _cleanup(self): _global_loop = None +_initialization_lock = Lock() atexit.register(partial(_cleanup, _global_loop)) @@ -320,15 +371,42 @@ class AsyncoreConnection(Connection, asyncore.dispatcher): @classmethod def initialize_reactor(cls): - global _global_loop - if not _global_loop: - _global_loop = AsyncoreLoop() - else: - current_pid = os.getpid() - if _global_loop._pid != current_pid: - log.debug("Detected fork, clearing and reinitializing reactor state") - cls.handle_fork() - _global_loop = AsyncoreLoop() + global _global_loop, _dispatcher_map + with _initialization_lock: + if _global_loop is not None and _global_loop._pid != os.getpid(): + # Close descriptors directly: Connection.close() can acquire + # parent-owned locks or schedule timers on a vanished runner. + for dispatcher in tuple(_dispatcher_map.values()): + try: + if dispatcher is _global_loop._loop_dispatcher: + dispatcher.close() + else: + dispatcher.socket.close() + except OSError: + pass + _dispatcher_map = {} + _global_loop = None + if _global_loop is None: + candidate = AsyncoreLoop() + ready = Event() + try: + candidate.call_soon(ready.set) + if not ready.wait(5): + raise RuntimeError("Asyncore reactor startup did not complete") + except BaseException: + candidate._cleanup() + raise + _global_loop = candidate + + @classmethod + def _management_dispatch(cls, callback): + if _global_loop is None: + raise RuntimeError("Asyncore reactor is unavailable") + _global_loop.call_soon(callback) + + @classmethod + def _management_in_context(cls): + return _global_loop is not None and current_thread() is _global_loop._thread @classmethod def handle_fork(cls): @@ -476,3 +554,12 @@ def writable(self): def readable(self): return self._readable or ((self.is_control_connection or self._continuous_paging_sessions) and not (self.is_defunct or self.is_closed)) + + +def _reset_management_startup_lock_after_fork(): + global _initialization_lock + _initialization_lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index f963d283ac..a4ee93bb63 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -17,7 +17,7 @@ import os import socket import ssl -from threading import Lock, RLock, Thread +from threading import Event, Lock, RLock, Thread, current_thread import time from cassandra import DependencyException @@ -70,7 +70,7 @@ def __init__(self): # re-entered by a thread that already holds it, via _is_owned(): # private, but threading.Condition relies on it too. self._lock = RLock() - self._lock_thread = Lock() + self._lock_thread = RLock() self._thread = None @@ -83,6 +83,9 @@ def __init__(self): self._closed_conns = set() # connections closed while their thread held _lock; drained by the loop self._deferred_destroyed = [] + self._management_callbacks = deque() + self._management_users = set() + self._cleanup_started = False self._preparer = libev.Prepare(self._loop, self._loop_will_run) # prevent _preparer from keeping the loop from returning @@ -91,23 +94,27 @@ def __init__(self): self._timers = TimerManager() self._loop_timer = libev.Timer(self._loop, self._on_loop_timer) + self._management_keeper = libev.Timer(self._loop, self._keep_management_alive) def maybe_start(self): - should_start = False - with self._lock: - if not self._started: - log.debug("Starting libev event loop") - self._started = True - should_start = True - - if should_start: - with self._lock_thread: - if not self._shutdown: + # Serialize all admissions with startup, including its failure path. + with self._lock_thread: + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + if not self._started: + log.debug("Starting libev event loop") + self._started = True self._thread = Thread(target=self._run_loop, name="event_loop") self._thread.daemon = True - self._thread.start() - - self._notifier.send() + try: + self._thread.start() + except BaseException: + if not self._thread.is_alive(): + self._started = False + self._thread = None + raise + self._notifier.send() def _run_loop(self): while True: @@ -124,12 +131,13 @@ def _run_loop(self): # connection_created(), then observes _started == False and # starts a fresh thread). There is no interleaving in which # the new connection is invisible to both. - if not self._shutdown and self._live_conns: + if not self._shutdown and (self._live_conns or self._management_callbacks): log.debug("Restarting event loop") continue else: # all Connections have been closed, no active watchers log.debug("All Connections currently closed, event loop ended") + shutdown = self._shutdown self._started = False # A connection may have been closed after the last # _loop_will_run() pass, which is where closed @@ -144,13 +152,19 @@ def _run_loop(self): self._close_conns(to_stop) break self._drain_deferred() + # Only the shutdown owner may drain after publishing stopped state. + # Ordinary idle exit allows a replacement runner to claim new work. + if shutdown: + while self._management_callbacks: + self._dispatch_management() def _cleanup(self): - # TODO: unguarded write; self._notify() below wakes _run_loop to re-check. - self._shutdown = True - if not self._thread: - return - + with self._lock_thread: + with self._lock: + if self._cleanup_started: + return + self._cleanup_started = True + self._shutdown = True # Stop the prepare watcher first to prevent race conditions. # It was unref'd when started, so ref the loop back first: stopping it # unbalanced leaves libev's active count at -1, and a loop thread still @@ -161,6 +175,7 @@ def _cleanup(self): self._loop.ref() self._preparer.stop() self._loop_timer.stop() + self._management_keeper.stop() # TODO: unguarded read; sets are copy-on-write, so at most a stale # snapshot (redundant idempotent close), never a torn read. @@ -176,15 +191,37 @@ def _cleanup(self): # PYTHON-752 Thread might have just been created and not started # Use longer timeout to allow proper cleanup with self._lock_thread: - self._thread.join(timeout=5.0) + thread = self._thread + if thread is not None and thread is not current_thread(): + thread.join(timeout=5.0) - if self._thread.is_alive(): + if self._thread is not None and self._thread.is_alive(): log.warning( "Event loop thread could not be joined, so shutdown may not be clean. " "Please call Cluster.shutdown() to avoid this.") log.debug("Event loop thread was joined") + def _cleanup_after_fork(self): + # Only this thread survived. Do not enter inherited Python locks, + # invoke connection callbacks, or join vanished parent threads. + self._shutdown = True + for conn in self._live_conns | self._new_conns | self._closed_conns: + try: + conn._socket.close() + except OSError: + pass + conn._read_watcher = conn._write_watcher = None + self._live_conns = self._new_conns = self._closed_conns = set() + self._deferred_destroyed = [] + self._management_callbacks.clear() + self._management_users.clear() + # Break watcher/callback cycles and release the native loop, whose + # destructor closes its internal wakeup descriptors in the child. + self._preparer = self._loop_timer = self._management_keeper = None + self._notifier = None + self._loop = None + def add_timer(self, timer): self._timers.add_timer(timer) self._notifier.send() # wake up in case this timer is earlier @@ -203,6 +240,82 @@ def _update_timer(self): def _on_loop_timer(self): self._timers.service_timeouts() + def call_soon(self, callback): + with self._lock_thread: + with self._lock: + if self._shutdown: + raise RuntimeError("Libev reactor is shutting down") + self._management_callbacks.append(callback) + try: + self.maybe_start() + except BaseException: + with self._lock: + try: + self._management_callbacks.remove(callback) + except ValueError: + return # Already claimed: the dispatch was accepted. + raise + + def _keep_management_alive(self): + if self._management_users and not self._shutdown: + self._management_keeper.start(3600) + # Cleanup or release can stop the timer while start() is in flight. + if self._shutdown or not self._management_users: + self._management_keeper.stop() + + def borrow_management(self): + token = object() + ready, abandoned = Event(), Event() + + def acquire(): + if abandoned.is_set(): + return + self._management_users.add(token) + self._keep_management_alive() + if abandoned.is_set(): + release(lambda: None) + else: + ready.set() + + def release(completed): + def detach(): + self._management_users.discard(token) + if not self._management_users: + self._management_keeper.stop() + completed() + if current_thread() is self._thread: + detach() + elif self._shutdown or self._thread is None or not self._thread.is_alive(): + # Shutdown closes admission before the runner finishes draining. + # Release remains idempotent while that runner unwinds. + with self._lock: + detach() + else: + self.call_soon(detach) + + if current_thread() is self._thread: + acquire() + else: + self.call_soon(acquire) + if not ready.wait(5): + abandoned.set() + release(lambda: None) + raise RuntimeError('Libev management borrow did not complete') + return release + + def _dispatch_management(self): + # Bounded batches keep management completions fair to socket I/O. + for _ in range(64): + with self._lock: + if not self._management_callbacks: + return + callback = self._management_callbacks.popleft() + try: + callback() + except BaseException: + log.exception("Error in management reactor dispatch") + self._notifier.send() + def notify(self): self._notifier.send() @@ -288,6 +401,7 @@ def _close_conns(conns): def _loop_will_run(self, prepare): changed = False self._drain_deferred() + self._dispatch_management() # TODO: unguarded read; copy-on-write set, self-heals next tick. for conn in self._live_conns: @@ -341,6 +455,7 @@ def _atexit_cleanup(): _global_loop = None +_initialization_lock = Lock() atexit.register(_atexit_cleanup) @@ -356,13 +471,35 @@ class LibevConnection(Connection): @classmethod def initialize_reactor(cls): global _global_loop - if not _global_loop: - _global_loop = LibevLoop() - else: - if _global_loop._pid != os.getpid(): - log.debug("Detected fork, clearing and reinitializing reactor state") - cls.handle_fork() - _global_loop = LibevLoop() + with _initialization_lock: + if _global_loop is not None and _global_loop._pid != os.getpid(): + _global_loop._cleanup_after_fork() + _global_loop = None + if _global_loop is None: + candidate = LibevLoop() + ready = Event() + try: + candidate.call_soon(ready.set) + if not ready.wait(5): + raise RuntimeError("Libev reactor startup did not complete") + except BaseException: + candidate._cleanup() + raise + _global_loop = candidate + + @classmethod + def _management_dispatch(cls, callback): + if _global_loop is None: + raise RuntimeError("Libev reactor is unavailable") + _global_loop.call_soon(callback) + + @classmethod + def _management_borrow(cls): + return _global_loop.borrow_management() + + @classmethod + def _management_in_context(cls): + return _global_loop is not None and current_thread() is _global_loop._thread @classmethod def handle_fork(cls): @@ -520,3 +657,12 @@ def push(self, data): with self._deque_lock: self.deque.extend(chunks) _global_loop.notify() + + +def _reset_management_startup_lock_after_fork(): + global _initialization_lock + _initialization_lock = Lock() + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork) diff --git a/tests/unit/io/test_asyncioreactor.py b/tests/unit/io/test_asyncioreactor.py index 76aca20ef3..31bef870da 100644 --- a/tests/unit/io/test_asyncioreactor.py +++ b/tests/unit/io/test_asyncioreactor.py @@ -41,7 +41,13 @@ def tearDownClass(cls): if skip_me: return if ASYNCIO_AVAILABLE and AsyncioConnection._loop: - AsyncioConnection._loop.stop() + loop = AsyncioConnection._loop + loop.call_soon_threadsafe(loop.stop) + AsyncioConnection._loop_thread.join(5) + assert not AsyncioConnection._loop_thread.is_alive() + loop.close() + AsyncioConnection._loop = None + AsyncioConnection._loop_thread = None @property def create_timer(self): diff --git a/tests/unit/io/test_libevreactor_shutdown.py b/tests/unit/io/test_libevreactor_shutdown.py index af5b420888..27bfe1e3e5 100644 --- a/tests/unit/io/test_libevreactor_shutdown.py +++ b/tests/unit/io/test_libevreactor_shutdown.py @@ -26,6 +26,7 @@ import tempfile import os from pathlib import Path +from unittest.mock import patch from cassandra import DependencyException @@ -216,6 +217,17 @@ def test_cleanup_with_fix_properly_shuts_down(self): from cassandra.io import libevreactor from cassandra.io.libevreactor import _cleanup, _atexit_cleanup + # Simulated process exit must not stop the shared reactor used by later + # tests. Restore its singleton even if an assertion below fails. + singleton = patch.object(libevreactor, '_global_loop', None) + singleton.start() + self.addCleanup(singleton.stop) + def close_private_loop(): + loop = libevreactor._global_loop + if loop is not None and not loop._shutdown: + _cleanup(loop) + self.addCleanup(close_private_loop) + LibevConnection.initialize_reactor() # Verify the loop exists diff --git a/tests/unit/test_cluster.py b/tests/unit/test_cluster.py index 063020ca6b..716f285264 100644 --- a/tests/unit/test_cluster.py +++ b/tests/unit/test_cluster.py @@ -2731,7 +2731,7 @@ def test_initialized_cluster_connect_after_scheduler_cleanup_is_rejected(self): allow_control_connection_query_fallback= ControlConnectionQueryFallback.SkipPoolCreation, ) -with patch.object(cluster.connection_class, 'initialize_reactor'), \ +with patch.object(cluster.connection_class, 'initialize_reactor', wraps=cluster.connection_class.initialize_reactor), \ patch.object(cluster.control_connection, 'connect'), \ patch.object(cluster, '_populate_hosts'), \ patch.object(cluster.profile_manager, 'check_supported'): @@ -2792,7 +2792,7 @@ def test_late_connect_does_not_join_reentrant_executor_shutdown(self): allow_control_connection_query_fallback= ControlConnectionQueryFallback.SkipPoolCreation, ) -with patch.object(cluster.connection_class, 'initialize_reactor'), \ +with patch.object(cluster.connection_class, 'initialize_reactor', wraps=cluster.connection_class.initialize_reactor), \ patch.object(cluster.control_connection, 'connect'), \ patch.object(cluster, '_populate_hosts'), \ patch.object(cluster.profile_manager, 'check_supported'): @@ -4130,7 +4130,7 @@ def shutdown_before_submit(fn, *args, **kwargs): cluster.sessions = (session, live_session) cluster._finalize_add = Mock() - def collect_futures(futures, callback): + def collect_futures(futures, callback, host=None): callback([future.exception(timeout=5) or future.result(timeout=5) for future in futures]) @@ -6692,7 +6692,7 @@ class _ResumableConnection(Connection): supports_tls_session_resumption = True -class _NonResumableConnection(Connection): +class _NonResumableConnection(connection_class): supports_tls_session_resumption = False diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 7d2b7e7359..53ef1f9d92 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -47,6 +47,7 @@ ConstantReconnectionPolicy, ExponentialReconnectionPolicy, IdentityTranslator) +from tests import connection_class from tests.unit.utils import new_session_with_pool_state PEER_IP = "foobar" @@ -161,6 +162,7 @@ def remove_host_by_host_id(self, host_id, endpoint=None): class MockCluster(object): + connection_class = connection_class max_schema_agreement_wait = 5 profile_manager = ProfileManager() reconnection_policy = ConstantReconnectionPolicy(2) diff --git a/tests/unit/test_lifecycle_coordination.py b/tests/unit/test_lifecycle_coordination.py index db22525ad4..6c5ae2d72a 100644 --- a/tests/unit/test_lifecycle_coordination.py +++ b/tests/unit/test_lifecycle_coordination.py @@ -13,7 +13,7 @@ # limitations under the License. from concurrent.futures import CancelledError, Future -from threading import Barrier, Event, Lock, Thread, get_ident +from threading import Barrier, Event, Lock, RLock, Thread, get_ident from unittest.mock import Mock, patch import pytest @@ -33,6 +33,23 @@ from tests.unit.utils import new_session_with_pool_state +@pytest.fixture(autouse=True) +def close_test_coordinators(monkeypatch): + coordinators = [] + initialize = LifecycleCoordinator.__init__ + + def tracked(self, *args, **kwargs): + initialize(self, *args, **kwargs) + coordinators.append(self) + + monkeypatch.setattr(LifecycleCoordinator, '__init__', tracked) + yield + for coordinator in coordinators: + coordinator.close() + if coordinator._runtime is not None: + coordinator._runtime.join(5) + + def test_pool_results_wait_for_every_operation_and_preserve_failures(): coordinator = LifecycleCoordinator() callback = Mock() @@ -91,7 +108,7 @@ def hold_drainer(): assert not drainer.is_alive() assert len(observations) == 1 thread_id, results = observations[0] - assert thread_id == drainer.ident + assert thread_id == coordinator._runtime._threads[3].ident assert sorted(results) == list(range(8)) @@ -179,7 +196,8 @@ def finish_cleanup(): session.remove_pool.assert_called_once_with(host) session.update_created_pools.assert_called_once_with() assert observations == [ - ('notification', drainer.ident), ('reconcile', drainer.ident)] + ('notification', cluster._lifecycle._runtime._threads[3].ident), + ('reconcile', cluster._lifecycle._runtime._threads[3].ident)] @pytest.mark.parametrize('completion', ['queued', 'late']) @@ -885,7 +903,7 @@ def shutdown_from_listener(_host): [result], lambda results: cluster._on_up_pools_completed(host, results)) finished.result(timeout=5) assert cluster.is_shutdown - assert shutdown_returned.is_set() + assert shutdown_returned.wait(5) assert cluster._lifecycle.closed assert not host._currently_handling_node_up listener.on_up.assert_called_once_with(host) @@ -2575,6 +2593,7 @@ def take_then_shutdown(replacement): host.get_and_set_reconnection_handler = take_then_shutdown cluster.remove_host(host, refresh_nodes=False) + cluster._lifecycle._runtime.join(5) handler.cancel.assert_called_once_with() @@ -2874,3 +2893,643 @@ def stop(): if shutdown is not None: assert not shutdown.is_alive() assert not shutdown_errors + + +@pytest.mark.parametrize('transition', ['down', 'remove']) +@pytest.mark.parametrize('backlog', ['io', 'callbacks']) +def test_negative_producer_survives_full_ordinary_admission(topology, transition, backlog): + from cassandra._management import ManagementOverloaded + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + cluster._discount_down_events = False + cluster._start_reconnector = Mock() + cluster.control_connection.on_down = Mock() + cluster.control_connection.on_remove = Mock() + cluster._lifecycle._ensure_runtime() + manager = cluster._lifecycle._runtime + manager.limit = 2 + entered, release = Event(), Event() + listener = Mock() + callback = listener.on_down if transition == 'down' else listener.on_remove + cluster.register_listener(listener) + physical = [] + if backlog == 'io': + class ControlledExecutor: + def submit(self, function): + future = Future() + physical.append(future) + entered.set() + return future + executor = ControlledExecutor() + futures = [manager.submit(lambda r: manager.work(r, executor, lambda: 1)) + for _ in range(2)] + assert entered.wait(5) + # C processes both admission envelopes before the transition is sent. + # Polling native record state uses a condition, not a timing delay. + with manager._condition: + assert manager._condition.wait_for( + lambda: all(f._record.physical for f in futures), timeout=5) + else: + def blocked_listener(): + entered.set() + assert release.wait(5) + first = manager.notify(blocked_listener) + assert entered.wait(5) + second = manager.notify(lambda: None) + futures = [first, second] + try: + with pytest.raises(ManagementOverloaded): + manager.submit(lambda record: None) + if transition == 'down': + assert cluster.on_down(host, False) + assert host.is_up is False + else: + cluster.remove_host(host, refresh_nodes=False) + assert host._is_removed + assert cluster.metadata.get_host_by_host_id(host.host_id) is None + callback.assert_not_called() + assert len(manager._health) == 1 + finally: + release.set() + for future in physical: + future.set_result(None) + # An explicit C observation runs after native negative slot consumption. + # Completion is signalled by the actual application callback. + observed = Event() + callback.side_effect = lambda *args: observed.set() + if callback.call_count: + observed.set() + assert observed.wait(5) + callback.assert_called_once_with(host) + for future in futures: + future.result(5) + + +def test_deferred_observation_replacement_keeps_latest_causal_order(topology): + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + entered, release, complete = Event(), Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + observed = [] + try: + cluster._lifecycle.submit_readiness( + host, lambda *args: observed.append('superseded-readiness')) + cluster._lifecycle.update_location(host, 'dc2', 'rack2') + cluster._lifecycle.submit_readiness( + host, lambda *args: (observed.append((host.datacenter, host.rack)), complete.set())) + assert len(cluster._lifecycle._pending_observations) == 1 + assert list(cluster._lifecycle._pending_observations[id(host)][1]) == ['location', 'readiness'] + finally: + release.set() + active.result(5) + assert complete.wait(5) + assert observed == [('dc2', 'rack2')] + + +def test_deferred_positive_observation_follows_admitted_events(topology): + cluster, _ = topology + coordinator = cluster._lifecycle + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = coordinator._ensure_runtime() + manager.limit = 2 + entered, release, completed = Event(), Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + order = [] + try: + with coordinator.transaction(drain=False): + coordinator.submit(lambda: order.append('admitted')) + coordinator.submit_observation( + host, 'readiness', lambda: (order.append('deferred'), completed.set())) + assert manager.counts()['ordinary'] == 2 + assert id(host) in coordinator._pending_observations + release.set() + coordinator.drain() + assert completed.wait(5) + assert order == ['admitted', 'deferred'] + active.result(5) + finally: + release.set() + + +@pytest.mark.parametrize('running', [False, True]) +def test_up_pool_admission_failure_releases_claim_and_cancels_started_futures(topology, running): + from cassandra._management import ManagementOverloaded, compatibility_work + cluster, _ = topology + coordinator = cluster._lifecycle + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + host.set_down() + generation = getattr(host, '_readiness_generation', 0) + manager = coordinator._ensure_runtime() + manager.limit = 1 + entered, release = Event(), Event() + occupied = manager.submit(lambda record: (entered.set(), release.wait(5))) + assert entered.wait(5) + started = Future() + if running: + assert started.set_running_or_notify_cancel() + first, second = Mock(), Mock() + first.add_or_renew_pool.return_value = started + def submit_pool(*args, **kwargs): + with compatibility_work(manager): + pytest.fail('saturated pool work was admitted') + second.add_or_renew_pool.side_effect = submit_pool + cluster.sessions = [first, second] + cluster._cleanup_failed_on_up_handling = Mock() + try: + assert coordinator.begin_up(host, generation) + with pytest.raises(ManagementOverloaded): + cluster._on_up_prepared(host, generation) + assert started.cancelled() is not running + cluster._cleanup_failed_on_up_handling.assert_called_once_with(host) + assert not host._currently_handling_node_up + if running: + # Cancellation cannot own a physical job that already started. + # Its native result remains available after failed-UP cleanup. + started.set_result(True) + assert started.result(0) is True + assert not host._currently_handling_node_up + assert coordinator.begin_up(host, generation), 'later UP was stranded' + coordinator.finish_up(host, generation) + finally: + release.set() + occupied.result(5) + + +def test_transaction_readiness_never_waits_inside_notification_handler(monkeypatch): + from cassandra import _lifecycle + from cassandra._management import execution_context + class GuardedEvent(Event): + def wait(self, timeout=None): + assert execution_context() != 'callback', 'handler waited for transaction caller' + return super().wait(timeout) + coordinator = LifecycleCoordinator() + manager = coordinator._ensure_runtime() + monkeypatch.setattr(_lifecycle, 'Event', GuardedEvent) + delivered = [] + with coordinator.transaction(drain=False): + coordinator.submit(lambda: delivered.append(True)) + # C must remain usable while the transaction caller owns publication. + assert manager.submit(lambda record: 23).result(5) == 23 + drainer = Thread(target=coordinator.drain) + drainer.start() + try: + drainer.join(2) + assert not drainer.is_alive(), 'transaction readiness stranded drain' + assert delivered == [True] + finally: + coordinator.close() + drainer.join(5) + + +def test_unknown_host_overload_is_explicit(topology): + from cassandra._management import ManagementOverloaded + from cassandra.pool import Host + from cassandra.policies import SimpleConvictionPolicy + cluster, _ = topology + coordinator = cluster._lifecycle + from uuid import uuid4 + unknown = Host(DefaultEndPoint('127.0.0.99'), SimpleConvictionPolicy, host_id=uuid4()) + manager = coordinator._ensure_runtime() + manager.limit = 1 + entered, release = Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + callback = Mock() + try: + with pytest.raises(ManagementOverloaded): + coordinator.submit_observation(unknown, 'pool-results', callback, [True]) + callback.assert_not_called() + assert not coordinator._pending_observations + assert manager._unknown_refresh is None + finally: + release.set() + active.result(5) + + +@pytest.mark.parametrize('race', [False, True]) +def test_coordinator_close_cannot_leave_first_runtime_open(monkeypatch, race): + from cassandra import _lifecycle + from cassandra._management import ManagementClosed + coordinator = LifecycleCoordinator() + entered, release, closing, closure_claimed = Event(), Event(), Event(), Event() + original_idle_set = coordinator._idle.set + def idle_set(): + if coordinator._closed: + closure_claimed.set() + original_idle_set() + coordinator._idle.set = idle_set + original_channel = _lifecycle.ReactorChannel + created, outcomes = [], [] + def channel(*args, **kwargs): + if race: + entered.set() + assert release.wait(5) + return original_channel(*args, **kwargs) + original_start = _lifecycle.ManagementRuntime.start + def start(runtime): + if runtime not in created: + created.append(runtime) + return original_start(runtime) + monkeypatch.setattr(_lifecycle, 'ReactorChannel', channel) + monkeypatch.setattr(_lifecycle.ManagementRuntime, 'start', start) + def first_use(): + try: + coordinator._ensure_runtime() + outcomes.append('started') + except ManagementClosed: + outcomes.append('closed') + if not race: + coordinator.close() + try: + first_use() + finally: + for runtime in created: + runtime.close() + runtime.join(5) + assert not created + assert outcomes == ['closed'] + return + starter = Thread(target=first_use) + stopper = Thread(target=lambda: (coordinator.close(), closing.set())) + starter.start() + try: + assert entered.wait(5) + stopper.start() + # close claims closure before waiting for provisional installation. + assert closure_claimed.wait(5) + release.set() + starter.join(5) + stopper.join(5) + assert not starter.is_alive() and not stopper.is_alive() + assert closing.is_set() + assert outcomes == ['closed'] + assert len(created) == 1 + created[0].join(5) + assert created[0].finalized.is_set() + assert created[0].closed.is_set() + assert coordinator._runtime is None + finally: + release.set() + starter.join(5) + if stopper.ident is not None: + stopper.join(5) + for runtime in created: + runtime.close() + runtime.join(5) + + +def test_runtime_closure_releases_waiting_lifecycle_drain(): + coordinator = LifecycleCoordinator() + manager = coordinator._ensure_runtime() + entered, release, waiting, finished = Event(), Event(), Event(), Event() + active = manager.notify(lambda: (entered.set(), release.wait(5))) + assert entered.wait(5) + with coordinator.transaction(drain=False): + coordinator.submit(lambda: pytest.fail('closed notification ran')) + original_wait = coordinator._idle.wait + def observed_wait(timeout=None): + waiting.set() + return original_wait(timeout) + coordinator._idle.wait = observed_wait + drainer = Thread(target=lambda: (coordinator.drain(), finished.set())) + drainer.start() + try: + assert waiting.wait(5) + manager.close() + assert finished.wait(2), 'runtime closure stranded drain' + assert not coordinator._events + assert coordinator._idle.is_set() + finally: + release.set() + coordinator.close() + drainer.join(5) + from cassandra._management import ManagementClosed + with pytest.raises(ManagementClosed): + active.result(5) + + +def test_failed_async_handoff_releases_concurrent_drain(): + coordinator = LifecycleCoordinator() + manager = coordinator._ensure_runtime() + entered, reject, waiting, finished = Event(), Event(), Event(), Event() + callback_entered, release_callback = Event(), Event() + active = manager.notify(lambda: (callback_entered.set(), release_callback.wait(5))) + assert callback_entered.wait(5) + class RejectingExecutor: + def submit(self, callback): + entered.set() + assert reject.wait(5) + raise RuntimeError('executor rejected handoff') + submitter = Thread(target=lambda: coordinator.submit_async( + RejectingExecutor(), lambda: pytest.fail('withdrawn event ran'))) + original_wait = coordinator._idle.wait + def observed_wait(timeout=None): + waiting.set() + return original_wait(timeout) + coordinator._idle.wait = observed_wait + submitter.start() + drainer = Thread(target=lambda: (coordinator.drain(), finished.set())) + try: + assert entered.wait(5) + # Keep callback delivery behind an actual active notification while + # the other producer releases this event's transaction barrier. + # The callback lane must not consume the event before withdrawal. + drainer.start() + assert waiting.wait(5) + reject.set() + submitter.join(5) + assert finished.wait(2), 'withdrawn last event stranded drain' + assert not coordinator._events + assert coordinator._idle.is_set() + assert not coordinator._draining + finally: + reject.set() + release_callback.set() + active.result(5) + coordinator.close() + submitter.join(5) + if drainer.ident is not None: + drainer.join(5) + + +def test_async_submission_from_active_delivery_reuses_drainer(): + coordinator = LifecycleCoordinator() + executor = Mock() + order = [] + def callback(): + coordinator.submit_async(executor, lambda: order.append('inner')) + order.append('outer') + submitter = Thread(target=lambda: coordinator.submit(callback)) + submitter.start() + try: + submitter.join(2) + assert not submitter.is_alive(), 'nested asynchronous admission lost its active drainer' + assert order == ['outer', 'inner'] + executor.submit.assert_called_once_with(coordinator.drain) + assert not coordinator._events + finally: + coordinator.close() + submitter.join(5) + + +@pytest.mark.parametrize('api', ['connect', 'refresh', 'keyspace']) +def test_actual_listener_sync_wrappers_finish_at_full_ordinary_capacity(topology, api): + cluster, connection = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + cluster._is_setup = True + cluster.executor.shutdown() + from concurrent.futures import ThreadPoolExecutor + cluster.executor = ThreadPoolExecutor(1) + session = new_session_with_pool_state() + session.cluster = cluster + session.execute = lambda statement: cluster.executor.submit(lambda: None).result(5) + session._initial_connect_futures = set() + session.shutdown = Mock() + session._default_consistency_level = None + deferred = Event() + def create(keyspace): + cluster._lifecycle.submit_readiness(host, lambda *args: deferred.set()) + return session + cluster._new_session = create + entered, release, completed = Event(), Event(), Event() + results = [] + def listener(): + entered.set() + assert release.wait(5) + if api == 'connect': + results.append(cluster.connect(wait_for_all_pools=True) is session) + elif api == 'refresh': + # Actual topology reader/publisher, with its socket response mocked. + connection.peer_results[1][1][3:5] = ['new-dc', 'new-rack'] + connection.wait_for_responses.return_value = _node_meta_results( + connection.local_results, connection.peer_results) + cluster.refresh_nodes() + results.append(True) + else: + session.set_keyspace('ks') + results.append(True) + completed.set() + notification = manager.notify(listener) + assert entered.wait(5) + assert manager._credits == 1 + release.set() + assert completed.wait(5) + notification.result(5) + assert results == [True] + assert manager._reentry_active is False + if api == 'connect': + assert deferred.wait(5) + + +def test_actual_sync_refresh_waits_for_admission_before_side_effects(topology): + cluster, _ = topology + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + gate = Event() + held = manager.submit(lambda record: gate.wait(5)) + started, finished = Event(), Event() + control = cluster.control_connection.refresh_node_list_and_token_map + cluster.control_connection.refresh_node_list_and_token_map = Mock(return_value=True) + def caller(): + started.set() + cluster.refresh_nodes() + finished.set() + thread = Thread(target=caller) + thread.start() + try: + assert started.wait(5) + assert not finished.is_set() + cluster.control_connection.refresh_node_list_and_token_map.assert_not_called() + # Admission waits without retaining locks required by owner publication. + with cluster._lifecycle.transaction(drain=False), cluster._lock: + assert manager._credits == 1 + gate.set() + held.result(5) + assert finished.wait(5) + cluster.control_connection.refresh_node_list_and_token_map.assert_called_once_with(False) + finally: + gate.set() + thread.join(5) + cluster.control_connection.refresh_node_list_and_token_map = control + + +@pytest.mark.parametrize('raises', [False, True]) +def test_listener_policy_reentry_releases_native_phase_before_nested_refresh(topology, raises): + from cassandra.cluster import EXEC_PROFILE_DEFAULT + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + cluster._is_setup = True + observed = [] + class ReentrantPolicy(RoundRobinPolicy): + def distance(self, host): + from cassandra._management import execution_context + observed.append((execution_context(), manager._reentry_active)) + cluster.refresh_nodes() + if raises: + raise ValueError('policy rejected candidate') + return HostDistance.LOCAL + policy = ReentrantPolicy() + profile = ProfileManager() + profile.profiles[EXEC_PROFILE_DEFAULT] = ExecutionProfile(load_balancing_policy=policy) + cluster.profile_manager = profile + session = new_session_with_pool_state() + session.cluster = cluster + session.shutdown = Mock() + session._initial_connect_futures = set() + def create(keyspace): + cluster.profile_manager.distance(host) + return session + cluster._new_session = create + complete = Event() + def listener(): + if raises: + with pytest.raises(ValueError, match='policy rejected'): + cluster.connect() + else: + assert cluster.connect() is session + complete.set() + notification = manager.notify(listener) + assert complete.wait(5) + notification.result(5) + assert observed == [('callback', False)] + assert manager._reentry_active is False + + +def test_listener_connect_windows_real_pool_fanout_with_one_physical_worker(topology): + from concurrent.futures import ThreadPoolExecutor + from contextlib import nullcontext + from uuid import UUID + cluster, _ = topology + # Keep real Cluster.connect and Session construction; replace socket/pool + # construction only. 128 pools exceed the dedicated 64-result window. + cluster.metadata._hosts = {} + cluster.metadata._host_id_by_endpoint = {} + for number in range(128): + host, _ = cluster.add_host(DefaultEndPoint('127.0.0.%d' % (number + 1)), + signal=False, host_id=UUID(int=number + 10)) + host.set_up() + manager = cluster._lifecycle._ensure_runtime() + manager.limit = 1 + cluster._is_setup = True + cluster.executor.shutdown() + cluster.executor = ThreadPoolExecutor(1) + entered, release, connected = Event(), Event(), Event() + pools, sessions = [], [] + def create_pool(host, distance, session, **kwargs): + if not pools: + entered.set() + assert release.wait(5) + pool = Mock(host=host, host_distance=distance, is_shutdown=False, + _keyspace=None, _membership_lock=RLock()) + pool._lock_usable_connection.return_value = nullcontext(True) + pool.shutdown.side_effect = lambda: setattr(pool, 'is_shutdown', True) + pools.append(pool) + return pool + def listener(): + sessions.append(cluster.connect(wait_for_all_pools=True)) + connected.set() + with patch('cassandra.cluster.HostConnection', side_effect=create_pool): + notification = manager.notify(listener) + try: + assert entered.wait(5) + with manager._condition: + assert manager._condition.wait_for(lambda: manager._reentry_credits == 64, timeout=5) + assert manager._credits == 1 + assert manager._reentry_active + assert not connected.is_set() + finally: + release.set() + assert connected.wait(5) + notification.result(5) + assert len(pools) == len(sessions[0]._pools) == 128 + assert manager._reentry_credits == 0 + assert not manager._reentry_active + + +@pytest.mark.parametrize('context', ['cluster', 'pool', 'io']) +def test_managed_cluster_shutdown_defers_initial_pool_wait_and_retains_cleanup(topology, context): + from concurrent.futures import ThreadPoolExecutor + from contextlib import nullcontext + from cassandra._management import ManagementClosed + cluster, _ = topology + host = cluster.metadata.get_host_by_host_id(HOST_ID_3) + manager = cluster._lifecycle._ensure_runtime() + cluster.executor.shutdown() + cluster.executor = ThreadPoolExecutor(1) + session = new_session_with_pool_state() + session.cluster = cluster + session._management_runtime = manager + session._profile_manager = cluster.profile_manager + session._initial_connect_futures = set() + session.keyspace = None + session.is_shutdown = False + cluster.sessions.add(session) + entered, release, returned = Event(), Event(), Event() + candidate = Mock(host=host, is_shutdown=False, _keyspace=None, + _membership_lock=RLock()) + candidate._lock_usable_connection.return_value = nullcontext(True) + def construct(*args, **kwargs): + entered.set() + assert release.wait(5) + return candidate + with patch('cassandra.cluster.HostConnection', side_effect=construct): + initial = session.add_or_renew_pool(host, False) + session._initial_connect_futures.add(initial) + assert entered.wait(5) + def shutdown(*args): + cluster.shutdown() + returned.set() + try: + if context == 'io': + cluster.connection_class._management_dispatch(shutdown) + else: + manager.submit(shutdown, pool=object() if context == 'pool' else None) + assert returned.wait(5), 'shutdown waited on invoking execution context' + assert cluster.is_shutdown + assert manager.closed.is_set() + assert not manager.finalized.is_set() + assert manager._cleanup_obligations == 1 + finally: + release.set() + manager.join(5) + assert session.is_shutdown + candidate.shutdown.assert_called_once_with() + assert cluster.executor._shutdown + assert manager._cleanup_obligations == 0 + assert not manager._records + + +def test_managed_close_submission_rejection_is_recorded_without_inline_cleanup(topology): + cluster, _ = topology + manager = cluster._lifecycle._ensure_runtime() + original_submit = cluster.executor.submit + rejected, returned = RuntimeError('cleanup executor rejected'), Event() + effects = [] + cluster.executor.submit = Mock(side_effect=rejected) + def handler(record): + try: + with pytest.raises(RuntimeError, match='cleanup executor rejected'): + cluster.shutdown() + effects.append('returned') + finally: + returned.set() + try: + manager.submit(handler) + assert returned.wait(5) + manager.join(5) + assert manager.cleanup_error is rejected + assert effects == ['returned'] + assert manager._cleanup_obligations == 0 + # Transport cleanup has not been falsely reported as run. + assert not cluster.scheduler.is_shutdown + finally: + cluster.executor.submit = original_submit + cluster._shutdown(wait_for_executor=True) diff --git a/tests/unit/test_management_reactors.py b/tests/unit/test_management_reactors.py new file mode 100644 index 0000000000..768258cad2 --- /dev/null +++ b/tests/unit/test_management_reactors.py @@ -0,0 +1,754 @@ +"""Socket-free tests of the actual supported reactor completion adapters.""" + +import asyncio +import importlib +from concurrent.futures import CancelledError, Future +from threading import Event, Thread, current_thread, get_ident +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from cassandra import DependencyException +from cassandra._management import ( + ManagementContextError, ManagementRuntime, ReactorChannel, execution_context) +from cassandra.cluster import Cluster, Session + + +@pytest.fixture(autouse=True) +def close_borrowed_channels(monkeypatch): + channels = [] + initialize = ReactorChannel.__init__ + def tracked(self, *args, **kwargs): + initialize(self, *args, **kwargs) + channels.append(self) + monkeypatch.setattr(ReactorChannel, '__init__', tracked) + yield + for channel in channels: + detached = Event() + channel.close(detached.set) + assert detached.wait(5) + + +@pytest.fixture(params=['asyncio', 'libev', 'asyncore']) +def connection_class(request): + try: + module = importlib.import_module('cassandra.io.' + { + 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', + 'asyncore': 'asyncorereactor'}[request.param]) + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + return getattr(module, {'asyncio': 'AsyncioConnection', + 'libev': 'LibevConnection', + 'asyncore': 'AsyncoreConnection'}[request.param]) + + +def test_idle_reactor_completion_and_raw_io_sync_guard(connection_class): + channel = ReactorChannel(connection_class) + for _ in range(3): + finished = Event() + observations = [] + def on_io(): + assert execution_context() is None # Ordinary ResponseFuture context. + assert connection_class._management_in_context() + cluster = SimpleNamespace(connection_class=connection_class, + _lifecycle=None, control_connection=Mock()) + session = SimpleNamespace(cluster=cluster, execute=Mock()) + for call in (lambda: Cluster.connect(cluster), + lambda: Cluster.refresh_nodes(cluster), + lambda: Cluster.refresh_schema_metadata(cluster), + lambda: Session.set_keyspace(session, 'ks')): + with pytest.raises(ManagementContextError): + call() + cluster.control_connection.assert_not_called() + session.execute.assert_not_called() + observations.append(get_ident()) + finished.set() + channel.dispatch(on_io) + assert finished.wait(5) + assert observations[0] != get_ident() + + +def test_clusters_borrow_reactor_without_closing_other_channel(connection_class): + channel = ReactorChannel(connection_class) + first = ManagementRuntime(channel.dispatch, on_finalized=channel.close) + second_channel = ReactorChannel(connection_class) + second = ManagementRuntime(second_channel.dispatch, on_finalized=second_channel.close) + callback_entered, release, other_done = Event(), Event(), Event() + handler_entered, handler_release = Event(), Event() + try: + def handler(record): + handler_entered.set() + assert handler_release.wait(5) + return 3 + future = first.submit(handler) + assert handler_entered.wait(5) + def hold(future): + callback_entered.set() + assert release.wait(5) + future.add_done_callback(hold) + handler_release.set() + assert future.result(5) == 3 + assert callback_entered.wait(5) + first.close() + assert not first.finalized.is_set() + # Shared reactor may be blocked by application code; native state still + # progresses and the second channel works as soon as that code returns. + other = second.submit(lambda record: 8) + other.add_done_callback(lambda f: other_done.set()) + assert other.result(5) == 8 + release.set() + assert other_done.wait(5) + first.join(5) + assert second.submit(lambda record: 9).result(5) == 9 + finally: + handler_release.set() + release.set() + first.close() + second.close() + first.join(5) + second.join(5) + + +@pytest.mark.parametrize('terminal', ['result', 'error', 'cancel']) +@pytest.mark.parametrize('registered', ['before', 'after']) +def test_asyncio_chaining_progresses_while_listener_waits(connection_class, terminal, registered): + channel = ReactorChannel(connection_class) + manager = ManagementRuntime(channel.dispatch, on_finalized=channel.close) + application_loop = asyncio.new_event_loop() + app_thread = Thread(target=application_loop.run_forever) + app_thread.start() + entered, complete = Event(), Event() + release_handler = Event() + outcomes = [] + try: + def listener(): + def handler(record): + assert release_handler.wait(5) + if terminal == 'error': + raise ValueError('expected') + return 12 + source = manager.submit(handler) + if registered == 'after': + if terminal == 'cancel': + source.cancel() + release_handler.set() + try: + source.result(5) + except (ValueError, CancelledError): + pass + async def consume(): + wrapped = asyncio.wrap_future(source) + entered.set() + return await wrapped + chained = asyncio.run_coroutine_threadsafe(consume(), application_loop) + if registered == 'before': + assert entered.wait(5) + if terminal == 'cancel': + source.cancel() + release_handler.set() + try: + outcomes.append(chained.result(5)) + except (ValueError, CancelledError) as exc: + outcomes.append(type(exc)) + complete.set() + notification = manager.notify(listener) + assert complete.wait(5) + notification.result(5) + assert outcomes == [{'result': 12, 'error': ValueError, 'cancel': CancelledError}[terminal]] + finally: + release_handler.set() + try: + manager.close() + manager.join(5) + finally: + application_loop.call_soon_threadsafe(application_loop.stop) + app_thread.join(5) + assert not app_thread.is_alive() + application_loop.close() + + +def test_libev_dispatch_enqueued_during_idle_exit_is_not_lost(): + try: + module = importlib.import_module('cassandra.io.libevreactor') + except (ImportError, DependencyException) as exc: + pytest.skip(str(exc)) + loop = module.LibevLoop() + native = loop._loop + returned, release, first, second = Event(), Event(), Event(), Event() + class PausedExit: + paused = False + def start(self): + native.start() + if not self.paused: + self.paused = True + returned.set() + assert release.wait(5) + def __getattr__(self, name): + return getattr(native, name) + loop._loop = PausedExit() + try: + loop.call_soon(first.set) + assert first.wait(5) + assert returned.wait(5) + loop.call_soon(second.set) + release.set() + assert second.wait(5), 'idle exit lost a completion queued before exit committed' + loop._thread.join(5) + assert not loop._thread.is_alive() + finally: + release.set() + loop._cleanup() + + +def test_libev_retired_runner_does_not_claim_replacement_work(monkeypatch): + module = pytest.importorskip('cassandra.io.libevreactor') + loop = module.LibevLoop() + native, drain = loop._loop, loop._drain_deferred + retired, release_old, replacement, release_new, delivered = [Event() for _ in range(5)] + old_threads, observations = [], [] + def pause_retired(): + drain() + if not loop._started and not retired.is_set(): + old_threads.append(current_thread()) + retired.set() + assert release_old.wait(5) + class PausedReplacement: + def start(self): + if retired.is_set() and current_thread() is not old_threads[0]: + replacement.set() + assert release_new.wait(5) + native.start() + def __getattr__(self, name): + return getattr(native, name) + monkeypatch.setattr(loop, '_drain_deferred', pause_retired) + loop._loop = PausedReplacement() + def completion(): + observations.append((get_ident(), current_thread() is loop._thread)) + delivered.set() + try: + loop.call_soon(lambda: None) + assert retired.wait(5), 'runner did not publish its idle exit' + loop.call_soon(completion) + assert replacement.wait(5) + new_thread = loop._thread + assert new_thread is not old_threads[0] + release_old.set() + old_threads[0].join(5) + assert not old_threads[0].is_alive() + assert observations == [], 'retired runner consumed new-generation work' + release_new.set() + assert delivered.wait(5) + new_thread.join(5) + assert observations == [(new_thread.ident, True)] + assert not new_thread.is_alive() + finally: + release_old.set() + release_new.set() + loop._cleanup() + for thread in old_threads: + thread.join(5) + + +@pytest.mark.parametrize('arming_path', ['acquire', 'keepalive']) +def test_libev_keeper_arming_cannot_outlive_cleanup(monkeypatch, arming_path): + module = pytest.importorskip('cassandra.io.libevreactor') + loop = module.LibevLoop() + native = loop._management_keeper + arming, resume, stopped, armed = [Event() for _ in range(4)] + releases, errors = [], [] + class PausedKeeper: + active = False + def start(self, timeout): + arming.set() + assert resume.wait(5) + native.start(timeout) + self.active = True + armed.set() + def stop(self): + native.stop() + self.active = False + stopped.set() + keeper = PausedKeeper() + monkeypatch.setattr(loop, '_management_keeper', keeper) + def borrow(): + try: + releases.append(loop.borrow_management()) + except BaseException as exc: + errors.append(exc) + borrower = Thread(target=borrow) if arming_path == 'acquire' else None + stopper = None + try: + if borrower is not None: + borrower.start() + else: + loop._management_users.add(object()) + loop.call_soon(loop._keep_management_alive) + assert arming.wait(5) + runner = loop._thread + stopper = Thread(target=loop._cleanup) + stopper.start() + assert stopped.wait(5), 'cleanup did not stop the in-flight keeper' + assert loop._shutdown + resume.set() + assert armed.wait(5) + runner.join(5) + assert not runner.is_alive(), 'keeper was resurrected after cleanup' + assert not keeper.active + stopper.join(5) + assert not stopper.is_alive() + if borrower is not None: + borrower.join(5) + assert not borrower.is_alive() + assert errors == [] + assert len(releases) == 1 + finally: + resume.set() + keeper.stop() + loop.notify() + if borrower is not None: + borrower.join(5) + for release in releases: + release(lambda: None) + if stopper is not None: + stopper.join(5) + loop._cleanup() + + +def test_chaining_application_loop_closes_when_management_join_raises(monkeypatch, connection_class): + # Exercise the actual chaining test's failure cleanup, then supply our own + # fallback cleanup so a negative control cannot leak a non-daemon thread. + ReactorChannel(connection_class) + app_threads, app_loops = [], [] + make_loop, thread_class, join = asyncio.new_event_loop, Thread, ManagementRuntime.join + def new_loop(): + loop = make_loop() + app_loops.append(loop) + return loop + def new_thread(*args, **kwargs): + thread = thread_class(*args, **kwargs) + app_threads.append(thread) + return thread + def failed_join(manager, timeout): + join(manager, timeout) + raise RuntimeError('forced finalization failure') + monkeypatch.setattr(asyncio, 'new_event_loop', new_loop) + monkeypatch.setattr(__name__ + '.Thread', new_thread) + monkeypatch.setattr(ManagementRuntime, 'join', failed_join) + try: + with pytest.raises(RuntimeError, match='forced finalization failure'): + test_asyncio_chaining_progresses_while_listener_waits(connection_class, 'result', 'before') + assert len(app_threads) == len(app_loops) == 1 + assert not app_threads[0].is_alive() + assert app_loops[0].is_closed() + finally: + for loop, thread in zip(app_loops, app_threads): + if thread.is_alive(): + loop.call_soon_threadsafe(loop.stop) + thread.join(5) + if not thread.is_alive() and not loop.is_closed(): + loop.close() + + +def test_reactor_startup_lock_is_reset_in_fork_child(connection_class): + import os + import select + import signal + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + channel = ReactorChannel(connection_class) + module = importlib.import_module(connection_class.__module__) + lock = (connection_class._lock if module.__name__.endswith('asyncioreactor') + else module._initialization_lock) + entered, release = Event(), Event() + def holder(): + with lock: + entered.set() + assert release.wait(10) + thread = Thread(target=holder) + thread.start() + assert entered.wait(5) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + child_channel = ReactorChannel(connection_class) + os.write(write_fd, b'started') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready, 'new child reactor entered inherited startup lock' + assert os.read(read_fd, 8) == b'started' + finally: + release.set() + thread.join(5) + os.close(read_fd) + os.waitpid(pid, 0) + + +@pytest.mark.parametrize('reactor', ['libev', 'asyncore']) +def test_startup_failure_does_not_acknowledge_concurrent_dispatch(monkeypatch, reactor): + module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + if reactor == 'asyncore': + monkeypatch.setattr(module, '_dispatcher_map', {}) + class Dispatcher: + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + Event().wait(0.001) + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() + starting, fail, second_entered, accepted, delivered = [Event() for _ in range(5)] + admission_attempted = Event() + original_thread = module.Thread + calls = [] + class FailedStart: + def start(self): + starting.set() + assert fail.wait(5) + raise RuntimeError('thread creation failed') + def is_alive(self): + return False + def factory(*args, **kwargs): + calls.append(True) + return FailedStart() if len(calls) == 1 else original_thread(*args, **kwargs) + monkeypatch.setattr(module, 'Thread', factory) + errors, rejected_callbacks = [], [] + def first(): + try: + loop.call_soon(lambda: rejected_callbacks.append(True)) + except RuntimeError as exc: + errors.append(str(exc)) + def second(): + second_entered.set() + loop.call_soon(delivered.set) + accepted.set() + first_thread, second_thread = Thread(target=first), Thread(target=second) + lock_name = '_lock_thread' if reactor == 'libev' else '_state_lock' + admission_lock = getattr(loop, lock_name) + blocked = [] + class ObservedLock: + def __enter__(self): + if get_ident() == second_thread.ident: + acquired = admission_lock.acquire(False) + blocked.append(not acquired) + admission_attempted.set() + if acquired: + return self + admission_lock.acquire() + return self + def __exit__(self, *args): + admission_lock.release() + setattr(loop, lock_name, ObservedLock()) + try: + first_thread.start() + assert starting.wait(5) + second_thread.start() + assert second_entered.wait(5) + assert admission_attempted.wait(5) + assert blocked == [True], 'second admission did not block on provisional startup' + assert not accepted.wait(0.05) + fail.set() + first_thread.join(5) + second_thread.join(5) + assert accepted.is_set() + assert delivered.wait(5) + assert errors == ['thread creation failed'] + assert rejected_callbacks == [] + finally: + fail.set() + first_thread.join(5) + if second_thread.ident is not None: + second_thread.join(5) + loop._cleanup() + + +def test_asyncore_dispatch_accepted_during_runner_failure_is_delivered(monkeypatch): + module = pytest.importorskip('cassandra.io.asyncorereactor') + monkeypatch.setattr(module, '_dispatcher_map', {}) + entered, fail, delivered = Event(), Event(), Event() + class Dispatcher: + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + entered.set() + assert fail.wait(5) + raise RuntimeError('dispatcher failed') + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = module.AsyncoreLoop() + observations = [] + try: + loop.maybe_start() + assert entered.wait(5) + def completion(): + observations.append(get_ident()) + delivered.set() + loop.call_soon(completion) + fail.set() + loop._thread.join(5) + assert delivered.is_set() + assert observations == [loop._thread.ident] + with pytest.raises(RuntimeError, match='shutting down'): + loop.call_soon(lambda: None) + assert not loop._management_callbacks + finally: + fail.set() + loop._cleanup() + + +def test_asyncore_ordinary_connection_recovers_without_reviving_management(monkeypatch): + import socket + module = pytest.importorskip('cassandra.io.asyncorereactor') + monkeypatch.setattr(module, '_dispatcher_map', {}) + entered, fail, finish_failure, recovered = [Event() for _ in range(4)] + class Dispatcher: + failed = False + def validate(self): + pass + def notify_loop(self): + pass + def loop(self, timeout): + if fail.is_set() and not self.failed: + entered.set() + assert finish_failure.wait(5) + self.failed = True + raise OSError('transient poll failure') + if self.failed: + recovered.set() + Event().wait(0.001) + def close(self): + pass + monkeypatch.setattr(module.AsyncoreLoop, '_loop_dispatch_class', Dispatcher) + loop = module.AsyncoreLoop() + monkeypatch.setattr(module, '_global_loop', loop) + # This channel predates service loss and must never become usable again. + channel = None + ours, peer = socket.socketpair() + connection = None + try: + channel = ReactorChannel(module.AsyncoreConnection) + old_runner = loop._thread + fail.set() + assert entered.wait(5) + delivered = Event() + channel.dispatch(delivered.set) + finish_failure.set() + old_runner.join(5) + assert not old_runner.is_alive() + assert delivered.is_set() + with pytest.raises(RuntimeError): + channel.dispatch(lambda: None) + monkeypatch.setattr(module.AsyncoreConnection, '_connect_socket', + lambda self: setattr(self, '_socket', ours)) + monkeypatch.setattr(module.AsyncoreConnection, '_send_options_message', lambda self: None) + connection = module.AsyncoreConnection(connect_timeout=1) + assert recovered.wait(5) + assert loop._thread is not old_runner + assert loop._thread.is_alive() + assert module._dispatcher_map[ours.fileno()] is connection + assert connection.socket is ours + with pytest.raises(RuntimeError): + channel.dispatch(lambda: None) + with pytest.raises(RuntimeError): + ReactorChannel(module.AsyncoreConnection) + assert not loop._management_callbacks + finally: + fail.set() + finish_failure.set() + if channel is not None: + channel.close() + if connection is not None: + connection.close() + loop._cleanup() + ours.close() + peer.close() + + +@pytest.mark.parametrize('reactor', ['asyncio', 'libev', 'asyncore']) +@pytest.mark.parametrize('count', [1, 130]) +def test_reactor_shutdown_delivers_already_accepted_callbacks(monkeypatch, reactor, count): + module = pytest.importorskip('cassandra.io.' + { + 'asyncio': 'asyncioreactor', 'libev': 'libevreactor', + 'asyncore': 'asyncorereactor'}[reactor]) + entered, release, delivered, stopping = [Event() for _ in range(4)] + if reactor == 'asyncore': + monkeypatch.setattr(module, '_dispatcher_map', {}) + if reactor == 'asyncio': + connection = type('IsolatedConnection', (module.AsyncioConnection,), + {'_loop': None, '_loop_thread': None}) + connection.initialize_reactor() + dispatch, runner = connection._management_dispatch, connection._loop_thread + def stop(): + stopping.set() + def cleanup(): + if runner.is_alive(): + connection._loop.call_soon_threadsafe(connection._loop.stop) + runner.join(5) + assert not runner.is_alive() + connection._loop.close() + else: + loop = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor])() + dispatch = loop.call_soon + def stop(): + stopping.set() + loop._cleanup() + def cleanup(): + loop._cleanup() + runner = None + observations = [] + def hold(): + entered.set() + assert release.wait(5) + if reactor == 'asyncio': + # Stop in the current ready-queue turn, before any of the drain + # handles queued while this ordinary loop callback was held. + connection._loop.stop() + stopper = None + try: + if reactor == 'asyncio': + connection._loop.call_soon_threadsafe(hold) + else: + dispatch(hold) + assert entered.wait(5) + def completion(): + observations.append(get_ident()) + if len(observations) == count: + delivered.set() + for _ in range(count): + dispatch(completion) + if reactor == 'asyncio': + stop() + else: + runner = loop._thread + stopper = Thread(target=stop) + stopper.start() + assert stopping.wait(5) + # Cleanup commits shutdown under the admission lock before join. + for _ in range(500): + with (loop._lock if reactor == 'libev' else loop._state_lock): + if loop._shutdown: + break + Event().wait(0.001) + assert loop._shutdown + release.set() + assert delivered.wait(5) + runner.join(5) + assert not runner.is_alive() + if stopper is not None: + stopper.join(5) + assert not stopper.is_alive() + assert observations == [runner.ident] * count + with pytest.raises(RuntimeError): + dispatch(lambda: None) + finally: + release.set() + if stopper is not None: + stopper.join(5) + cleanup() + + +@pytest.mark.parametrize('reactor', ['libev', 'asyncore']) +def test_fork_reinitialization_closes_inherited_descriptors(monkeypatch, reactor): + import os + import select + import signal + import socket + from types import SimpleNamespace + if not hasattr(os, 'fork'): + pytest.skip('fork unavailable') + module = pytest.importorskip('cassandra.io.' + reactor + 'reactor') + connection = getattr(module, {'libev': 'LibevConnection', + 'asyncore': 'AsyncoreConnection'}[reactor]) + connection.initialize_reactor() + inherited = module._global_loop + ours, peer = socket.socketpair() + if reactor == 'libev': + class Connection: + _socket = ours + _read_watcher = _write_watcher = None + conn = Connection() + inherited._live_conns.add(conn) + else: + module._dispatcher_map[ours.fileno()] = SimpleNamespace(socket=ours) + original_factory = getattr(module, {'libev': 'LibevLoop', 'asyncore': 'AsyncoreLoop'}[reactor]) + def checked_factory(): + assert ours.fileno() == -1 + if reactor == 'libev': + assert inherited._loop is None + assert inherited._notifier is None + else: + assert inherited._loop_dispatcher.write_fd is None + return original_factory() + monkeypatch.setattr(module, original_factory.__name__, checked_factory) + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + connection.initialize_reactor() + os.write(write_fd, b'closed') + finally: + os._exit(0) + os.close(write_fd) + try: + ready, _, _ = select.select([read_fd], [], [], 5) + if not ready: + os.kill(pid, signal.SIGKILL) + assert ready + assert os.read(read_fd, 6) == b'closed' + # The child closed only its descriptor copies. + ours.send(b'x') + assert peer.recv(1) == b'x' + finally: + if reactor == 'libev': + inherited._live_conns.discard(conn) + else: + module._dispatcher_map.pop(ours.fileno(), None) + ours.close() + peer.close() + os.close(read_fd) + os.waitpid(pid, 0) + + +def test_channel_startup_failure_unwinds_all_management_threads(): + from cassandra._lifecycle import LifecycleCoordinator + class FailingConnection: + @classmethod + def initialize_reactor(cls): + pass + @classmethod + def _management_in_context(cls): + return False + @classmethod + def _management_dispatch(cls, callback): + raise RuntimeError('bridge admission failed') + cluster = SimpleNamespace(connection_class=FailingConnection, is_shutdown=False) + # SimpleNamespace is not weak-referenceable; use an ordinary test lifetime. + class Lifetime: + pass + lifetime = Lifetime() + lifetime.connection_class = FailingConnection + lifetime.is_shutdown = False + coordinator = LifecycleCoordinator(lifetime) + import threading + before = {thread.ident for thread in threading.enumerate() + if thread.name.startswith('cassandra-management-')} + with pytest.raises(RuntimeError, match='bridge admission failed'): + coordinator._ensure_runtime() + assert coordinator._runtime is None + after = {thread.ident for thread in threading.enumerate() + if thread.name.startswith('cassandra-management-')} + assert after == before diff --git a/tests/unit/test_management_runtime.py b/tests/unit/test_management_runtime.py index 6853bd1989..4a16b9881c 100644 --- a/tests/unit/test_management_runtime.py +++ b/tests/unit/test_management_runtime.py @@ -1,7 +1,7 @@ from collections import deque from concurrent.futures import ( Future, ThreadPoolExecutor, as_completed, wait, TimeoutError) -from threading import Barrier, Event, Lock, Thread, get_ident +from threading import Barrier, Event, Lock, Thread, current_thread, get_ident import pytest @@ -502,6 +502,10 @@ def test_completed_executor_future_does_not_dispose_on_registering_owner( manager, dispatch = runtime resource = object() disposal_threads = [] + disposal_contexts = [] + def dispose(value): + disposal_threads.append(current_thread()) + disposal_contexts.append(execution_context()) with ThreadPoolExecutor(1) as executor: class ClosingExecutor: def submit(self, function): @@ -514,13 +518,15 @@ def submit(self, function): return physical future = manager.submit(lambda record: manager.work( record, ClosingExecutor(), lambda: resource, - dispose=lambda value: disposal_threads.append(get_ident()))) + dispose=dispose)) with pytest.raises(ManagementClosed): future.result(5) manager.join(5) assert len(disposal_threads) == 1 - assert disposal_threads[0] not in { - thread.ident for thread in manager._threads} + # Native thread IDs may be reused after close retires an owner. Keep the + # actual Thread objects to distinguish that owner from a later worker. + assert all(disposal_threads[0] is not thread for thread in manager._threads) + assert disposal_contexts == ['worker'] assert manager.counts()['ordinary'] == 0 @@ -829,6 +835,54 @@ def result_queued(): assert completion.ident not in {thread.ident for thread in manager._threads} +@pytest.mark.parametrize('health_notification', [False, True]) +def test_detached_coordinator_does_not_keep_runtime_threads_alive( + health_notification): + import gc + import weakref + from functools import partial + from cassandra._lifecycle import LifecycleCoordinator + coordinator = LifecycleCoordinator() + coordinator.submit(lambda: None) + manager = coordinator._runtime + reference = weakref.ref(coordinator) + release = Event() + try: + if health_notification: + entered, delivered = Event(), Event() + def hold(): + entered.set() + assert release.wait(5) + def completed(callback): + callback() + delivered.set() + manager.limit = 1 + manager.notify(hold, wait=True) + assert entered.wait(5) + incarnation = object() + manager.register_health(incarnation) + manager.observe_negative( + incarnation, None, partial(completed, coordinator.drain)) + release.set() + assert delivered.wait(5) + # Lifecycle idle is signalled inside its callback, before the native + # notification reservation finishes. A FIFO fence passes the earlier + # C admission and L callback frames before testing idle retention. + manager.notify(lambda: None, wait=True).result(5) + with manager._condition: + assert manager._condition.wait_for( + lambda: not manager._records, 5) + del coordinator + gc.collect() + assert reference() is None + manager.join(5) + assert all(not thread.is_alive() for thread in manager._threads) + finally: + release.set() + manager.close() + manager.join(5) + + def test_synchronous_admission_waits_without_allocating_pending_records( runtime, monkeypatch): manager, dispatch = runtime