Skip to content

Commit 9b000a4

Browse files
committed
session: serialize pool recovery through lifecycle
Queue recovery timers and pool-creation results through the lifecycle FIFO so they cannot reconcile pools or advance retry schedules during host notifications. Discard queued and late recovery when the coordinator closes, retaining the failed-pool ownership checks at delivery. Add regressions for timer and completion ordering, coordinator close, initial admission and stale queued recovery. Refs #382.
1 parent b4aa97b commit 9b000a4

2 files changed

Lines changed: 139 additions & 3 deletions

File tree

‎cassandra/cluster.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4734,8 +4734,12 @@ def _recover_pool(self, failed_pool):
47344734
these retries without disturbing another session's healthy pool.
47354735
Failed creation, including authentication failure, remains retryable
47364736
here while this session still owns the retired pool.
4737+
4738+
Timers and completion callbacks join the lifecycle FIFO before
4739+
reconciling pools or advancing the retry schedule.
47374740
"""
47384741
session_ref = weakref.ref(self)
4742+
lifecycle = self.cluster._lifecycle
47394743
host = failed_pool.host
47404744
schedule = iter(self.cluster.reconnection_policy.new_schedule())
47414745

@@ -4771,7 +4775,8 @@ def retry(completed=None):
47714775
log.warning(
47724776
"Session pool recovery schedule exhausted for %s", host)
47734777
return
4774-
session.cluster.scheduler.schedule(delay, recover)
4778+
session.cluster.scheduler.schedule(
4779+
delay, partial(lifecycle.submit, recover))
47754780

47764781
def recover():
47774782
"""Reconcile this host without adopting a newer pool lifecycle."""
@@ -4788,11 +4793,11 @@ def recover():
47884793
if futures:
47894794
# Reconciliation of this single host has one completion.
47904795
for future in futures:
4791-
future.add_done_callback(retry)
4796+
future.add_done_callback(partial(lifecycle.submit, retry))
47924797
else:
47934798
retry()
47944799

4795-
retry()
4800+
lifecycle.submit(retry)
47964801

47974802
def _shutdown_detached_pool(self, pool, on_shutdown_error=None):
47984803
"""Close an unindexed pool even when asynchronous cleanup is rejected.

‎tests/unit/test_host_connection_pool.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,137 @@ def test_discounted_pool_recovery_retries_cancelled_work(
707707
assert not scheduled
708708

709709

710+
@pytest.mark.parametrize('worker', [False, True])
711+
def test_discounted_recovery_waits_for_lifecycle_notification(
712+
discounted_pool_recovery, worker):
713+
"""A timer cannot run distance callbacks during a host notification."""
714+
host, session, failed, healthy, scheduled, work = discounted_pool_recovery
715+
recover = scheduled.pop(0)[1]
716+
distance = session.cluster.profile_manager.distance
717+
distance.reset_mock()
718+
during_notification = []
719+
720+
def notify():
721+
if worker:
722+
timer = Thread(target=recover)
723+
timer.start()
724+
timer.join(5)
725+
during_notification.append(timer.is_alive())
726+
else:
727+
recover()
728+
during_notification.append(False)
729+
during_notification.extend([distance.call_count, len(work)])
730+
731+
session.cluster._lifecycle.submit(notify)
732+
assert during_notification == [False, 0, 0]
733+
distance.assert_called_once_with(host)
734+
assert len(work) == 1
735+
_finish_pool_recovery_work(work)
736+
assert session._pools[host] is not failed
737+
assert not healthy.is_shutdown
738+
assert not scheduled
739+
740+
741+
@pytest.mark.parametrize('completion', ['failure', 'exception', 'cancelled'])
742+
def test_discounted_recovery_completion_waits_for_lifecycle_notification(
743+
discounted_pool_recovery, completion):
744+
"""Worker results consume the retry schedule after active notifications."""
745+
host, session, failed, healthy, scheduled, work = discounted_pool_recovery
746+
scheduled.pop(0)[1]()
747+
future = work.pop(0)[3]
748+
during_notification = []
749+
750+
def notify():
751+
def complete():
752+
if completion == 'exception':
753+
future.set_exception(ConnectionException('creation failed'))
754+
elif completion == 'cancelled':
755+
future.cancel()
756+
else:
757+
future.set_result(False)
758+
759+
worker = Thread(target=complete)
760+
worker.start()
761+
worker.join(5)
762+
during_notification.extend([worker.is_alive(), len(scheduled)])
763+
764+
session.cluster._lifecycle.submit(notify)
765+
assert during_notification == [False, 0]
766+
assert len(scheduled) == 1
767+
assert scheduled[0][0] == 2
768+
scheduled.pop(0)[1]()
769+
_finish_pool_recovery_work(work)
770+
assert session._pools[host] is not failed
771+
assert not healthy.is_shutdown
772+
assert not scheduled
773+
774+
775+
@pytest.mark.parametrize('phase', ['timer', 'completion'])
776+
@pytest.mark.parametrize('close', ['queued', 'late'])
777+
def test_discounted_recovery_abandons_queued_work_on_lifecycle_close(
778+
discounted_pool_recovery, phase, close):
779+
"""Closing the coordinator discards queued recovery and retry results."""
780+
host, session, failed, healthy, scheduled, work = discounted_pool_recovery
781+
recover = scheduled.pop(0)[1]
782+
if phase == 'completion':
783+
recover()
784+
future = work.pop(0)[3]
785+
786+
def notify():
787+
if close == 'late':
788+
session.cluster._lifecycle.close()
789+
if phase == 'timer':
790+
recover()
791+
else:
792+
future.set_result(False)
793+
session.cluster._lifecycle.close()
794+
795+
session.cluster._lifecycle.submit(notify)
796+
assert not work
797+
assert not scheduled
798+
assert session._pools[host] is failed
799+
800+
801+
def test_discounted_recovery_initial_retry_joins_lifecycle_queue(
802+
discounted_pool_recovery):
803+
"""Initial recovery admission uses the same ordering as later results."""
804+
host, session, failed, healthy, scheduled, work = discounted_pool_recovery
805+
scheduled.clear()
806+
during_notification = []
807+
808+
def notify():
809+
session._recover_pool(failed)
810+
during_notification.append(len(scheduled))
811+
812+
session.cluster._lifecycle.submit(notify)
813+
assert during_notification == [0]
814+
assert len(scheduled) == 1
815+
assert scheduled[0][0] == 1
816+
817+
818+
@pytest.mark.parametrize('supersession', ['pool_replaced', 'pool_removed'])
819+
def test_discounted_recovery_rechecks_ownership_after_queueing(
820+
discounted_pool_recovery, supersession):
821+
"""A queued timer must still own its failed pool when delivery starts."""
822+
host, session, failed, healthy, scheduled, work = discounted_pool_recovery
823+
recover = scheduled.pop(0)[1]
824+
distance = session.cluster.profile_manager.distance
825+
distance.reset_mock()
826+
827+
def notify():
828+
recover()
829+
if supersession == 'pool_replaced':
830+
session._pools[host] = healthy
831+
else:
832+
session._pools.pop(host)
833+
834+
session.cluster._lifecycle.submit(notify)
835+
distance.assert_not_called()
836+
assert not work
837+
assert not scheduled
838+
assert not healthy.is_shutdown
839+
840+
710841
def test_queued_replacement_cannot_remove_new_socket(socket_pool):
711842
host, session, pool, original, replacement = socket_pool
712843
pool.borrow_connection(timeout=0)

0 commit comments

Comments
 (0)