Skip to content

Commit 4747f2b

Browse files
committed
reactors: preserve runner ownership during teardown and recovery
Drain libev completions after exit only for the shutdown owner, so a retired idle runner cannot claim a replacement runner's work. Stop a management keeper armed concurrently with cleanup or final borrower release. Separate permanent asyncore management failure from ordinary I/O shutdown. Drain accepted completions before retiring the failed runner, reject stale management channels, and allow ordinary connections to restart I/O. Gate tests at the actual admission, idle-exit and timer-arm boundaries. Exercise asyncio final draining before queued normal drains can run, and always stop the chaining test's application loop when management join fails.
1 parent 8008e60 commit 4747f2b

3 files changed

Lines changed: 277 additions & 23 deletions

File tree

‎cassandra/io/asyncorereactor.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,8 @@ def __init__(self):
219219
self._pid = os.getpid()
220220
self._loop_lock = Lock()
221221
self._state_lock = RLock()
222-
self._management_lock = self._state_lock
223222
self._management_callbacks = deque()
223+
self._management_failed = False
224224
self._started = False
225225
self._shutdown = False
226226
self._cleanup_started = False
@@ -255,8 +255,8 @@ def maybe_start(self):
255255
raise
256256

257257
def call_soon(self, callback):
258-
with self._management_lock:
259-
if self._shutdown:
258+
with self._state_lock:
259+
if self._shutdown or self._management_failed:
260260
raise RuntimeError("Asyncore reactor is shutting down")
261261
self._management_callbacks.append(callback)
262262
try:
@@ -271,7 +271,7 @@ def call_soon(self, callback):
271271

272272
def _dispatch_management(self):
273273
for _ in range(64):
274-
with self._management_lock:
274+
with self._state_lock:
275275
if not self._management_callbacks:
276276
return
277277
callback = self._management_callbacks.popleft()
@@ -295,11 +295,11 @@ def _run_loop(self):
295295
except BaseException as exc:
296296
self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc)
297297
finally:
298-
# Commit service loss atomically with admission. Accepted
299-
# callbacks still run on this runner before it relinquishes
300-
# ownership; later submissions receive an explicit rejection.
298+
# Management service loss is permanent for existing channels.
299+
# Ordinary I/O can restart after this runner drains accepted
300+
# completions and relinquishes ownership.
301301
with self._state_lock:
302-
self._shutdown = True
302+
self._management_failed = True
303303
while self._management_callbacks:
304304
self._dispatch_management()
305305
with self._state_lock:

‎cassandra/io/libevreactor.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ def _run_loop(self):
137137
else:
138138
# all Connections have been closed, no active watchers
139139
log.debug("All Connections currently closed, event loop ended")
140+
shutdown = self._shutdown
140141
self._started = False
141142
# A connection may have been closed after the last
142143
# _loop_will_run() pass, which is where closed
@@ -151,10 +152,11 @@ def _run_loop(self):
151152
self._close_conns(to_stop)
152153
break
153154
self._drain_deferred()
154-
# Shutdown rejects new admissions; finish the accepted queue on this
155-
# reactor thread even when cleanup has stopped the prepare watcher.
156-
while self._management_callbacks:
157-
self._dispatch_management()
155+
# Only the shutdown owner may drain after publishing stopped state.
156+
# Ordinary idle exit allows a replacement runner to claim new work.
157+
if shutdown:
158+
while self._management_callbacks:
159+
self._dispatch_management()
158160

159161
def _cleanup(self):
160162
with self._lock_thread:
@@ -257,6 +259,9 @@ def call_soon(self, callback):
257259
def _keep_management_alive(self):
258260
if self._management_users and not self._shutdown:
259261
self._management_keeper.start(3600)
262+
# Cleanup or release can stop the timer while start() is in flight.
263+
if self._shutdown or not self._management_users:
264+
self._management_keeper.stop()
260265

261266
def borrow_management(self):
262267
token = object()
@@ -266,7 +271,7 @@ def acquire():
266271
if abandoned.is_set():
267272
return
268273
self._management_users.add(token)
269-
self._management_keeper.start(3600)
274+
self._keep_management_alive()
270275
if abandoned.is_set():
271276
release(lambda: None)
272277
else:
@@ -281,7 +286,8 @@ def detach():
281286
if current_thread() is self._thread:
282287
detach()
283288
elif self._shutdown or self._thread is None or not self._thread.is_alive():
284-
# Catastrophic service loss: no native runner can touch watchers.
289+
# Shutdown closes admission before the runner finishes draining.
290+
# Release remains idempotent while that runner unwinds.
285291
with self._lock:
286292
detach()
287293
else:

0 commit comments

Comments
 (0)