Skip to content

Commit e190e35

Browse files
committed
reactors: dispatch management completion on shared I/O loops
1 parent 2778602 commit e190e35

6 files changed

Lines changed: 490 additions & 55 deletions

File tree

‎cassandra/io/asyncioreactor.py‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import os
88
import socket
99
import ssl
10-
from threading import Lock, Thread
10+
from threading import Event, Lock, Thread, current_thread
1111

1212

1313
log = logging.getLogger(__name__)
@@ -258,13 +258,34 @@ def initialize_reactor(cls):
258258
cls._pid = os.getpid()
259259
if cls._loop is None:
260260
assert cls._loop_thread is None
261-
cls._loop = asyncio.new_event_loop()
262-
# daemonize so the loop will be shut down on interpreter
263-
# shutdown
264-
cls._loop_thread = Thread(
265-
target=cls._loop.run_forever, daemon=True, name="asyncio_thread"
266-
)
267-
cls._loop_thread.start()
261+
loop = asyncio.new_event_loop()
262+
thread = Thread(target=loop.run_forever, daemon=True,
263+
name="asyncio_thread")
264+
ready = Event()
265+
try:
266+
loop.call_soon(ready.set)
267+
thread.start()
268+
if not ready.wait(5):
269+
raise RuntimeError("Asyncio reactor startup did not complete")
270+
except BaseException:
271+
if thread.is_alive():
272+
loop.call_soon_threadsafe(loop.stop)
273+
thread.join(5)
274+
if not thread.is_alive():
275+
loop.close()
276+
raise
277+
cls._loop, cls._loop_thread = loop, thread
278+
279+
@classmethod
280+
def _management_dispatch(cls, callback):
281+
loop = cls._loop
282+
if loop is None or loop.is_closed() or not cls._loop_thread.is_alive():
283+
raise RuntimeError("Asyncio reactor is unavailable")
284+
loop.call_soon_threadsafe(callback)
285+
286+
@classmethod
287+
def _management_in_context(cls):
288+
return current_thread() is cls._loop_thread
268289

269290
@classmethod
270291
def create_timer(cls, timeout, callback):
@@ -381,3 +402,12 @@ async def handle_read(self):
381402
log.debug("Connection %s closed by server", self)
382403
self.close()
383404
return
405+
406+
407+
def _reset_management_startup_lock_after_fork():
408+
# A parent thread may own this lock when another thread forks.
409+
AsyncioConnection._lock = Lock()
410+
411+
412+
if hasattr(os, 'register_at_fork'):
413+
os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork)

‎cassandra/io/asyncorereactor.py‎

Lines changed: 81 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import os
1919
import socket
2020
import sys
21-
from threading import Lock, Thread, Event
21+
from threading import Lock, Thread, Event, current_thread
2222
import time
2323
import sys
2424
import ssl
@@ -210,6 +210,9 @@ class AsyncoreLoop(object):
210210
def __init__(self):
211211
self._pid = os.getpid()
212212
self._loop_lock = Lock()
213+
self._state_lock = Lock()
214+
self._management_lock = Lock()
215+
self._management_callbacks = deque()
213216
self._started = False
214217
self._shutdown = False
215218

@@ -228,21 +231,38 @@ def __init__(self):
228231
self._loop_dispatcher = dispatcher
229232

230233
def maybe_start(self):
231-
should_start = False
232-
did_acquire = False
233-
try:
234-
did_acquire = self._loop_lock.acquire(False)
235-
if did_acquire and not self._started:
236-
self._started = True
237-
should_start = True
238-
finally:
239-
if did_acquire:
240-
self._loop_lock.release()
241-
242-
if should_start:
243-
self._thread = Thread(target=self._run_loop, name="asyncore_cassandra_driver_event_loop")
244-
self._thread.daemon = True
245-
self._thread.start()
234+
with self._state_lock:
235+
if self._shutdown or self._started:
236+
return
237+
self._started = True
238+
self._thread = Thread(target=self._run_loop,
239+
name="asyncore_cassandra_driver_event_loop", daemon=True)
240+
try:
241+
self._thread.start()
242+
except BaseException:
243+
self._started = False
244+
self._thread = None
245+
raise
246+
247+
def call_soon(self, callback):
248+
with self._management_lock:
249+
if self._shutdown:
250+
raise RuntimeError("Asyncore reactor is shutting down")
251+
self._management_callbacks.append(callback)
252+
self.maybe_start()
253+
self.wake_loop()
254+
255+
def _dispatch_management(self):
256+
for _ in range(64):
257+
with self._management_lock:
258+
if not self._management_callbacks:
259+
return
260+
callback = self._management_callbacks.popleft()
261+
try:
262+
callback()
263+
except BaseException:
264+
log.exception("Error in management reactor dispatch")
265+
self.wake_loop()
246266

247267
def wake_loop(self):
248268
self._loop_dispatcher.notify_loop()
@@ -254,10 +274,12 @@ def _run_loop(self):
254274
try:
255275
self._loop_dispatcher.loop(self.timer_resolution)
256276
self._timers.service_timeouts()
277+
self._dispatch_management()
257278
except Exception as exc:
258279
self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc)
259280
break
260-
self._started = False
281+
with self._state_lock:
282+
self._started = False
261283

262284
self._maybe_log_debug("Asyncore event loop ended")
263285

@@ -281,13 +303,14 @@ def add_timer(self, timer):
281303
def _cleanup(self):
282304
global _dispatcher_map
283305

284-
self._shutdown = True
285-
if not self._thread:
286-
return
306+
with self._state_lock:
307+
self._shutdown = True
308+
self.wake_loop()
287309

288310
log.debug("Waiting for event loop thread to join...")
289-
self._thread.join(timeout=1.0)
290-
if self._thread.is_alive():
311+
if self._thread is not None and self._thread is not current_thread():
312+
self._thread.join(timeout=1.0)
313+
if self._thread is not None and self._thread.is_alive():
291314
log.warning(
292315
"Event loop thread could not be joined, so shutdown may not be clean. "
293316
"Please call Cluster.shutdown() to avoid this.")
@@ -306,6 +329,7 @@ def _cleanup(self):
306329

307330

308331
_global_loop = None
332+
_initialization_lock = Lock()
309333
atexit.register(partial(_cleanup, _global_loop))
310334

311335

@@ -320,15 +344,32 @@ class AsyncoreConnection(Connection, asyncore.dispatcher):
320344

321345
@classmethod
322346
def initialize_reactor(cls):
323-
global _global_loop
324-
if not _global_loop:
325-
_global_loop = AsyncoreLoop()
326-
else:
327-
current_pid = os.getpid()
328-
if _global_loop._pid != current_pid:
329-
log.debug("Detected fork, clearing and reinitializing reactor state")
330-
cls.handle_fork()
331-
_global_loop = AsyncoreLoop()
347+
global _global_loop, _dispatcher_map
348+
with _initialization_lock:
349+
if _global_loop is not None and _global_loop._pid != os.getpid():
350+
_dispatcher_map = {}
351+
_global_loop = None
352+
if _global_loop is None:
353+
candidate = AsyncoreLoop()
354+
ready = Event()
355+
try:
356+
candidate.call_soon(ready.set)
357+
if not ready.wait(5):
358+
raise RuntimeError("Asyncore reactor startup did not complete")
359+
except BaseException:
360+
candidate._cleanup()
361+
raise
362+
_global_loop = candidate
363+
364+
@classmethod
365+
def _management_dispatch(cls, callback):
366+
if _global_loop is None:
367+
raise RuntimeError("Asyncore reactor is unavailable")
368+
_global_loop.call_soon(callback)
369+
370+
@classmethod
371+
def _management_in_context(cls):
372+
return _global_loop is not None and current_thread() is _global_loop._thread
332373

333374
@classmethod
334375
def handle_fork(cls):
@@ -476,3 +517,12 @@ def writable(self):
476517

477518
def readable(self):
478519
return self._readable or ((self.is_control_connection or self._continuous_paging_sessions) and not (self.is_defunct or self.is_closed))
520+
521+
522+
def _reset_management_startup_lock_after_fork():
523+
global _initialization_lock
524+
_initialization_lock = Lock()
525+
526+
527+
if hasattr(os, 'register_at_fork'):
528+
os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork)

0 commit comments

Comments
 (0)