Skip to content
Open
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 78 additions & 8 deletions cassandra/io/asyncioreactor.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import threading
from collections import deque

from cassandra.connection import Connection, ConnectionShutdown
import sys
Expand All @@ -7,7 +8,7 @@
import os
import socket
import ssl
from threading import Lock, Thread
from threading import Event, Lock, Thread, current_thread


log = logging.getLogger(__name__)
Expand Down Expand Up @@ -133,6 +134,7 @@ class AsyncioConnection(Connection):

_lock = Lock()
_loop_thread = None
_management_state = None

_write_queue = None
_write_queue_lock = None
Expand Down Expand Up @@ -258,13 +260,72 @@ def initialize_reactor(cls):
cls._pid = os.getpid()
if cls._loop is None:
assert cls._loop_thread is None
cls._loop = asyncio.new_event_loop()
# daemonize so the loop will be shut down on interpreter
# shutdown
cls._loop_thread = Thread(
target=cls._loop.run_forever, daemon=True, name="asyncio_thread"
)
cls._loop_thread.start()
loop = asyncio.new_event_loop()
pending, management_lock = deque(), Lock()
accepting = [True]

def drain(final=False):
count = 0
while final or count < 64:
with management_lock:
if not pending:
return
callback = pending.popleft()
count += 1
try:
callback()
except BaseException:
log.exception("Error in management reactor dispatch")
with management_lock:
if pending and accepting[0]:
loop.call_soon(drain)

def run():
try:
loop.run_forever()
finally:
with management_lock:
accepting[0] = False
drain(final=True)

thread = Thread(target=run, daemon=True,
name="asyncio_thread")
ready = Event()
try:
loop.call_soon(ready.set)
thread.start()
if not ready.wait(5):
raise RuntimeError("Asyncio reactor startup did not complete")
except BaseException:
if thread.is_alive():
loop.call_soon_threadsafe(loop.stop)
thread.join(5)
if not thread.is_alive():
loop.close()
raise
cls._loop, cls._loop_thread = loop, thread
cls._management_state = (pending, management_lock, accepting, drain)

@classmethod
def _management_dispatch(cls, callback):
loop = cls._loop
if cls._management_state is None:
raise RuntimeError("Asyncio reactor is unavailable")
pending, lock, accepting, drain = cls._management_state
with lock:
if (not accepting[0] or loop is None or loop.is_closed() or
not cls._loop_thread.is_alive()):
raise RuntimeError("Asyncio reactor is unavailable")
pending.append(callback)
try:
loop.call_soon_threadsafe(drain)
except BaseException:
pending.remove(callback)
raise

@classmethod
def _management_in_context(cls):
return current_thread() is cls._loop_thread

@classmethod
def create_timer(cls, timeout, callback):
Expand Down Expand Up @@ -381,3 +442,12 @@ async def handle_read(self):
log.debug("Connection %s closed by server", self)
self.close()
return


def _reset_management_startup_lock_after_fork():
# A parent thread may own this lock when another thread forks.
AsyncioConnection._lock = Lock()


if hasattr(os, 'register_at_fork'):
os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork)
157 changes: 122 additions & 35 deletions cassandra/io/asyncorereactor.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import os
import socket
import sys
from threading import Lock, Thread, Event
from threading import Lock, RLock, Thread, Event, current_thread
import time
import sys
import ssl
Expand Down Expand Up @@ -137,6 +137,14 @@ def notify_loop(self):
self._notified = True
os.write(self.write_fd, b'x')

def close(self):
try:
asyncore.dispatcher.close(self)
finally:
if self.write_fd is not None:
os.close(self.write_fd)
self.write_fd = None


class _AsyncoreUDPDispatcher(_AsyncoreDispatcher):
"""
Expand Down Expand Up @@ -210,8 +218,12 @@ class AsyncoreLoop(object):
def __init__(self):
self._pid = os.getpid()
self._loop_lock = Lock()
self._state_lock = RLock()
self._management_callbacks = deque()
self._management_failed = False
self._started = False
self._shutdown = False
self._cleanup_started = False

self._thread = None

Expand All @@ -228,36 +240,70 @@ def __init__(self):
self._loop_dispatcher = dispatcher

def maybe_start(self):
should_start = False
did_acquire = False
try:
did_acquire = self._loop_lock.acquire(False)
if did_acquire and not self._started:
self._started = True
should_start = True
finally:
if did_acquire:
self._loop_lock.release()
with self._state_lock:
if self._shutdown or self._started:
return
self._started = True
self._thread = Thread(target=self._run_loop,
name="asyncore_cassandra_driver_event_loop", daemon=True)
try:
self._thread.start()
except BaseException:
if not self._thread.is_alive():
self._started = False
self._thread = None
raise

def call_soon(self, callback):
with self._state_lock:
if self._shutdown or self._management_failed:
raise RuntimeError("Asyncore reactor is shutting down")
self._management_callbacks.append(callback)
try:
self.maybe_start()
self.wake_loop()
except BaseException:
try:
self._management_callbacks.remove(callback)
except ValueError:
return
raise

if should_start:
self._thread = Thread(target=self._run_loop, name="asyncore_cassandra_driver_event_loop")
self._thread.daemon = True
self._thread.start()
def _dispatch_management(self):
for _ in range(64):
with self._state_lock:
if not self._management_callbacks:
return
callback = self._management_callbacks.popleft()
try:
callback()
except BaseException:
log.exception("Error in management reactor dispatch")
self.wake_loop()

def wake_loop(self):
self._loop_dispatcher.notify_loop()

def _run_loop(self):
log.debug("Starting asyncore event loop")
with self._loop_lock:
while not self._shutdown:
try:
try:
while not self._shutdown:
self._loop_dispatcher.loop(self.timer_resolution)
self._timers.service_timeouts()
except Exception as exc:
self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc)
break
self._started = False
self._dispatch_management()
except BaseException as exc:
self._maybe_log_debug("Asyncore event loop stopped unexpectedly", exc_info=exc)
finally:
# Management service loss is permanent for existing channels.
# Ordinary I/O can restart after this runner drains accepted
# completions and relinquishes ownership.
with self._state_lock:
self._management_failed = True
while self._management_callbacks:
self._dispatch_management()
with self._state_lock:
self._started = False

self._maybe_log_debug("Asyncore event loop ended")

Expand All @@ -281,13 +327,17 @@ def add_timer(self, timer):
def _cleanup(self):
global _dispatcher_map

self._shutdown = True
if not self._thread:
return
with self._state_lock:
if self._cleanup_started:
return
self._cleanup_started = True
self._shutdown = True
self.wake_loop()

log.debug("Waiting for event loop thread to join...")
self._thread.join(timeout=1.0)
if self._thread.is_alive():
if self._thread is not None and self._thread is not current_thread():
self._thread.join(timeout=1.0)
if self._thread is not None and self._thread.is_alive():
log.warning(
"Event loop thread could not be joined, so shutdown may not be clean. "
"Please call Cluster.shutdown() to avoid this.")
Expand All @@ -306,6 +356,7 @@ def _cleanup(self):


_global_loop = None
_initialization_lock = Lock()
atexit.register(partial(_cleanup, _global_loop))


Expand All @@ -320,15 +371,42 @@ class AsyncoreConnection(Connection, asyncore.dispatcher):

@classmethod
def initialize_reactor(cls):
global _global_loop
if not _global_loop:
_global_loop = AsyncoreLoop()
else:
current_pid = os.getpid()
if _global_loop._pid != current_pid:
log.debug("Detected fork, clearing and reinitializing reactor state")
cls.handle_fork()
_global_loop = AsyncoreLoop()
global _global_loop, _dispatcher_map
with _initialization_lock:
if _global_loop is not None and _global_loop._pid != os.getpid():
# Close descriptors directly: Connection.close() can acquire
# parent-owned locks or schedule timers on a vanished runner.
for dispatcher in tuple(_dispatcher_map.values()):
try:
if dispatcher is _global_loop._loop_dispatcher:
dispatcher.close()
else:
dispatcher.socket.close()
except OSError:
pass
_dispatcher_map = {}
_global_loop = None
if _global_loop is None:
candidate = AsyncoreLoop()
ready = Event()
try:
candidate.call_soon(ready.set)
if not ready.wait(5):
raise RuntimeError("Asyncore reactor startup did not complete")
except BaseException:
candidate._cleanup()
raise
_global_loop = candidate

@classmethod
def _management_dispatch(cls, callback):
if _global_loop is None:
raise RuntimeError("Asyncore reactor is unavailable")
_global_loop.call_soon(callback)

@classmethod
def _management_in_context(cls):
return _global_loop is not None and current_thread() is _global_loop._thread

@classmethod
def handle_fork(cls):
Expand Down Expand Up @@ -476,3 +554,12 @@ def writable(self):

def readable(self):
return self._readable or ((self.is_control_connection or self._continuous_paging_sessions) and not (self.is_defunct or self.is_closed))


def _reset_management_startup_lock_after_fork():
global _initialization_lock
_initialization_lock = Lock()


if hasattr(os, 'register_at_fork'):
os.register_at_fork(after_in_child=_reset_management_startup_lock_after_fork)
Loading
Loading