From ecf9f76f75dfe69e065bbb838cb81afb7886b033 Mon Sep 17 00:00:00 2001 From: mparram Date: Mon, 21 Sep 2026 14:13:11 +0200 Subject: [PATCH 1/5] feat(nanokvm-usb): embedded RFB/VNC with HID and optional TCP bind Co-authored-by: Cursor --- .../jumpstarter-driver-nanokvm-usb/README.md | 77 +- .../examples/exporter.yaml | 4 + .../__init__.py | 6 +- .../jumpstarter_driver_nanokvm_usb/client.py | 66 +- .../jumpstarter_driver_nanokvm_usb/device.py | 136 ++-- .../jumpstarter_driver_nanokvm_usb/driver.py | 117 +++- .../driver_test.py | 229 +++++- .../frame_pump.py | 110 +++ .../protocol.py | 4 +- .../v4l2_ctl_mjpeg.py | 9 +- .../v4l2_mjpeg.py | 4 +- .../vnc_server.py | 659 ++++++++++++++++++ .../pyproject.toml | 1 + python/uv.lock | 2 + 14 files changed, 1331 insertions(+), 93 deletions(-) create mode 100644 python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/frame_pump.py create mode 100644 python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/README.md b/python/packages/jumpstarter-driver-nanokvm-usb/README.md index 5acb84948..cbc84f1fd 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/README.md +++ b/python/packages/jumpstarter-driver-nanokvm-usb/README.md @@ -16,7 +16,8 @@ hardware through: - **Video capture**: Snapshots and live JPEG frame streams from the UVC device - **Keyboard control**: Paste text and press keys via serial HID - **Mouse control**: Absolute and relative movement, clicks, and scrolling -- **Composite driver**: Access video and HID through a unified `NanoKVMUSB` interface +- **VNC**: Embedded RFB server (view the captured HDMI and control HID from noVNC/TigerVNC) +- **Composite driver**: Access video, HID, and VNC through a unified `NanoKVMUSB` interface ## Installation @@ -42,6 +43,10 @@ export: video_fps: 30 screen_width: 1920 screen_height: 1080 + vnc_enabled: true + # vnc_password: "secret" + # vnc_tcp_port: 5900 + # vnc_tcp_bind: "0.0.0.0" # LAN; use vnc_password ``` ### Config parameters @@ -56,15 +61,23 @@ export: | video_fps | Capture rate for `stream()` | int | no | 30 | | screen_width | Target screen width for relative mouse moves | int | no | 1920 | | screen_height | Target screen height for relative mouse moves | int | no | 1080 | +| vnc_enabled | Start an embedded RFB server (Unix socket child `vnc`) | bool | no | true | +| vnc_password | VNC password (VncAuth). Empty/None = no authentication | str | no | | +| vnc_tcp_port | Also bind RFB TCP on the exporter (`None` = Unix socket only) | int | no | | +| vnc_tcp_bind | Address for `vnc_tcp_port` (`127.0.0.1` or `0.0.0.0` for LAN) | str | no | 127.0.0.1 | +| vnc_encrypt | Default noVNC `encrypt` URL flag | bool | no | false | ## Architecture -The driver is a composite with two child interfaces: +The driver is a composite with three child interfaces: 1. **video**: UVC snapshot capture and live frame streaming 2. **hid**: Keyboard and mouse control over USB serial +3. **vnc**: Unix-socket RFB endpoint (noVNC / any VNC client) -Both children share a single `NanoKVMUSBDevice` instance on the exporter so the +Video stream and VNC share a single capture pump so `/dev/video*` is opened once. +Keyboard and mouse events from the VNC client are translated to the same HID path as `hid`. +The children share a single `NanoKVMUSBDevice` instance on the exporter so the serial port and camera are opened once. ## Video streaming @@ -76,16 +89,14 @@ the Jumpstarter connection to whichever **client** opens the stream. ### Lifecycle 1. A client calls `video.stream("stream")` (context manager) or `open_stream()`. -2. The exporter starts an async task that captures JPEG frames from UVC and sends - them through the stream. +2. The exporter's shared FramePump already captures JPEG frames from UVC; the + stream task forwards those frames to the client. 3. The client reads frames with `stream.receive()` — each message is one JPEG. 4. When the client closes the context (or calls `close()`), the exporter stops - capturing and releases the stream. + the stream task. Capture continues if VNC or another stream is still using + the pump. -While the stream is active, the exporter dedicates a background task to video -capture. This does **not** block the whole exporter process (it is async), but it -does keep the UVC device busy until the client disconnects. HID commands remain -available on the `hid` child during streaming. +HID commands remain available on the `hid` child during streaming and VNC. For recording, OCR, frame deduplication, and preprocessing without blocking ``jmp shell``, use ``edge-clearance-delivery/video-receiver/`` (``stream-bridge.py`` + @@ -110,11 +121,44 @@ with video.stream("stream") as stream: frame_jpeg = stream.receive() ``` +## VNC + +The exporter runs an RFB 3.8 server on a Unix socket. Jumpstarter tunnels that +socket to the **client** (same pattern as QEMU): you do not need to be on the +exporter host. Keyboard and mouse in the VNC client go to the NanoKVM-USB HID. + +**From the Jumpstarter client** (any machine with a lease): + +```bash +j nanokvm-usb vnc session +``` + +That opens noVNC against a TCP/WebSocket port on *your* machine. For a native +viewer (TigerVNC, Remmina) on the client: + +```bash +j nanokvm-usb vnc forward-tcp 5900 +# then connect to localhost:5900 +``` + +Or from Python: + +```python +with lease.drivers["nanokvm-usb"].session() as url: + print(url) # noVNC URL +``` + +**TCP on the exporter** (optional): set `vnc_tcp_port` to bind RFB on the exporter +host. Default `vnc_tcp_bind` is `127.0.0.1` (local viewers on that machine only). +Use `vnc_tcp_bind: "0.0.0.0"` to accept LAN clients without a Jumpstarter tunnel. +Binding a non-loopback address without `vnc_password` exposes an unauthenticated +session on the network; set a password. + ## API reference ### NanoKVMUSBClient -Composite client with `video` and `hid` children. +Composite client with `video`, `hid`, and `vnc` children. ### NanoKVMUSBVideoClient @@ -130,6 +174,13 @@ Composite client with `video` and `hid` children. :members: paste_text, press_key, reset_hid, mouse_move_abs, mouse_move_rel, mouse_click, mouse_scroll ``` +### NanoKVMUSBVNCClient + +```{eval-rst} +.. autoclass:: jumpstarter_driver_nanokvm_usb.client.NanoKVMUSBVNCClient() + :members: session +``` + ## CLI usage ```bash @@ -143,6 +194,10 @@ j nanokvm-usb hid press enter # Mouse j nanokvm-usb hid mouse move 0.5 0.5 j nanokvm-usb hid mouse click --button left --x 0.5 --y 0.5 + +# VNC (view + HID) via Jumpstarter tunnel +j nanokvm-usb vnc session +j nanokvm-usb vnc forward-tcp 5900 ``` ## Host requirements diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml b/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml index b4bb07faa..05d780080 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml +++ b/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml @@ -17,3 +17,7 @@ export: video_fps: 30 screen_width: 1920 screen_height: 1080 + vnc_enabled: true + # vnc_password: "secret" + # vnc_tcp_port: 5900 + # vnc_tcp_bind: "0.0.0.0" # LAN; use vnc_password diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/__init__.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/__init__.py index 446ac9902..b136bfaa5 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/__init__.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/__init__.py @@ -1,13 +1,15 @@ -from .client import NanoKVMUSBClient, NanoKVMUSBHIDClient, NanoKVMUSBVideoClient -from .driver import NanoKVMUSB, NanoKVMUSBHID, NanoKVMUSBVideo +from .client import NanoKVMUSBClient, NanoKVMUSBHIDClient, NanoKVMUSBVideoClient, NanoKVMUSBVNCClient +from .driver import NanoKVMUSB, NanoKVMUSBHID, NanoKVMUSBVideo, NanoKVMUSBVNC from .mouse import MouseButton __all__ = [ "NanoKVMUSB", "NanoKVMUSBVideo", "NanoKVMUSBHID", + "NanoKVMUSBVNC", "NanoKVMUSBClient", "NanoKVMUSBVideoClient", "NanoKVMUSBHIDClient", + "NanoKVMUSBVNCClient", "MouseButton", ] diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/client.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/client.py index 8dbbbd6b3..92f0789ec 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/client.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/client.py @@ -1,17 +1,27 @@ import io +import webbrowser from base64 import b64decode from contextlib import contextmanager from dataclasses import dataclass +import anyio import click from jumpstarter_driver_composite.client import CompositeClient +from jumpstarter_driver_network.adapters.novnc import NovncAdapter +from jumpstarter_driver_network.client import NetworkClient from PIL import Image from .mouse import MouseButton, resolve_button from jumpstarter.client import DriverClient from jumpstarter.client.decorators import driver_click_group -__all__ = ["NanoKVMUSBVideoClient", "NanoKVMUSBHIDClient", "NanoKVMUSBClient", "MouseButton"] +__all__ = [ + "NanoKVMUSBVideoClient", + "NanoKVMUSBHIDClient", + "NanoKVMUSBClient", + "NanoKVMUSBVNCClient", + "MouseButton", +] def _decode_cli_escapes(text: str) -> str: @@ -161,6 +171,51 @@ def scroll(dx, dy): return base +class NanoKVMUSBVNCClient(NetworkClient): + """Client for the embedded RFB endpoint (noVNC session).""" + + def get_default_encrypt(self) -> bool: + return bool(self.call("get_default_encrypt")) + + @contextmanager + def session(self, *, encrypt: bool | None = None): + use_encrypt = self.get_default_encrypt() if encrypt is None else encrypt + with NovncAdapter(client=self, method="connect", encrypt=use_encrypt) as url: + yield url + + def cli(self): + base = super().cli() + + @base.command() + @click.option("--browser/--no-browser", default=True, help="Open the session in a web browser.") + @click.option( + "--encrypt", + "encrypt_override", + flag_value=True, + default=None, + help="Force an encrypted noVNC connection.", + ) + @click.option( + "--no-encrypt", + "encrypt_override", + flag_value=False, + help="Force an unencrypted noVNC connection.", + ) + def session(browser: bool, encrypt_override: bool | None): + """Open a noVNC session to the captured HDMI and HID.""" + with self.session(encrypt=encrypt_override) as url: + click.echo(f"To connect, please visit: {url}") + if browser: + webbrowser.open(url) + click.echo("Press Ctrl+C to close the VNC session.") + try: + self.portal.call(anyio.sleep_forever) + except (KeyboardInterrupt, anyio.get_cancelled_exc_class()): + click.echo("\nClosing VNC session.") + + return base + + @dataclass(kw_only=True) class NanoKVMUSBClient(CompositeClient): """ @@ -169,7 +224,16 @@ class NanoKVMUSBClient(CompositeClient): Provides access to: - video: UVC snapshot capture and live stream - hid: keyboard and mouse control over USB serial + - vnc: embedded RFB / noVNC session (when vnc_enabled) """ + @contextmanager + def session(self, *, encrypt: bool | None = None): + """Open a noVNC session via the ``vnc`` child.""" + if not hasattr(self, "vnc"): + raise RuntimeError("VNC is disabled (vnc_enabled=false)") + with self.vnc.session(encrypt=encrypt) as url: + yield url + def cli(self): return super().cli() diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py index 4ef57f74c..939e8c6d4 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py @@ -5,6 +5,7 @@ import threading import time +from .frame_pump import FramePump from .keyboard import KeyboardReport, resolve_key_code from .mouse import ( MouseButton, @@ -58,6 +59,8 @@ def __init__( self._buttons = 0 self._connected = False self._connect_lock = threading.RLock() + self._hid_lock = threading.RLock() + self._pump: FramePump | None = None @property def is_connected(self) -> bool: @@ -92,15 +95,22 @@ def connect(self) -> InfoPacket | None: ) self._connected = True + if self._video_device is not None: + if self._pump is None: + self._pump = FramePump(self.capture_frame_jpeg, fps=self._video_fps) + self._pump.start() return info except Exception: self.close() raise def close(self) -> None: - self._serial.close() - self._video.close() - self._connected = False + with self._connect_lock: + if self._pump is not None: + self._pump.stop() + self._serial.close() + self._video.close() + self._connected = False def get_info(self) -> InfoPacket: packet = CmdPacket(addr=self._addr, cmd=CmdEvent.GET_INFO) @@ -114,24 +124,34 @@ def _send_keyboard(self, report: list[int]) -> None: self._serial.write(packet.encode()) def press_key(self, key: str, hold: float = KEY_HOLD_DELAY) -> None: - code = resolve_key_code(key) - report = self._keyboard.key_down(code) - self._send_keyboard(report) - time.sleep(hold) - report = self._keyboard.key_up(code) - self._send_keyboard(report) + with self._hid_lock: + code = resolve_key_code(key) + report = self._keyboard.key_down(code) + self._send_keyboard(report) + time.sleep(hold) + report = self._keyboard.key_up(code) + self._send_keyboard(report) + + def hid_key(self, key: str, down: bool) -> None: + """Press or release a named key (used by the RFB server).""" + with self._hid_lock: + code = resolve_key_code(key) + report = self._keyboard.key_down(code) if down else self._keyboard.key_up(code) + self._send_keyboard(report) def release_all_keys(self) -> None: - report = self._keyboard.reset() - self._send_keyboard(report) + with self._hid_lock: + report = self._keyboard.reset() + self._send_keyboard(report) def type_text(self, text: str, delay: float = INTER_KEY_DELAY) -> None: - for ch in text: - down, up = self._keyboard.char_to_report(ch) - self._send_keyboard(down) - time.sleep(KEY_HOLD_DELAY) - self._send_keyboard(up) - time.sleep(delay) + with self._hid_lock: + for ch in text: + down, up = self._keyboard.char_to_report(ch) + self._send_keyboard(down) + time.sleep(KEY_HOLD_DELAY) + self._send_keyboard(up) + time.sleep(delay) def _send_mouse(self, report: list[int]) -> None: cmd = CmdEvent.SEND_MS_REL_DATA if report[0] == 0x01 else CmdEvent.SEND_MS_ABS_DATA @@ -139,16 +159,29 @@ def _send_mouse(self, report: list[int]) -> None: self._serial.write(packet.encode()) def mouse_move_abs(self, x: float, y: float) -> None: - report = build_absolute_report(x, y, buttons=self._buttons) - self._send_mouse(report) + with self._hid_lock: + report = build_absolute_report(x, y, buttons=self._buttons) + self._send_mouse(report) + + def mouse_pointer(self, x: float, y: float, buttons: int, wheel: int = 0) -> None: + """Absolute pointer update from the RFB server.""" + with self._hid_lock: + self._buttons = buttons & 0x1F + report = build_absolute_report(x, y, buttons=self._buttons, wheel=wheel) + self._send_mouse(report) def mouse_move_to(self, x: float, y: float) -> None: - self.mouse_move_relative(-self.screen_width * 2, -self.screen_height * 2) - target_x = int(x * self.screen_width) - target_y = int(y * self.screen_height) - self.mouse_move_relative(target_x, target_y) + with self._hid_lock: + self._mouse_move_relative_locked(-self.screen_width * 2, -self.screen_height * 2) + target_x = int(x * self.screen_width) + target_y = int(y * self.screen_height) + self._mouse_move_relative_locked(target_x, target_y) def mouse_move_relative(self, dx: int, dy: int, step_delay: float = 0.005) -> None: + with self._hid_lock: + self._mouse_move_relative_locked(dx, dy, step_delay) + + def _mouse_move_relative_locked(self, dx: int, dy: int, step_delay: float = 0.005) -> None: while dx != 0 or dy != 0: chunk_x = max(-127, min(127, dx)) chunk_y = max(-127, min(127, dy)) @@ -168,33 +201,37 @@ def mouse_click( ) -> None: btn_bit = resolve_button(button) - if x is not None and y is not None: - self.mouse_move_to(x, y) - self._buttons |= btn_bit - report = build_absolute_report(x, y, buttons=self._buttons) - self._send_mouse(report) - time.sleep(hold) - self._buttons &= ~btn_bit - report = build_absolute_report(x, y, buttons=self._buttons) - self._send_mouse(report) - else: - self._buttons |= btn_bit - report = build_relative_report(buttons=self._buttons) - self._send_mouse(report) - time.sleep(hold) - self._buttons &= ~btn_bit - report = build_relative_report(buttons=self._buttons) - self._send_mouse(report) + with self._hid_lock: + if x is not None and y is not None: + self._mouse_move_relative_locked(-self.screen_width * 2, -self.screen_height * 2) + self._mouse_move_relative_locked(int(x * self.screen_width), int(y * self.screen_height)) + self._buttons |= btn_bit + report = build_absolute_report(x, y, buttons=self._buttons) + self._send_mouse(report) + time.sleep(hold) + self._buttons &= ~btn_bit + report = build_absolute_report(x, y, buttons=self._buttons) + self._send_mouse(report) + else: + self._buttons |= btn_bit + report = build_relative_report(buttons=self._buttons) + self._send_mouse(report) + time.sleep(hold) + self._buttons &= ~btn_bit + report = build_relative_report(buttons=self._buttons) + self._send_mouse(report) def mouse_scroll(self, dx: int, dy: int) -> None: - wheel = dy if dy != 0 else dx - report = build_relative_report(wheel=wheel, buttons=self._buttons) - self._send_mouse(report) + with self._hid_lock: + wheel = dy if dy != 0 else dx + report = build_relative_report(wheel=wheel, buttons=self._buttons) + self._send_mouse(report) def mouse_reset(self) -> None: - self._buttons = 0 - report = build_relative_report(buttons=0) - self._send_mouse(report) + with self._hid_lock: + self._buttons = 0 + report = build_relative_report(buttons=0) + self._send_mouse(report) def reset_hid(self) -> None: self.release_all_keys() @@ -206,6 +243,13 @@ def capture_frame_jpeg(self, quality: int | None = None) -> bytes: self._video.discard_stale_frames(self._video_discard_stale) return self._video.read_frame_jpeg(q) + def snapshot_jpeg(self, skip_frames: int = 3) -> bytes: + if self._pump is not None and self._pump.is_running: + return self._pump.wait_n_frames(max(1, int(skip_frames) + 1)) + for _ in range(max(0, int(skip_frames))): + self.capture_frame_jpeg() + return self.capture_frame_jpeg() + def __enter__(self) -> NanoKVMUSBDevice: self.connect() return self diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py index 29962636c..283879126 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py @@ -1,17 +1,29 @@ +import shutil +import tempfile from base64 import b64encode from contextlib import asynccontextmanager from dataclasses import dataclass, field +from pathlib import Path import anyio from anyio import to_thread from jumpstarter_driver_composite.driver import Composite +from jumpstarter_driver_network.driver import UnixNetwork from .device import NanoKVMUSBDevice +from .frame_pump import FramePump from .keyboard import resolve_key_code from .mouse import MouseButton, resolve_button +from .vnc_server import RfbServer, is_loopback_bind from jumpstarter.driver import Driver, export, exportstream -__all__ = ["NanoKVMUSBVideo", "NanoKVMUSBHID", "NanoKVMUSB", "MouseButton"] +__all__ = [ + "NanoKVMUSBVideo", + "NanoKVMUSBHID", + "NanoKVMUSB", + "NanoKVMUSBVNC", + "MouseButton", +] @dataclass(kw_only=True) @@ -86,10 +98,7 @@ def _capture() -> bytes: device = self.device assert device is not None device.ensure_connected() - for _ in range(skip_frames): - device.capture_frame_jpeg() - data = device.capture_frame_jpeg() - return data + return device.snapshot_jpeg(skip_frames) device = await self._ensure_device() self.device = device @@ -108,19 +117,31 @@ async def stream(self): send_stream, receive_stream = anyio.create_memory_object_stream(max_buffer_size=buffer_size) async def stream_video(): - frame_interval = 1.0 / self.video_fps if self.video_fps > 0 else 0.0 + device = self.device + assert device is not None + state = {"gen": -1} + + def _next_frame() -> bytes: + pump = getattr(device, "_pump", None) + if isinstance(pump, FramePump): + gen = state["gen"] + got = pump.wait_jpeg( + timeout=2.0, + after_generation=gen if gen >= 0 else None, + ) + if got is None: + raise ConnectionError("Timed out waiting for video frame") + jpeg, state["gen"] = got + return jpeg + return device.capture_frame_jpeg(self.video_jpeg_quality) + async with send_stream: while True: - data = await to_thread.run_sync( - self.device.capture_frame_jpeg, # type: ignore[union-attr] - self.video_jpeg_quality, - ) + data = await to_thread.run_sync(_next_frame) try: await send_stream.send(data) except anyio.BrokenResourceError: break - if frame_interval > 0: - await anyio.sleep(frame_interval) async with anyio.create_task_group() as tg: tg.start_soon(stream_video) @@ -150,9 +171,7 @@ async def press_key(self, key: str): try: resolve_key_code(key) except ValueError: - self.logger.warning( - f"press_key should be used with single characters, got: {key}" - ) + self.logger.warning(f"press_key should be used with single characters, got: {key}") device = await self._ensure_device() @@ -211,12 +230,28 @@ async def mouse_scroll(self, dx: int, dy: int): self.logger.debug(f"Mouse scrolled: ({dx}, {dy})") +@dataclass(kw_only=True) +class NanoKVMUSBVNC(UnixNetwork): + """Unix-socket RFB endpoint with a noVNC session client.""" + + default_encrypt: bool = False + + @export + async def get_default_encrypt(self) -> bool: + return self.default_encrypt + + @classmethod + def client(cls) -> str: + return "jumpstarter_driver_nanokvm_usb.client.NanoKVMUSBVNCClient" + + @dataclass(kw_only=True) class NanoKVMUSB(Composite): """ Composite driver for NanoKVM-USB devices. - Provides video capture and HID control over USB serial + UVC. + Provides video capture, HID control, and an embedded RFB/VNC endpoint + over USB serial + UVC. """ serial_port: str @@ -232,8 +267,15 @@ class NanoKVMUSB(Composite): v4l2_ctl_executable: str | None = None screen_width: int = 1920 screen_height: int = 1080 + vnc_enabled: bool = True + vnc_password: str | None = None + vnc_tcp_port: int | None = None + vnc_tcp_bind: str = "127.0.0.1" + vnc_encrypt: bool = False _shared_device: NanoKVMUSBDevice = field(init=False, repr=False) + _vnc_server: RfbServer | None = field(init=False, repr=False, default=None) + _vnc_dir: str | None = field(init=False, repr=False, default=None) def __post_init__(self): self._shared_device = NanoKVMUSBDevice( @@ -275,16 +317,55 @@ def __post_init__(self): screen_height=self.screen_height, ), } - for child in self.children.values(): - child._owns_device = False + if self.vnc_enabled: + self._vnc_dir = tempfile.mkdtemp(prefix="nanokvm-usb-vnc-") + vnc_path = str(Path(self._vnc_dir) / "vnc.sock") + self._vnc_server = RfbServer( + vnc_path, + pump=lambda: self._shared_device._pump, + hid=self._shared_device, + width=self.video_width, + height=self.video_height, + password=self.vnc_password or None, + tcp_port=self.vnc_tcp_port, + tcp_bind=self.vnc_tcp_bind, + on_client=self._shared_device.ensure_connected, + ) + self.children["vnc"] = NanoKVMUSBVNC( + path=vnc_path, + default_encrypt=self.vnc_encrypt, + ) + for name in ("video", "hid"): + self.children[name]._owns_device = False super().__post_init__() + if self._vnc_server is not None: + if self.vnc_tcp_port is not None and not is_loopback_bind(self.vnc_tcp_bind) and not self.vnc_password: + self.logger.warning( + "RFB TCP bind %s:%s has no vnc_password; the session is reachable on the network", + self.vnc_tcp_bind, + self.vnc_tcp_port, + ) + self._vnc_server.start() @classmethod def client(cls) -> str: return "jumpstarter_driver_nanokvm_usb.client.NanoKVMUSBClient" def close(self): + if self._vnc_server is not None: + try: + self._vnc_server.stop() + except Exception as exc: + self.logger.debug(f"Error stopping RFB server: {exc}") + self._vnc_server = None + try: + super().close() + except Exception as exc: + self.logger.debug(f"Error closing child drivers: {exc}") + if self._vnc_dir is not None: + shutil.rmtree(self._vnc_dir, ignore_errors=True) + self._vnc_dir = None try: self._shared_device.close() except Exception as exc: diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py index a37fffc2f..d535704e3 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py @@ -1,12 +1,13 @@ """Tests for NanoKVM-USB driver.""" from io import BytesIO +from pathlib import Path from unittest.mock import MagicMock import pytest from PIL import Image -from .driver import NanoKVMUSB, NanoKVMUSBHID, NanoKVMUSBVideo +from .driver import NanoKVMUSB, NanoKVMUSBHID, NanoKVMUSBVideo, NanoKVMUSBVNC from .keyboard import KeyboardReport from .mouse import MouseButton, resolve_button from .v4l2_ctl_mjpeg import V4L2CtlMjpegCapture, _extract_jpegs @@ -26,7 +27,15 @@ def mock_device(): device.is_connected = True device.screen_width = 1920 device.screen_height = 1080 - device.capture_frame_jpeg.return_value = _jpeg_bytes() + jpeg = _jpeg_bytes() + device.capture_frame_jpeg.return_value = jpeg + + def _snapshot_jpeg(skip_frames=3): + for _ in range(int(skip_frames)): + device.capture_frame_jpeg() + return device.capture_frame_jpeg() + + device.snapshot_jpeg.side_effect = _snapshot_jpeg return device @@ -82,12 +91,14 @@ def test_nanokvm_usb_composite(mock_device): with serve(driver) as client: assert hasattr(client, "video") assert hasattr(client, "hid") + assert hasattr(client, "vnc") image = client.video.snapshot() assert isinstance(image, Image.Image) client.hid.paste_text("Test") mock_device.type_text.assert_called_with("Test") + assert client.vnc.get_default_encrypt() is False def test_nanokvm_usb_video_client_creation(): @@ -102,6 +113,19 @@ def test_nanokvm_usb_client_creation(): assert NanoKVMUSB.client() == "jumpstarter_driver_nanokvm_usb.client.NanoKVMUSBClient" +def test_nanokvm_usb_vnc_client_creation(): + assert NanoKVMUSBVNC.client() == "jumpstarter_driver_nanokvm_usb.client.NanoKVMUSBVNCClient" + + +def test_nanokvm_usb_vnc_disabled(): + driver = NanoKVMUSB(serial_port="/dev/null", video_device=0, vnc_enabled=False) + try: + assert "vnc" not in driver.children + assert driver._vnc_server is None + finally: + driver.close() + + def test_nanokvm_usb_mouse_move_abs(mock_device): hid = NanoKVMUSBHID(device=mock_device, serial_port="/dev/null", video_device=None) @@ -168,3 +192,204 @@ def test_protocol_packet_roundtrip(): assert decoded.addr == packet.addr assert decoded.cmd == packet.cmd assert decoded.data == packet.data + + +def test_frame_pump_fans_out_jpeg(): + from .frame_pump import FramePump + + jpeg = _jpeg_bytes(32, 24) + pump = FramePump(lambda: jpeg, fps=50) + pump.start() + try: + first = pump.wait_jpeg(timeout=2.0) + assert first is not None + data, gen = first + assert data == jpeg + second = pump.wait_jpeg(timeout=2.0, after_generation=gen) + assert second is not None + assert second[1] > gen + skipped = pump.wait_n_frames(3, timeout=2.0) + assert skipped == jpeg + finally: + pump.stop() + assert not pump.is_running + + +def test_keysym_and_pointer_mapping(): + from .vnc_server import keysym_to_key, rfb_buttons_to_hid + + assert keysym_to_key(ord("a")) == "KeyA" + assert keysym_to_key(ord("A")) == "KeyA" + assert keysym_to_key(ord("5")) == "Digit5" + assert keysym_to_key(0xFF0D) == "Enter" + assert keysym_to_key(0xFFBE) == "F1" + assert keysym_to_key(0xFFFF) == "Delete" + hid, wheel = rfb_buttons_to_hid(0x01) + assert hid == MouseButton.LEFT + assert wheel == 0 + hid, wheel = rfb_buttons_to_hid(0x08) + assert wheel == 1 + + +def test_des_encrypt_nist_vector(): + from .vnc_server import _des_ecb_encrypt, vnc_auth_response + + key = bytes.fromhex("133457799BBCDFF1") + plain = bytes.fromhex("0123456789ABCDEF") + assert _des_ecb_encrypt(plain, key) == bytes.fromhex("85E813540F0AB405") + response = vnc_auth_response(b"\x00" * 16, "secret") + assert len(response) == 16 + + +def test_rfb_handshake_and_input(tmp_path): + import socket + import struct + import time + + from .frame_pump import FramePump + from .vnc_server import RFB_VERSION, RfbServer, _recvexact + + jpeg = _jpeg_bytes(64, 48) + pump = FramePump(lambda: jpeg, fps=20) + pump.start() + hid = MagicMock() + sock_path = str(tmp_path / "vnc.sock") + server = RfbServer(sock_path, pump, hid, width=64, height=48) + server.start() + try: + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not Path(sock_path).exists(): + time.sleep(0.01) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(5) + client.connect(sock_path) + assert _recvexact(client, 12) == RFB_VERSION + client.sendall(RFB_VERSION) + header = _recvexact(client, 2) + assert header[0] == 1 + assert header[1] == 1 + client.sendall(bytes([1])) + assert struct.unpack("!I", _recvexact(client, 4))[0] == 0 + client.sendall(b"\x01") # ClientInit + server_init = _recvexact(client, 20) + width, height = struct.unpack("!HH", server_init[:4]) + assert (width, height) == (64, 48) + name_len = struct.unpack("!I", _recvexact(client, 4))[0] + assert _recvexact(client, name_len) == b"NanoKVM-USB" + client.sendall(b"\x02\x00" + struct.pack("!H", 1) + struct.pack("!i", 0)) + client.sendall(b"\x03\x00" + struct.pack("!HHHH", 0, 0, 64, 48)) + # KeyEvent: down, pad, keysym 'a' + client.sendall(b"\x04\x01\x00\x00" + struct.pack("!I", ord("a"))) + # PointerEvent: left button at 32,24 + client.sendall(b"\x05\x01" + struct.pack("!HH", 32, 24)) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not hid.hid_key.called: + time.sleep(0.02) + hid.hid_key.assert_called_with("KeyA", True) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not hid.mouse_pointer.called: + time.sleep(0.02) + args = hid.mouse_pointer.call_args[0] + assert args[2] == MouseButton.LEFT + client.close() + finally: + server.stop() + pump.stop() + + +def test_rfb_vnc_auth(tmp_path): + import socket + import struct + import time + + from .frame_pump import FramePump + from .vnc_server import RFB_VERSION, RfbServer, _recvexact, vnc_auth_response + + jpeg = _jpeg_bytes(16, 16) + pump = FramePump(lambda: jpeg, fps=10) + pump.start() + sock_path = str(tmp_path / "vnc-auth.sock") + server = RfbServer(sock_path, pump, MagicMock(), width=16, height=16, password="secret") + server.start() + try: + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not Path(sock_path).exists(): + time.sleep(0.01) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(5) + client.connect(sock_path) + assert _recvexact(client, 12) == RFB_VERSION + client.sendall(RFB_VERSION) + header = _recvexact(client, 2) + assert header == bytes([1, 2]) + client.sendall(bytes([2])) + challenge = _recvexact(client, 16) + client.sendall(vnc_auth_response(challenge, "secret")) + assert struct.unpack("!I", _recvexact(client, 4))[0] == 0 + client.close() + finally: + server.stop() + pump.stop() + + +def test_is_loopback_bind(): + from .vnc_server import is_loopback_bind + + assert is_loopback_bind("127.0.0.1") + assert is_loopback_bind("localhost") + assert not is_loopback_bind("0.0.0.0") + assert not is_loopback_bind("192.168.1.10") + + +def test_rfb_tcp_handshake(tmp_path): + import socket + import struct + import time + + from .frame_pump import FramePump + from .vnc_server import RFB_VERSION, RfbServer, _recvexact + + jpeg = _jpeg_bytes(16, 16) + pump = FramePump(lambda: jpeg, fps=10) + pump.start() + sock_path = str(tmp_path / "vnc-tcp.sock") + server = RfbServer( + sock_path, + pump, + MagicMock(), + width=16, + height=16, + tcp_port=0, + tcp_bind="127.0.0.1", + ) + server.start() + try: + endpoint = server.tcp_endpoint + assert endpoint is not None + host, port = endpoint + deadline = time.monotonic() + 2 + client = None + while time.monotonic() < deadline: + try: + client = socket.create_connection((host, port), timeout=2) + break + except OSError: + time.sleep(0.01) + assert client is not None + client.settimeout(5) + assert _recvexact(client, 12) == RFB_VERSION + client.sendall(RFB_VERSION) + header = _recvexact(client, 2) + assert header == bytes([1, 1]) + client.sendall(bytes([1])) + assert struct.unpack("!I", _recvexact(client, 4))[0] == 0 + client.sendall(b"\x01") + server_init = _recvexact(client, 20) + width, height = struct.unpack("!HH", server_init[:4]) + assert (width, height) == (16, 16) + name_len = struct.unpack("!I", _recvexact(client, 4))[0] + assert _recvexact(client, name_len) == b"NanoKVM-USB" + client.close() + finally: + server.stop() + pump.stop() diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/frame_pump.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/frame_pump.py new file mode 100644 index 000000000..4a65f20b1 --- /dev/null +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/frame_pump.py @@ -0,0 +1,110 @@ +"""Shared JPEG frame pump so video stream and VNC reuse one UVC capture.""" + +from __future__ import annotations + +import logging +import threading +import time +from collections.abc import Callable + +logger = logging.getLogger(__name__) + + +class FramePump: + """Capture JPEG frames on a background thread and fan them out to waiters.""" + + def __init__(self, capture: Callable[[], bytes], fps: int = 30) -> None: + self._capture = capture + self._fps = max(0, int(fps)) + self._cond = threading.Condition() + self._latest: bytes | None = None + self._generation = 0 + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + @property + def is_running(self) -> bool: + return self._thread is not None and self._thread.is_alive() + + @property + def generation(self) -> int: + with self._cond: + return self._generation + + def start(self) -> None: + if self.is_running: + return + self._stop.clear() + self._thread = threading.Thread(target=self._loop, name="nanokvm-usb-frame-pump", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + with self._cond: + self._cond.notify_all() + if self._thread is not None: + self._thread.join(timeout=2) + self._thread = None + + def wait_jpeg( + self, + timeout: float = 2.0, + after_generation: int | None = None, + ) -> tuple[bytes, int] | None: + """Block until a JPEG is available. + + If ``after_generation`` is set, wait for a strictly newer frame. + """ + + def _ready() -> bool: + if self._latest is None: + return False + if after_generation is None: + return True + return self._generation > after_generation + + with self._cond: + if not _ready(): + self._cond.wait_for(_ready, timeout=timeout) + if not _ready() or self._latest is None: + return None + return self._latest, self._generation + + def wait_n_frames(self, count: int, timeout: float = 5.0) -> bytes: + """Wait for ``count`` new frames and return the last JPEG.""" + if count < 1: + count = 1 + deadline = time.monotonic() + timeout + gen = -1 + jpeg: bytes | None = None + for _ in range(count): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + got = self.wait_jpeg(timeout=remaining, after_generation=gen) + if got is None: + break + jpeg, gen = got + if jpeg is None: + raise ConnectionError("Timed out waiting for video frame") + return jpeg + + def _loop(self) -> None: + interval = 1.0 / self._fps if self._fps > 0 else 0.0 + while not self._stop.is_set(): + started = time.monotonic() + try: + jpeg = self._capture() + except Exception: + logger.debug("frame pump capture failed", exc_info=True) + if self._stop.wait(0.2): + break + continue + with self._cond: + self._latest = jpeg + self._generation += 1 + self._cond.notify_all() + if interval > 0: + remaining = interval - (time.monotonic() - started) + if remaining > 0 and self._stop.wait(remaining): + break diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/protocol.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/protocol.py index bbd850286..96cfb5cc0 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/protocol.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/protocol.py @@ -64,9 +64,7 @@ def decode(cls, buf: bytes | list[int]) -> CmdPacket: expected &= 0xFF if expected != checksum: - raise ValueError( - f"Checksum mismatch: expected 0x{expected:02X}, got 0x{checksum:02X}" - ) + raise ValueError(f"Checksum mismatch: expected 0x{expected:02X}, got 0x{checksum:02X}") payload = data[header_idx + 5 : header_idx + 5 + data_len] return cls(addr=addr, cmd=cmd, data=payload) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_ctl_mjpeg.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_ctl_mjpeg.py index 7bda3f499..3d3608193 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_ctl_mjpeg.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_ctl_mjpeg.py @@ -70,9 +70,7 @@ def open(self, device: int | str, width: int, height: int, fps: int) -> None: requested = self._v4l2_ctl_executable or "v4l2-ctl" executable = resolve_v4l2_ctl_executable(self._v4l2_ctl_executable) if executable is None: - raise OSError( - f"v4l2-ctl not found: {requested!r} (install v4l-utils or fix v4l2_ctl_executable)" - ) + raise OSError(f"v4l2-ctl not found: {requested!r} (install v4l-utils or fix v4l2_ctl_executable)") if self.is_open: self.close() @@ -88,10 +86,7 @@ def open(self, device: int | str, width: int, height: int, fps: int) -> None: proc = self._start_process() time.sleep(_STARTUP_POLL_S) if proc.poll() is not None: - raise OSError( - f"v4l2-ctl exited immediately while opening {self._device} " - f"(exit code {proc.returncode})" - ) + raise OSError(f"v4l2-ctl exited immediately while opening {self._device} (exit code {proc.returncode})") except OSError: self._cleanup_process() raise diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_mjpeg.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_mjpeg.py index 2f4f9419e..aa9caff85 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_mjpeg.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/v4l2_mjpeg.py @@ -285,9 +285,7 @@ def open(self, device: int | str, width: int, height: int, fps: int) -> None: self._buffers = buffers self._width = actual_w self._height = actual_h - logger.info( - "V4L2 MJPEG passthrough on %s (%sx%s @ %sfps)", path, actual_w, actual_h, fps - ) + logger.info("V4L2 MJPEG passthrough on %s (%sx%s @ %sfps)", path, actual_w, actual_h, fps) except Exception: _release_mmap_buffers(buffers) if fd is not None: diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py new file mode 100644 index 000000000..b17544fb8 --- /dev/null +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py @@ -0,0 +1,659 @@ +"""Embedded RFB 3.8 server: JPEG frames in, HID events out.""" + +from __future__ import annotations + +import logging +import os +import select +import socket +import struct +import threading +from collections.abc import Callable +from io import BytesIO +from ipaddress import ip_address +from typing import Protocol + +import numpy as np +from PIL import Image + +from .frame_pump import FramePump +from .mouse import MouseButton + +logger = logging.getLogger(__name__) + +RFB_VERSION = b"RFB 003.008\n" +SEC_NONE = 1 +SEC_VNCAUTH = 2 +ENCODING_RAW = 0 + +MSG_SET_PIXEL_FORMAT = 0 +MSG_SET_ENCODINGS = 2 +MSG_FB_UPDATE_REQUEST = 3 +MSG_KEY_EVENT = 4 +MSG_POINTER_EVENT = 5 +MSG_CLIENT_CUT_TEXT = 6 + +# X11 keysyms used by RFB +_KEYSYM_NAMED: dict[int, str] = { + 0xFF08: "Backspace", + 0xFF09: "Tab", + 0xFF0D: "Enter", + 0xFF1B: "Escape", + 0xFFFF: "Delete", + 0xFF50: "Home", + 0xFF51: "ArrowLeft", + 0xFF52: "ArrowUp", + 0xFF53: "ArrowRight", + 0xFF54: "ArrowDown", + 0xFF55: "PageUp", + 0xFF56: "PageDown", + 0xFF57: "End", + 0xFF63: "Insert", + 0xFFE1: "ShiftLeft", + 0xFFE2: "ShiftRight", + 0xFFE3: "ControlLeft", + 0xFFE4: "ControlRight", + 0xFFE7: "MetaLeft", + 0xFFE8: "MetaRight", + 0xFFE9: "AltLeft", + 0xFFEA: "AltRight", + 0xFFEB: "MetaLeft", + 0xFFEC: "MetaRight", + 0x0020: "Space", +} +for _i in range(12): + _KEYSYM_NAMED[0xFFBE + _i] = f"F{_i + 1}" + +_UNSHIFTED_PUNCT: dict[int, str] = { + ord(" "): "Space", + ord("`"): "Backquote", + ord("-"): "Minus", + ord("="): "Equal", + ord("["): "BracketLeft", + ord("]"): "BracketRight", + ord("\\"): "Backslash", + ord(";"): "Semicolon", + ord("'"): "Quote", + ord(","): "Comma", + ord("."): "Period", + ord("/"): "Slash", + # shifted variants still use the unshifted HID key; Shift is a separate event + ord("~"): "Backquote", + ord("!"): "Digit1", + ord("@"): "Digit2", + ord("#"): "Digit3", + ord("$"): "Digit4", + ord("%"): "Digit5", + ord("^"): "Digit6", + ord("&"): "Digit7", + ord("*"): "Digit8", + ord("("): "Digit9", + ord(")"): "Digit0", + ord("_"): "Minus", + ord("+"): "Equal", + ord("{"): "BracketLeft", + ord("}"): "BracketRight", + ord("|"): "Backslash", + ord(":"): "Semicolon", + ord('"'): "Quote", + ord("<"): "Comma", + ord(">"): "Period", + ord("?"): "Slash", +} + + +class HidTarget(Protocol): + def hid_key(self, key: str, down: bool) -> None: ... + def mouse_pointer(self, x: float, y: float, buttons: int, wheel: int = 0) -> None: ... + + +def keysym_to_key(keysym: int) -> str | None: + """Map an RFB/X11 keysym to a NanoKVM-USB key name.""" + if keysym in _KEYSYM_NAMED: + return _KEYSYM_NAMED[keysym] + if 0x61 <= keysym <= 0x7A: # a-z + return f"Key{chr(keysym).upper()}" + if 0x41 <= keysym <= 0x5A: # A-Z + return f"Key{chr(keysym)}" + if 0x30 <= keysym <= 0x39: + return f"Digit{chr(keysym)}" + if keysym in _UNSHIFTED_PUNCT: + return _UNSHIFTED_PUNCT[keysym] + if 0x20 <= keysym <= 0x7E: + ch = chr(keysym) + if ch.isalpha(): + return f"Key{ch.upper()}" + if ch.isdigit(): + return f"Digit{ch}" + return None + + +def rfb_buttons_to_hid(mask: int) -> tuple[int, int]: + """Translate RFB pointer button mask to HID buttons and wheel delta.""" + hid = 0 + if mask & 0x01: + hid |= MouseButton.LEFT + if mask & 0x02: + hid |= MouseButton.MIDDLE + if mask & 0x04: + hid |= MouseButton.RIGHT + if mask & 0x80: + hid |= MouseButton.BACK + wheel = 0 + if mask & 0x08: + wheel = 1 + elif mask & 0x10: + wheel = -1 + return hid, wheel + + +def is_loopback_bind(host: str) -> bool: + """Return True if ``host`` only accepts connections from the local machine.""" + if host.lower() in {"localhost", "localhost.localdomain"}: + return True + try: + return ip_address(host).is_loopback + except ValueError: + return False + + +def _recvexact(sock: socket.socket, n: int) -> bytes: + buf = bytearray() + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise ConnectionError("RFB client closed the connection") + buf.extend(chunk) + return bytes(buf) + + +def _bitrev8(value: int) -> int: + value = ((value * 0x0202020202) & 0x010884422010) % 1023 + return value + + +def vnc_auth_response(challenge: bytes, password: str) -> bytes: + """Encrypt the 16-byte VNC challenge with the bit-reversed DES password.""" + if len(challenge) != 16: + raise ValueError("VNC challenge must be 16 bytes") + key = password.encode("latin-1", "replace")[:8].ljust(8, b"\x00") + des_key = bytes(_bitrev8(b) for b in key) + return _des_ecb_encrypt(challenge[:8], des_key) + _des_ecb_encrypt(challenge[8:], des_key) + + +# DES encrypt (ECB, one 8-byte block). Tables from FIPS 46-3 / public domain d3des. +# fmt: off +_IP = ( + 58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, + 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, + 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, + 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7, +) +_FP = ( + 40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, + 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, + 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, + 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25, +) +_E = ( + 32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, 12, 13, + 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, 22, 23, 24, 25, + 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1, +) +_P = ( + 16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, + 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25, +) +_PC1 = ( + 57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, + 10, 2, 59, 51, 43, 35, 27, 19, 11, 3, 60, 52, 44, 36, + 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, 46, 38, 30, 22, + 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4, +) +_PC2 = ( + 14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, + 23, 19, 12, 4, 26, 8, 16, 7, 27, 20, 13, 2, + 41, 52, 31, 37, 47, 55, 30, 40, 51, 45, 33, 48, + 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32, +) +_SHIFTS = (1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1) +_SBOX = ( + ( + 14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, + 0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, + 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0, + 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13, + ), + ( + 15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, + 3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, + 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15, + 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9, + ), + ( + 10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, + 13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, + 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7, + 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12, + ), + ( + 7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, + 13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, + 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4, + 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14, + ), + ( + 2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, + 14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, + 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14, + 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3, + ), + ( + 12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, + 10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, + 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6, + 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13, + ), + ( + 4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, + 13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, + 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2, + 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12, + ), + ( + 13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, + 1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, + 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8, + 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11, + ), +) +# fmt: on + + +def _permute(value: int, table: tuple[int, ...], nbits: int) -> int: + out = 0 + for bit in table: + out = (out << 1) | ((value >> (nbits - bit)) & 1) + return out + + +def _des_subkeys(key: bytes) -> list[int]: + k = int.from_bytes(key, "big") + cd = _permute(k, _PC1, 64) + c = (cd >> 28) & 0xFFFFFFF + d = cd & 0xFFFFFFF + keys = [] + for shift in _SHIFTS: + c = ((c << shift) | (c >> (28 - shift))) & 0xFFFFFFF + d = ((d << shift) | (d >> (28 - shift))) & 0xFFFFFFF + keys.append(_permute((c << 28) | d, _PC2, 56)) + return keys + + +def _des_f(r: int, subkey: int) -> int: + er = _permute(r, _E, 32) ^ subkey + s = 0 + for i in range(8): + chunk = (er >> (42 - 6 * i)) & 0x3F + row = ((chunk & 0x20) >> 4) | (chunk & 1) + col = (chunk >> 1) & 0xF + s = (s << 4) | _SBOX[i][row * 16 + col] + return _permute(s, _P, 32) + + +def _des_ecb_encrypt(block: bytes, key: bytes) -> bytes: + ip = _permute(int.from_bytes(block, "big"), _IP, 64) + left, right = ip >> 32, ip & 0xFFFFFFFF + for subkey in _des_subkeys(key): + left, right = right, left ^ _des_f(right, subkey) + preout = ((right & 0xFFFFFFFF) << 32) | (left & 0xFFFFFFFF) + return _permute(preout, _FP, 64).to_bytes(8, "big") + + +def _default_pixel_format() -> dict[str, int]: + return { + "bits_per_pixel": 32, + "depth": 24, + "big_endian": 0, + "true_colour": 1, + "red_max": 255, + "green_max": 255, + "blue_max": 255, + "red_shift": 16, + "green_shift": 8, + "blue_shift": 0, + } + + +def _pack_pixel_format(pf: dict[str, int]) -> bytes: + return struct.pack( + "!BBBBHHHBBB3x", + pf["bits_per_pixel"], + pf["depth"], + pf["big_endian"], + pf["true_colour"], + pf["red_max"], + pf["green_max"], + pf["blue_max"], + pf["red_shift"], + pf["green_shift"], + pf["blue_shift"], + ) + + +def _unpack_pixel_format(data: bytes) -> dict[str, int]: + values = struct.unpack("!BBBBHHHBBB3x", data) + keys = ( + "bits_per_pixel", + "depth", + "big_endian", + "true_colour", + "red_max", + "green_max", + "blue_max", + "red_shift", + "green_shift", + "blue_shift", + ) + return dict(zip(keys, values, strict=True)) + + +def pack_rgb_frame(image: Image.Image, pf: dict[str, int]) -> bytes: + """Pack an RGB image into RFB raw pixels for ``pf``.""" + rgb = image.convert("RGB") + bpp = pf["bits_per_pixel"] + if bpp not in (16, 32): + bpp = 32 + pf = _default_pixel_format() + arr = np.asarray(rgb, dtype=np.uint16 if bpp == 16 else np.uint32) + r = arr[:, :, 0].astype(np.uint32) + g = arr[:, :, 1].astype(np.uint32) + b = arr[:, :, 2].astype(np.uint32) + r = r * pf["red_max"] // 255 + g = g * pf["green_max"] // 255 + b = b * pf["blue_max"] // 255 + pix = (r << pf["red_shift"]) | (g << pf["green_shift"]) | (b << pf["blue_shift"]) + dtype = ">u4" if pf["big_endian"] else " None: + self.path = path + self._pump = pump + self._hid = hid + self._width = max(1, int(width)) + self._height = max(1, int(height)) + self._password = password + self._tcp_port = tcp_port + self._tcp_bind = tcp_bind + self._on_client = on_client + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._listen: socket.socket | None = None + self._tcp: socket.socket | None = None + self._clients: list[socket.socket] = [] + self._clients_lock = threading.Lock() + + def start(self) -> None: + if self._thread is not None: + return + if os.path.exists(self.path): + os.unlink(self.path) + os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True) + listen = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listen.bind(self.path) + listen.listen(4) + listen.setblocking(False) + self._listen = listen + if self._tcp_port is not None: + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tcp.bind((self._tcp_bind, self._tcp_port)) + tcp.listen(4) + tcp.setblocking(False) + self._tcp = tcp + self._tcp_port = tcp.getsockname()[1] + self._stop.clear() + self._thread = threading.Thread(target=self._accept_loop, name="nanokvm-usb-rfb", daemon=True) + self._thread.start() + if self._tcp is not None: + logger.info("RFB server listening on %s and tcp %s:%s", self.path, self._tcp_bind, self._tcp_port) + else: + logger.info("RFB server listening on %s", self.path) + + @property + def tcp_endpoint(self) -> tuple[str, int] | None: + if self._tcp is None: + return None + addr = self._tcp.getsockname() + return addr[0], addr[1] + + def stop(self) -> None: + self._stop.set() + for sock in (self._listen, self._tcp): + if sock is not None: + try: + sock.close() + except OSError: + pass + with self._clients_lock: + clients = list(self._clients) + for client in clients: + try: + client.close() + except OSError: + pass + if self._thread is not None: + self._thread.join(timeout=2) + self._thread = None + self._listen = None + self._tcp = None + if os.path.exists(self.path): + try: + os.unlink(self.path) + except OSError: + pass + + def _accept_loop(self) -> None: + while not self._stop.is_set(): + socks = [s for s in (self._listen, self._tcp) if s is not None] + if not socks: + break + try: + readable, _, _ = select.select(socks, [], [], 0.5) + except (OSError, ValueError): + break + for listener in readable: + try: + conn, _addr = listener.accept() + except OSError: + continue + conn.settimeout(30) + with self._clients_lock: + self._clients.append(conn) + threading.Thread( + target=self._session, + args=(conn,), + name="nanokvm-usb-rfb-client", + daemon=True, + ).start() + + def _session(self, conn: socket.socket) -> None: + try: + if self._on_client is not None: + self._on_client() + self._handshake(conn) + self._serve(conn) + except (ConnectionError, OSError, TimeoutError, struct.error) as exc: + logger.debug("RFB client disconnected: %s", exc) + except Exception: + logger.exception("RFB session failed") + finally: + try: + conn.close() + except OSError: + pass + with self._clients_lock: + if conn in self._clients: + self._clients.remove(conn) + + def _handshake(self, conn: socket.socket) -> None: + conn.sendall(RFB_VERSION) + client_ver = _recvexact(conn, 12) + if not client_ver.startswith(b"RFB "): + raise ConnectionError(f"invalid RFB version: {client_ver!r}") + if self._password: + conn.sendall(bytes([1, SEC_VNCAUTH])) + chosen = _recvexact(conn, 1)[0] + if chosen != SEC_VNCAUTH: + conn.sendall(struct.pack("!I", 1)) + raise ConnectionError("client did not select VncAuth") + challenge = os.urandom(16) + conn.sendall(challenge) + response = _recvexact(conn, 16) + if response != vnc_auth_response(challenge, self._password): + conn.sendall(struct.pack("!I", 1)) + raise ConnectionError("VNC authentication failed") + conn.sendall(struct.pack("!I", 0)) + else: + conn.sendall(bytes([1, SEC_NONE])) + chosen = _recvexact(conn, 1)[0] + if chosen != SEC_NONE: + conn.sendall(struct.pack("!I", 1)) + raise ConnectionError("client did not select None security") + conn.sendall(struct.pack("!I", 0)) + _recvexact(conn, 1) # ClientInit shared-flag + name = b"NanoKVM-USB" + conn.sendall( + struct.pack("!HH", self._width, self._height) + + _pack_pixel_format(_default_pixel_format()) + + struct.pack("!I", len(name)) + + name + ) + + def _serve(self, conn: socket.socket) -> None: + pf = _default_pixel_format() + last_gen = -1 + want_update = True + conn.setblocking(False) + while not self._stop.is_set(): + try: + readable, _, _ = select.select([conn], [], [], 0.05) + except (OSError, ValueError): + break + if readable: + result = self._read_client_message(conn, pf) + if result is None: + break + pf, requested = result + if requested: + want_update = True + if want_update: + sent = self._send_frame(conn, pf, last_gen) + if sent is not None: + last_gen = sent + want_update = False + + def _read_client_message( # noqa: C901 + self, conn: socket.socket, pf: dict[str, int] + ) -> tuple[dict[str, int], bool] | None: + try: + header = conn.recv(1) + except BlockingIOError: + header = b"" + if not header: + return None + msg = header[0] + requested = False + conn.setblocking(True) + try: + if msg == MSG_SET_PIXEL_FORMAT: + _recvexact(conn, 3) + pf = _unpack_pixel_format(_recvexact(conn, 16)) + elif msg == MSG_SET_ENCODINGS: + _recvexact(conn, 1) + count = struct.unpack("!H", _recvexact(conn, 2))[0] + if count: + _recvexact(conn, 4 * count) + elif msg == MSG_FB_UPDATE_REQUEST: + _recvexact(conn, 9) + requested = True + elif msg == MSG_KEY_EVENT: + payload = _recvexact(conn, 7) + down = payload[0] != 0 + keysym = struct.unpack("!I", payload[3:7])[0] + self._handle_key(keysym, down) + elif msg == MSG_POINTER_EVENT: + payload = _recvexact(conn, 5) + mask, x, y = struct.unpack("!BHH", payload) + self._handle_pointer(mask, x, y) + elif msg == MSG_CLIENT_CUT_TEXT: + _recvexact(conn, 3) + length = struct.unpack("!I", _recvexact(conn, 4))[0] + if length: + _recvexact(conn, length) + else: + logger.debug("ignoring unknown RFB client message %s", msg) + finally: + conn.setblocking(False) + return pf, requested + + def _handle_key(self, keysym: int, down: bool) -> None: + key = keysym_to_key(keysym) + if key is None: + logger.debug("unmapped RFB keysym 0x%04x", keysym) + return + try: + self._hid.hid_key(key, down) + except Exception: + logger.debug("HID key event failed", exc_info=True) + + def _handle_pointer(self, mask: int, x: int, y: int) -> None: + nx = 0.0 if self._width <= 1 else max(0.0, min(1.0, x / (self._width - 1))) + ny = 0.0 if self._height <= 1 else max(0.0, min(1.0, y / (self._height - 1))) + buttons, wheel = rfb_buttons_to_hid(mask) + try: + self._hid.mouse_pointer(nx, ny, buttons, wheel) + except Exception: + logger.debug("HID pointer event failed", exc_info=True) + + def _send_frame(self, conn: socket.socket, pf: dict[str, int], last_gen: int) -> int | None: + pump = self._pump() if callable(self._pump) else self._pump + if pump is None: + return None + got = pump.wait_jpeg(timeout=0.05, after_generation=last_gen if last_gen >= 0 else None) + if got is None: + return None + jpeg, gen = got + try: + image = Image.open(BytesIO(jpeg)).convert("RGB") + except Exception: + logger.debug("failed to decode JPEG for RFB", exc_info=True) + return gen + if image.size != (self._width, self._height): + image = image.resize((self._width, self._height)) + pixels = pack_rgb_frame(image, pf) + header = struct.pack("!BxH", 0, 1) + struct.pack("!HHHHi", 0, 0, self._width, self._height, ENCODING_RAW) + conn.setblocking(True) + try: + conn.sendall(header + pixels) + finally: + conn.setblocking(False) + return gen diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/pyproject.toml b/python/packages/jumpstarter-driver-nanokvm-usb/pyproject.toml index a88b6cb1d..a39a9b2c1 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/pyproject.toml +++ b/python/packages/jumpstarter-driver-nanokvm-usb/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "anyio>=4.10.0", "jumpstarter", "jumpstarter-driver-composite", + "jumpstarter-driver-network", "opencv-python-headless", "numpy", "pillow>=10.0.0", diff --git a/python/uv.lock b/python/uv.lock index 79bf2014e..1fdbfc77b 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -2526,6 +2526,7 @@ dependencies = [ { name = "click" }, { name = "jumpstarter" }, { name = "jumpstarter-driver-composite" }, + { name = "jumpstarter-driver-network" }, { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, { name = "opencv-python-headless" }, @@ -2548,6 +2549,7 @@ requires-dist = [ { name = "click" }, { name = "jumpstarter", editable = "packages/jumpstarter" }, { name = "jumpstarter-driver-composite", editable = "packages/jumpstarter-driver-composite" }, + { name = "jumpstarter-driver-network", editable = "packages/jumpstarter-driver-network" }, { name = "numpy" }, { name = "opencv-python-headless" }, { name = "pillow", specifier = ">=10.0.0" }, From e538b950d489b7eca85b33f503203c2ddd4738fe Mon Sep 17 00:00:00 2001 From: mparram Date: Mon, 21 Sep 2026 14:41:57 +0200 Subject: [PATCH 2/5] fix(nanokvm-usb): keep RFB socket timeout in blocking I/O Co-authored-by: Cursor --- .../jumpstarter_driver_nanokvm_usb/vnc_server.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py index b17544fb8..3c20ec903 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py @@ -32,6 +32,7 @@ MSG_KEY_EVENT = 4 MSG_POINTER_EVENT = 5 MSG_CLIENT_CUT_TEXT = 6 +_CLIENT_IO_TIMEOUT = 30 # X11 keysyms used by RFB _KEYSYM_NAMED: dict[int, str] = { @@ -485,7 +486,7 @@ def _accept_loop(self) -> None: conn, _addr = listener.accept() except OSError: continue - conn.settimeout(30) + conn.settimeout(_CLIENT_IO_TIMEOUT) with self._clients_lock: self._clients.append(conn) threading.Thread( @@ -552,7 +553,7 @@ def _serve(self, conn: socket.socket) -> None: pf = _default_pixel_format() last_gen = -1 want_update = True - conn.setblocking(False) + conn.settimeout(0) while not self._stop.is_set(): try: readable, _, _ = select.select([conn], [], [], 0.05) @@ -582,7 +583,7 @@ def _read_client_message( # noqa: C901 return None msg = header[0] requested = False - conn.setblocking(True) + conn.settimeout(_CLIENT_IO_TIMEOUT) try: if msg == MSG_SET_PIXEL_FORMAT: _recvexact(conn, 3) @@ -612,7 +613,7 @@ def _read_client_message( # noqa: C901 else: logger.debug("ignoring unknown RFB client message %s", msg) finally: - conn.setblocking(False) + conn.settimeout(0) return pf, requested def _handle_key(self, keysym: int, down: bool) -> None: @@ -651,9 +652,9 @@ def _send_frame(self, conn: socket.socket, pf: dict[str, int], last_gen: int) -> image = image.resize((self._width, self._height)) pixels = pack_rgb_frame(image, pf) header = struct.pack("!BxH", 0, 1) + struct.pack("!HHHHi", 0, 0, self._width, self._height, ENCODING_RAW) - conn.setblocking(True) + conn.settimeout(_CLIENT_IO_TIMEOUT) try: conn.sendall(header + pixels) finally: - conn.setblocking(False) + conn.settimeout(0) return gen From f12d164648f2fcc4764c5c940b4d524bd3f086aa Mon Sep 17 00:00:00 2001 From: mparram Date: Tue, 22 Sep 2026 10:55:23 +0200 Subject: [PATCH 3/5] fix(nanokvm-usb): inject VNC characters using DUT keyboard layout Co-authored-by: Cursor --- .../jumpstarter-driver-nanokvm-usb/README.md | 7 + .../examples/exporter.yaml | 1 + .../jumpstarter_driver_nanokvm_usb/device.py | 6 + .../jumpstarter_driver_nanokvm_usb/driver.py | 2 + .../driver_test.py | 32 +++- .../keyboard.py | 43 +++++ .../vnc_keymap.py | 179 ++++++++++++++++++ .../vnc_server.py | 118 +++--------- 8 files changed, 294 insertions(+), 94 deletions(-) create mode 100644 python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_keymap.py diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/README.md b/python/packages/jumpstarter-driver-nanokvm-usb/README.md index cbc84f1fd..80c3a4681 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/README.md +++ b/python/packages/jumpstarter-driver-nanokvm-usb/README.md @@ -44,6 +44,7 @@ export: screen_width: 1920 screen_height: 1080 vnc_enabled: true + # vnc_layout: es # DUT keyboard layout (default us) # vnc_password: "secret" # vnc_tcp_port: 5900 # vnc_tcp_bind: "0.0.0.0" # LAN; use vnc_password @@ -66,6 +67,7 @@ export: | vnc_tcp_port | Also bind RFB TCP on the exporter (`None` = Unix socket only) | int | no | | | vnc_tcp_bind | Address for `vnc_tcp_port` (`127.0.0.1` or `0.0.0.0` for LAN) | str | no | 127.0.0.1 | | vnc_encrypt | Default noVNC `encrypt` URL flag | bool | no | false | +| vnc_layout | DUT keyboard layout for VNC character injection (`us` or `es`) | str | no | us | ## Architecture @@ -154,6 +156,11 @@ Use `vnc_tcp_bind: "0.0.0.0"` to accept LAN clients without a Jumpstarter tunnel Binding a non-loopback address without `vnc_password` exposes an unauthenticated session on the network; set a password. +Printable keys are injected as HID combos for **`vnc_layout`** (the DUT OS +keyboard), not the VNC client's layout. Default is `us`. Set `vnc_layout: es` +if the DUT uses a Spanish keyboard so `@`, `ñ`, and AltGr characters land +correctly even when Remmina is using another layout. + ## API reference ### NanoKVMUSBClient diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml b/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml index 05d780080..76170d891 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml +++ b/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml @@ -18,6 +18,7 @@ export: screen_width: 1920 screen_height: 1080 vnc_enabled: true + # vnc_layout: es # DUT keyboard layout (default us) # vnc_password: "secret" # vnc_tcp_port: 5900 # vnc_tcp_bind: "0.0.0.0" # LAN; use vnc_password diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py index 939e8c6d4..32c519138 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py @@ -139,6 +139,12 @@ def hid_key(self, key: str, down: bool) -> None: report = self._keyboard.key_down(code) if down else self._keyboard.key_up(code) self._send_keyboard(report) + def hid_char(self, key: str, modifiers: frozenset[str], down: bool) -> None: + """Type a printable character using DUT-layout HID modifiers (RFB).""" + with self._hid_lock: + report = self._keyboard.printable_down(key, modifiers) if down else self._keyboard.printable_up() + self._send_keyboard(report) + def release_all_keys(self) -> None: with self._hid_lock: report = self._keyboard.reset() diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py index 283879126..662762cd5 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py @@ -272,6 +272,7 @@ class NanoKVMUSB(Composite): vnc_tcp_port: int | None = None vnc_tcp_bind: str = "127.0.0.1" vnc_encrypt: bool = False + vnc_layout: str = "us" _shared_device: NanoKVMUSBDevice = field(init=False, repr=False) _vnc_server: RfbServer | None = field(init=False, repr=False, default=None) @@ -329,6 +330,7 @@ def __post_init__(self): password=self.vnc_password or None, tcp_port=self.vnc_tcp_port, tcp_bind=self.vnc_tcp_bind, + layout=self.vnc_layout, on_client=self._shared_device.ensure_connected, ) self.children["vnc"] = NanoKVMUSBVNC( diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py index d535704e3..397563b71 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py @@ -8,7 +8,7 @@ from PIL import Image from .driver import NanoKVMUSB, NanoKVMUSBHID, NanoKVMUSBVideo, NanoKVMUSBVNC -from .keyboard import KeyboardReport +from .keyboard import MODIFIER_BITS, KeyboardReport from .mouse import MouseButton, resolve_button from .v4l2_ctl_mjpeg import V4L2CtlMjpegCapture, _extract_jpegs from jumpstarter.common.utils import serve @@ -216,6 +216,7 @@ def test_frame_pump_fans_out_jpeg(): def test_keysym_and_pointer_mapping(): + from .vnc_keymap import ALTGR, SHIFT, char_combo, is_swallowed_keysym, named_key, normalize_layout from .vnc_server import keysym_to_key, rfb_buttons_to_hid assert keysym_to_key(ord("a")) == "KeyA" @@ -224,6 +225,31 @@ def test_keysym_and_pointer_mapping(): assert keysym_to_key(0xFF0D) == "Enter" assert keysym_to_key(0xFFBE) == "F1" assert keysym_to_key(0xFFFF) == "Delete" + assert named_key(0xFFE5) == "CapsLock" + assert named_key(0xFFB0) == "Numpad0" + assert is_swallowed_keysym(0xFE03) + assert keysym_to_key(0xFE03) is None + + assert char_combo(ord("@"), "us") == ("Digit2", SHIFT) + assert char_combo(0x01000040, "us") == ("Digit2", SHIFT) + assert char_combo(ord("@"), "es") == ("Digit2", ALTGR) + assert char_combo(ord('"'), "us") == ("Quote", SHIFT) + assert char_combo(ord('"'), "es") == ("Digit2", SHIFT) + assert char_combo(ord("/"), "us") == ("Slash", frozenset()) + assert char_combo(ord("/"), "es") == ("Digit7", SHIFT) + assert char_combo(0xF1, "es") == ("Semicolon", frozenset()) + assert char_combo(0x010000F1, "es") == ("Semicolon", frozenset()) + with pytest.raises(ValueError, match="unsupported vnc_layout"): + normalize_layout("de") + + kb = KeyboardReport() + kb.key_down("ShiftLeft") + down = kb.printable_down("Slash", frozenset()) + assert down[0] & MODIFIER_BITS["ShiftLeft"] == 0 + assert down[2] == 0x38 + up = kb.printable_up() + assert up[0] & MODIFIER_BITS["ShiftLeft"] + hid, wheel = rfb_buttons_to_hid(0x01) assert hid == MouseButton.LEFT assert wheel == 0 @@ -283,9 +309,9 @@ def test_rfb_handshake_and_input(tmp_path): # PointerEvent: left button at 32,24 client.sendall(b"\x05\x01" + struct.pack("!HH", 32, 24)) deadline = time.monotonic() + 2 - while time.monotonic() < deadline and not hid.hid_key.called: + while time.monotonic() < deadline and not hid.hid_char.called: time.sleep(0.02) - hid.hid_key.assert_called_with("KeyA", True) + hid.hid_char.assert_called_with("KeyA", frozenset(), True) deadline = time.monotonic() + 2 while time.monotonic() < deadline and not hid.mouse_pointer.called: time.sleep(0.02) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py index af2a16bfb..73776ea6c 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py @@ -109,6 +109,25 @@ "ArrowLeft": 0x50, "ArrowDown": 0x51, "ArrowUp": 0x52, + "NumLock": 0x53, + "NumpadDivide": 0x54, + "NumpadMultiply": 0x55, + "NumpadSubtract": 0x56, + "NumpadAdd": 0x57, + "NumpadEnter": 0x58, + "Numpad1": 0x59, + "Numpad2": 0x5A, + "Numpad3": 0x5B, + "Numpad4": 0x5C, + "Numpad5": 0x5D, + "Numpad6": 0x5E, + "Numpad7": 0x5F, + "Numpad8": 0x60, + "Numpad9": 0x61, + "Numpad0": 0x62, + "NumpadDecimal": 0x63, + "IntlBackslash": 0x64, + "ContextMenu": 0x65, "ControlLeft": 0xE0, "ShiftLeft": 0xE1, "AltLeft": 0xE2, @@ -340,6 +359,30 @@ def _build_report(self) -> list[int]: report[2 + index] = keycode return report + def printable_down(self, key: str, combo_mods: frozenset[str]) -> list[int]: + """HID report for a character: combo modifiers, ignoring client Shift/AltGr.""" + keycode = KEYCODE_MAP.get(key) + if keycode is None: + raise ValueError(f"Unknown key: {key!r}") + shift_altgr = MODIFIER_BITS["ShiftLeft"] | MODIFIER_BITS["ShiftRight"] | MODIFIER_BITS["AltRight"] + modifier = self._modifier & ~shift_altgr + for mod in combo_mods: + bit = MODIFIER_BITS.get(mod) + if bit is None: + raise ValueError(f"Unknown modifier: {mod!r}") + modifier |= bit + report = [modifier, 0, keycode, 0, 0, 0, 0, 0] + slot = 3 + for pressed in self._pressed.values(): + if pressed == keycode or slot >= 8: + continue + report[slot] = pressed + slot += 1 + return report + + def printable_up(self) -> list[int]: + return self._build_report() + def char_to_report(self, ch: str) -> tuple[list[int], list[int]]: code = ord(ch) hid_code = CHAR_CODES.get(code) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_keymap.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_keymap.py new file mode 100644 index 000000000..c8f363c0d --- /dev/null +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_keymap.py @@ -0,0 +1,179 @@ +"""Map RFB/X11 keysyms to HID keys for a DUT keyboard layout. + +Printable characters are injected as HID combos for the configured DUT layout +so the VNC client's keyboard layout does not matter. HID cannot send Unicode; +the DUT OS must use the same layout as ``vnc_layout``. +""" + +from __future__ import annotations + +from .keyboard import CHAR_CODES, KEYCODE_MAP, MODIFIER_BITS, SHIFT_CHARS, _is_upper + +XK_UNICODE = 0x01000000 + +NONE: frozenset[str] = frozenset() +SHIFT: frozenset[str] = frozenset({"ShiftLeft"}) +ALTGR: frozenset[str] = frozenset({"AltRight"}) + +CharCombo = tuple[str, frozenset[str]] + +# Client-only modifiers used to produce a character keysym — do not forward. +_SWALLOWED_KEYSYMS: frozenset[int] = frozenset( + { + 0xFE03, # ISO_Level3_Shift (AltGr) + 0xFE04, # ISO_Level3_Latch + 0xFE05, # ISO_Level3_Lock + 0xFF7E, # Mode_switch + } +) + +_NAMED_KEYSYMS: dict[int, str] = { + 0xFF08: "Backspace", + 0xFF09: "Tab", + 0xFF0D: "Enter", + 0xFF1B: "Escape", + 0xFFFF: "Delete", + 0xFF50: "Home", + 0xFF51: "ArrowLeft", + 0xFF52: "ArrowUp", + 0xFF53: "ArrowRight", + 0xFF54: "ArrowDown", + 0xFF55: "PageUp", + 0xFF56: "PageDown", + 0xFF57: "End", + 0xFF63: "Insert", + 0xFFE1: "ShiftLeft", + 0xFFE2: "ShiftRight", + 0xFFE3: "ControlLeft", + 0xFFE4: "ControlRight", + 0xFFE5: "CapsLock", + 0xFFE7: "MetaLeft", + 0xFFE8: "MetaRight", + 0xFFE9: "AltLeft", + 0xFFEA: "AltRight", + 0xFFEB: "MetaLeft", + 0xFFEC: "MetaRight", + 0xFF7F: "NumLock", + 0xFF14: "ScrollLock", + 0xFF13: "Pause", + 0xFF61: "PrintScreen", + 0xFF67: "ContextMenu", + 0xFF8D: "NumpadEnter", + 0xFFAA: "NumpadMultiply", + 0xFFAB: "NumpadAdd", + 0xFFAD: "NumpadSubtract", + 0xFFAE: "NumpadDecimal", + 0xFFAF: "NumpadDivide", +} +for _i in range(12): + _NAMED_KEYSYMS[0xFFBE + _i] = f"F{_i + 1}" +for _i in range(10): + _NAMED_KEYSYMS[0xFFB0 + _i] = f"Numpad{_i}" + +_HID_TO_NAME = {code: name for name, code in KEYCODE_MAP.items() if name not in MODIFIER_BITS} + + +def normalize_keysym(keysym: int) -> int: + if keysym >= XK_UNICODE: + return keysym & 0xFFFFFF + return keysym + + +def is_swallowed_keysym(keysym: int) -> bool: + return keysym in _SWALLOWED_KEYSYMS + + +def named_key(keysym: int) -> str | None: + return _NAMED_KEYSYMS.get(keysym) + + +def normalize_layout(layout: str) -> str: + name = (layout or "us").strip().lower() + if name not in LAYOUTS: + supported = ", ".join(sorted(LAYOUTS)) + raise ValueError(f"unsupported vnc_layout {layout!r}; use one of: {supported}") + return name + + +def _us_layout() -> dict[int, CharCombo]: + table: dict[int, CharCombo] = {} + for code, hid in CHAR_CODES.items(): + if code in {9, 10}: + continue + name = _HID_TO_NAME.get(hid) + if name is None: + continue + mods = SHIFT if code in SHIFT_CHARS or _is_upper(code) else NONE + table[code] = (name, mods) + return table + + +def _es_layout() -> dict[int, CharCombo]: + table: dict[int, CharCombo] = { + ord(" "): ("Space", NONE), + **{ord(str(i)): (f"Digit{i}", NONE) for i in range(10)}, + ord("!"): ("Digit1", SHIFT), + ord('"'): ("Digit2", SHIFT), + 0x00B7: ("Digit3", SHIFT), # · + ord("$"): ("Digit4", SHIFT), + ord("%"): ("Digit5", SHIFT), + ord("&"): ("Digit6", SHIFT), + ord("/"): ("Digit7", SHIFT), + ord("("): ("Digit8", SHIFT), + ord(")"): ("Digit9", SHIFT), + ord("="): ("Digit0", SHIFT), + ord("'"): ("Minus", NONE), + ord("?"): ("Minus", SHIFT), + 0x00A1: ("Equal", NONE), # ¡ + 0x00BF: ("Equal", SHIFT), # ¿ + ord("@"): ("Digit2", ALTGR), + ord("#"): ("Digit3", ALTGR), + ord("~"): ("Digit4", ALTGR), + 0x20AC: ("Digit5", ALTGR), # € + 0x00AC: ("Digit6", ALTGR), # ¬ + ord("|"): ("Digit1", ALTGR), + ord("\\"): ("Backquote", ALTGR), + 0x00BA: ("Backquote", NONE), # º + 0x00AA: ("Backquote", SHIFT), # ª + ord("`"): ("BracketLeft", NONE), + ord("^"): ("BracketLeft", SHIFT), + ord("["): ("BracketLeft", ALTGR), + ord("+"): ("BracketRight", NONE), + ord("*"): ("BracketRight", SHIFT), + ord("]"): ("BracketRight", ALTGR), + 0x00B4: ("Quote", NONE), # ´ + 0x00A8: ("Quote", SHIFT), # ¨ + ord("{"): ("Quote", ALTGR), + 0x00E7: ("Backslash", NONE), # ç + 0x00C7: ("Backslash", SHIFT), # Ç + ord("}"): ("Backslash", ALTGR), + 0x00F1: ("Semicolon", NONE), # ñ + 0x00D1: ("Semicolon", SHIFT), # Ñ + ord("<"): ("IntlBackslash", NONE), + ord(">"): ("IntlBackslash", SHIFT), + ord(","): ("Comma", NONE), + ord(";"): ("Comma", SHIFT), + ord("."): ("Period", NONE), + ord(":"): ("Period", SHIFT), + ord("-"): ("Slash", NONE), + ord("_"): ("Slash", SHIFT), + } + for offset in range(26): + lower = ord("a") + offset + upper = ord("A") + offset + key = f"Key{chr(upper)}" + table[lower] = (key, NONE) + table[upper] = (key, SHIFT) + return table + + +LAYOUTS: dict[str, dict[int, CharCombo]] = { + "us": _us_layout(), + "es": _es_layout(), +} + + +def char_combo(keysym: int, layout: str = "us") -> CharCombo | None: + """Return (HID key, modifiers) to type the printable keysym on ``layout``.""" + table = LAYOUTS[normalize_layout(layout)] + return table.get(normalize_keysym(keysym)) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py index 3c20ec903..323036a56 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py @@ -18,6 +18,7 @@ from .frame_pump import FramePump from .mouse import MouseButton +from .vnc_keymap import char_combo, is_swallowed_keysym, named_key, normalize_layout logger = logging.getLogger(__name__) @@ -34,99 +35,22 @@ MSG_CLIENT_CUT_TEXT = 6 _CLIENT_IO_TIMEOUT = 30 -# X11 keysyms used by RFB -_KEYSYM_NAMED: dict[int, str] = { - 0xFF08: "Backspace", - 0xFF09: "Tab", - 0xFF0D: "Enter", - 0xFF1B: "Escape", - 0xFFFF: "Delete", - 0xFF50: "Home", - 0xFF51: "ArrowLeft", - 0xFF52: "ArrowUp", - 0xFF53: "ArrowRight", - 0xFF54: "ArrowDown", - 0xFF55: "PageUp", - 0xFF56: "PageDown", - 0xFF57: "End", - 0xFF63: "Insert", - 0xFFE1: "ShiftLeft", - 0xFFE2: "ShiftRight", - 0xFFE3: "ControlLeft", - 0xFFE4: "ControlRight", - 0xFFE7: "MetaLeft", - 0xFFE8: "MetaRight", - 0xFFE9: "AltLeft", - 0xFFEA: "AltRight", - 0xFFEB: "MetaLeft", - 0xFFEC: "MetaRight", - 0x0020: "Space", -} -for _i in range(12): - _KEYSYM_NAMED[0xFFBE + _i] = f"F{_i + 1}" - -_UNSHIFTED_PUNCT: dict[int, str] = { - ord(" "): "Space", - ord("`"): "Backquote", - ord("-"): "Minus", - ord("="): "Equal", - ord("["): "BracketLeft", - ord("]"): "BracketRight", - ord("\\"): "Backslash", - ord(";"): "Semicolon", - ord("'"): "Quote", - ord(","): "Comma", - ord("."): "Period", - ord("/"): "Slash", - # shifted variants still use the unshifted HID key; Shift is a separate event - ord("~"): "Backquote", - ord("!"): "Digit1", - ord("@"): "Digit2", - ord("#"): "Digit3", - ord("$"): "Digit4", - ord("%"): "Digit5", - ord("^"): "Digit6", - ord("&"): "Digit7", - ord("*"): "Digit8", - ord("("): "Digit9", - ord(")"): "Digit0", - ord("_"): "Minus", - ord("+"): "Equal", - ord("{"): "BracketLeft", - ord("}"): "BracketRight", - ord("|"): "Backslash", - ord(":"): "Semicolon", - ord('"'): "Quote", - ord("<"): "Comma", - ord(">"): "Period", - ord("?"): "Slash", -} - class HidTarget(Protocol): def hid_key(self, key: str, down: bool) -> None: ... + def hid_char(self, key: str, modifiers: frozenset[str], down: bool) -> None: ... def mouse_pointer(self, x: float, y: float, buttons: int, wheel: int = 0) -> None: ... -def keysym_to_key(keysym: int) -> str | None: - """Map an RFB/X11 keysym to a NanoKVM-USB key name.""" - if keysym in _KEYSYM_NAMED: - return _KEYSYM_NAMED[keysym] - if 0x61 <= keysym <= 0x7A: # a-z - return f"Key{chr(keysym).upper()}" - if 0x41 <= keysym <= 0x5A: # A-Z - return f"Key{chr(keysym)}" - if 0x30 <= keysym <= 0x39: - return f"Digit{chr(keysym)}" - if keysym in _UNSHIFTED_PUNCT: - return _UNSHIFTED_PUNCT[keysym] - if 0x20 <= keysym <= 0x7E: - ch = chr(keysym) - if ch.isalpha(): - return f"Key{ch.upper()}" - if ch.isdigit(): - return f"Digit{ch}" - return None +def keysym_to_key(keysym: int, layout: str = "us") -> str | None: + """Map a named RFB keysym, or the HID key of a printable on ``layout``.""" + named = named_key(keysym) + if named is not None: + return named + combo = char_combo(keysym, layout) + if combo is None: + return None + return combo[0] def rfb_buttons_to_hid(mask: int) -> tuple[int, int]: @@ -394,6 +318,7 @@ def __init__( password: str | None = None, tcp_port: int | None = None, tcp_bind: str = "127.0.0.1", + layout: str = "us", on_client: Callable[[], None] | None = None, ) -> None: self.path = path @@ -404,6 +329,7 @@ def __init__( self._password = password self._tcp_port = tcp_port self._tcp_bind = tcp_bind + self._layout = normalize_layout(layout) self._on_client = on_client self._stop = threading.Event() self._thread: threading.Thread | None = None @@ -617,14 +543,24 @@ def _read_client_message( # noqa: C901 return pf, requested def _handle_key(self, keysym: int, down: bool) -> None: - key = keysym_to_key(keysym) - if key is None: + if is_swallowed_keysym(keysym): + return + named = named_key(keysym) + if named is not None: + try: + self._hid.hid_key(named, down) + except Exception: + logger.debug("HID key event failed", exc_info=True) + return + combo = char_combo(keysym, self._layout) + if combo is None: logger.debug("unmapped RFB keysym 0x%04x", keysym) return + key, modifiers = combo try: - self._hid.hid_key(key, down) + self._hid.hid_char(key, modifiers, down) except Exception: - logger.debug("HID key event failed", exc_info=True) + logger.debug("HID character event failed", exc_info=True) def _handle_pointer(self, mask: int, x: int, y: int) -> None: nx = 0.0 if self._width <= 1 else max(0.0, min(1.0, x / (self._width - 1))) From 0451a485d6547579bc5bde6d7193c29215c5aede Mon Sep 17 00:00:00 2001 From: mparram Date: Thu, 24 Sep 2026 14:35:59 +0200 Subject: [PATCH 4/5] fix(nanokvm-usb): address VNC review feedback Extract DES, make VNC opt-in with a client cap, and pack 32bpp frames without extra RGB conversions. Co-authored-by: Cursor --- .../jumpstarter-driver-nanokvm-usb/README.md | 19 +- .../examples/exporter.yaml | 3 +- .../jumpstarter_driver_nanokvm_usb/des.py | 153 ++++++++++++ .../jumpstarter_driver_nanokvm_usb/device.py | 6 +- .../jumpstarter_driver_nanokvm_usb/driver.py | 13 +- .../driver_test.py | 101 +++++++- .../keyboard.py | 18 +- .../vnc_server.py | 223 ++++++------------ 8 files changed, 354 insertions(+), 182 deletions(-) create mode 100644 python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/des.py diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/README.md b/python/packages/jumpstarter-driver-nanokvm-usb/README.md index 80c3a4681..fa0e9ed63 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/README.md +++ b/python/packages/jumpstarter-driver-nanokvm-usb/README.md @@ -47,7 +47,8 @@ export: # vnc_layout: es # DUT keyboard layout (default us) # vnc_password: "secret" # vnc_tcp_port: 5900 - # vnc_tcp_bind: "0.0.0.0" # LAN; use vnc_password + # vnc_tcp_bind: "0.0.0.0" # LAN; VncAuth is not TLS + # vnc_max_clients: 2 ``` ### Config parameters @@ -62,20 +63,22 @@ export: | video_fps | Capture rate for `stream()` | int | no | 30 | | screen_width | Target screen width for relative mouse moves | int | no | 1920 | | screen_height | Target screen height for relative mouse moves | int | no | 1080 | -| vnc_enabled | Start an embedded RFB server (Unix socket child `vnc`) | bool | no | true | +| vnc_enabled | Start an embedded RFB server (Unix socket child `vnc`) | bool | no | false | | vnc_password | VNC password (VncAuth). Empty/None = no authentication | str | no | | | vnc_tcp_port | Also bind RFB TCP on the exporter (`None` = Unix socket only) | int | no | | | vnc_tcp_bind | Address for `vnc_tcp_port` (`127.0.0.1` or `0.0.0.0` for LAN) | str | no | 127.0.0.1 | | vnc_encrypt | Default noVNC `encrypt` URL flag | bool | no | false | | vnc_layout | DUT keyboard layout for VNC character injection (`us` or `es`) | str | no | us | +| vnc_max_clients | Maximum simultaneous RFB clients | int | no | 2 | ## Architecture -The driver is a composite with three child interfaces: +The driver is a composite with video and HID children, plus an optional VNC +child when `vnc_enabled` is true: 1. **video**: UVC snapshot capture and live frame streaming 2. **hid**: Keyboard and mouse control over USB serial -3. **vnc**: Unix-socket RFB endpoint (noVNC / any VNC client) +3. **vnc**: Unix-socket RFB endpoint (noVNC / any VNC client), opt-in Video stream and VNC share a single capture pump so `/dev/video*` is opened once. Keyboard and mouse events from the VNC client are translated to the same HID path as `hid`. @@ -125,6 +128,7 @@ with video.stream("stream") as stream: ## VNC +Set `vnc_enabled: true` to start the embedded RFB server (disabled by default). The exporter runs an RFB 3.8 server on a Unix socket. Jumpstarter tunnels that socket to the **client** (same pattern as QEMU): you do not need to be on the exporter host. Keyboard and mouse in the VNC client go to the NanoKVM-USB HID. @@ -153,13 +157,14 @@ with lease.drivers["nanokvm-usb"].session() as url: **TCP on the exporter** (optional): set `vnc_tcp_port` to bind RFB on the exporter host. Default `vnc_tcp_bind` is `127.0.0.1` (local viewers on that machine only). Use `vnc_tcp_bind: "0.0.0.0"` to accept LAN clients without a Jumpstarter tunnel. -Binding a non-loopback address without `vnc_password` exposes an unauthenticated -session on the network; set a password. +A non-loopback bind is reachable on the network even with `vnc_password`: VncAuth +is not TLS. `vnc_max_clients` defaults to 2. Printable keys are injected as HID combos for **`vnc_layout`** (the DUT OS keyboard), not the VNC client's layout. Default is `us`. Set `vnc_layout: es` if the DUT uses a Spanish keyboard so `@`, `ñ`, and AltGr characters land -correctly even when Remmina is using another layout. +correctly even when Remmina is using another layout. Only `us` and `es` are +built in; extra layout tables in `vnc_keymap.py` are welcome as pull requests. ## API reference diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml b/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml index 76170d891..e16d644da 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml +++ b/python/packages/jumpstarter-driver-nanokvm-usb/examples/exporter.yaml @@ -21,4 +21,5 @@ export: # vnc_layout: es # DUT keyboard layout (default us) # vnc_password: "secret" # vnc_tcp_port: 5900 - # vnc_tcp_bind: "0.0.0.0" # LAN; use vnc_password + # vnc_tcp_bind: "0.0.0.0" # LAN; VncAuth is not TLS + # vnc_max_clients: 2 diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/des.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/des.py new file mode 100644 index 000000000..fcf3366b1 --- /dev/null +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/des.py @@ -0,0 +1,153 @@ +"""RFB VncAuth DES (ECB, 8-byte blocks) with bit-reversed keys. + +RFB 3.8 VncAuth (RFC 6143 / RealVNC) encrypts a 16-byte challenge with DES +in ECB mode. The password is truncated or padded to 8 bytes and each byte is +bit-reversed before it is used as the DES key. + +The permutation tables and round function follow FIPS 46-3 (and the +public-domain d3des implementation). This is not PyDES. +""" + +from __future__ import annotations + +# fmt: off +_IP = ( + 58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, + 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, + 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, + 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7, +) +_FP = ( + 40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, + 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, + 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, + 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25, +) +_E = ( + 32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, 12, 13, + 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, 22, 23, 24, 25, + 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1, +) +_P = ( + 16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, + 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25, +) +_PC1 = ( + 57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, + 10, 2, 59, 51, 43, 35, 27, 19, 11, 3, 60, 52, 44, 36, + 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, 46, 38, 30, 22, + 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4, +) +_PC2 = ( + 14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, + 23, 19, 12, 4, 26, 8, 16, 7, 27, 20, 13, 2, + 41, 52, 31, 37, 47, 55, 30, 40, 51, 45, 33, 48, + 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32, +) +_SHIFTS = (1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1) +_SBOX = ( + ( + 14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, + 0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, + 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0, + 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13, + ), + ( + 15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, + 3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, + 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15, + 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9, + ), + ( + 10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, + 13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, + 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7, + 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12, + ), + ( + 7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, + 13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, + 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4, + 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14, + ), + ( + 2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, + 14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, + 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14, + 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3, + ), + ( + 12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, + 10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, + 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6, + 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13, + ), + ( + 4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, + 13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, + 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2, + 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12, + ), + ( + 13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, + 1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, + 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8, + 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11, + ), +) +# fmt: on + + +def _bitrev8(value: int) -> int: + value = ((value * 0x0202020202) & 0x010884422010) % 1023 + return value + + +def _permute(value: int, table: tuple[int, ...], nbits: int) -> int: + out = 0 + for bit in table: + out = (out << 1) | ((value >> (nbits - bit)) & 1) + return out + + +def _des_subkeys(key: bytes) -> list[int]: + k = int.from_bytes(key, "big") + cd = _permute(k, _PC1, 64) + c = (cd >> 28) & 0xFFFFFFF + d = cd & 0xFFFFFFF + keys = [] + for shift in _SHIFTS: + c = ((c << shift) | (c >> (28 - shift))) & 0xFFFFFFF + d = ((d << shift) | (d >> (28 - shift))) & 0xFFFFFFF + keys.append(_permute((c << 28) | d, _PC2, 56)) + return keys + + +def _des_f(r: int, subkey: int) -> int: + er = _permute(r, _E, 32) ^ subkey + s = 0 + for i in range(8): + chunk = (er >> (42 - 6 * i)) & 0x3F + row = ((chunk & 0x20) >> 4) | (chunk & 1) + col = (chunk >> 1) & 0xF + s = (s << 4) | _SBOX[i][row * 16 + col] + return _permute(s, _P, 32) + + +def des_ecb_encrypt(block: bytes, key: bytes) -> bytes: + """Encrypt one 8-byte block with DES-ECB (FIPS 46-3).""" + ip = _permute(int.from_bytes(block, "big"), _IP, 64) + left, right = ip >> 32, ip & 0xFFFFFFFF + for subkey in _des_subkeys(key): + left, right = right, left ^ _des_f(right, subkey) + preout = ((right & 0xFFFFFFFF) << 32) | (left & 0xFFFFFFFF) + return _permute(preout, _FP, 64).to_bytes(8, "big") + + +def vnc_auth_response(challenge: bytes, password: str) -> bytes: + """Encrypt the 16-byte VNC challenge with the bit-reversed DES password.""" + if len(challenge) != 16: + raise ValueError("VNC challenge must be 16 bytes") + key = password.encode("latin-1", "replace")[:8].ljust(8, b"\x00") + des_key = bytes(_bitrev8(b) for b in key) + return des_ecb_encrypt(challenge[:8], des_key) + des_ecb_encrypt(challenge[8:], des_key) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py index 32c519138..59227ce71 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/device.py @@ -62,6 +62,10 @@ def __init__( self._hid_lock = threading.RLock() self._pump: FramePump | None = None + @property + def pump(self) -> FramePump | None: + return self._pump + @property def is_connected(self) -> bool: return self._connected and self._serial.is_open @@ -142,7 +146,7 @@ def hid_key(self, key: str, down: bool) -> None: def hid_char(self, key: str, modifiers: frozenset[str], down: bool) -> None: """Type a printable character using DUT-layout HID modifiers (RFB).""" with self._hid_lock: - report = self._keyboard.printable_down(key, modifiers) if down else self._keyboard.printable_up() + report = self._keyboard.printable_down(key, modifiers) if down else self._keyboard.printable_up(key) self._send_keyboard(report) def release_all_keys(self) -> None: diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py index 662762cd5..5379e4223 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver.py @@ -122,7 +122,7 @@ async def stream_video(): state = {"gen": -1} def _next_frame() -> bytes: - pump = getattr(device, "_pump", None) + pump = device.pump if isinstance(pump, FramePump): gen = state["gen"] got = pump.wait_jpeg( @@ -267,12 +267,13 @@ class NanoKVMUSB(Composite): v4l2_ctl_executable: str | None = None screen_width: int = 1920 screen_height: int = 1080 - vnc_enabled: bool = True + vnc_enabled: bool = False vnc_password: str | None = None vnc_tcp_port: int | None = None vnc_tcp_bind: str = "127.0.0.1" vnc_encrypt: bool = False vnc_layout: str = "us" + vnc_max_clients: int = 2 _shared_device: NanoKVMUSBDevice = field(init=False, repr=False) _vnc_server: RfbServer | None = field(init=False, repr=False, default=None) @@ -323,7 +324,7 @@ def __post_init__(self): vnc_path = str(Path(self._vnc_dir) / "vnc.sock") self._vnc_server = RfbServer( vnc_path, - pump=lambda: self._shared_device._pump, + pump=lambda: self._shared_device.pump, hid=self._shared_device, width=self.video_width, height=self.video_height, @@ -331,6 +332,7 @@ def __post_init__(self): tcp_port=self.vnc_tcp_port, tcp_bind=self.vnc_tcp_bind, layout=self.vnc_layout, + max_clients=self.vnc_max_clients, on_client=self._shared_device.ensure_connected, ) self.children["vnc"] = NanoKVMUSBVNC( @@ -342,9 +344,10 @@ def __post_init__(self): super().__post_init__() if self._vnc_server is not None: - if self.vnc_tcp_port is not None and not is_loopback_bind(self.vnc_tcp_bind) and not self.vnc_password: + if self.vnc_tcp_port is not None and not is_loopback_bind(self.vnc_tcp_bind): self.logger.warning( - "RFB TCP bind %s:%s has no vnc_password; the session is reachable on the network", + "RFB TCP bind %s:%s is reachable on the network; " + "vnc_password (VncAuth) is not transport encryption", self.vnc_tcp_bind, self.vnc_tcp_port, ) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py index 397563b71..f7e671818 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py @@ -77,6 +77,7 @@ def test_nanokvm_usb_composite(mock_device): driver = NanoKVMUSB( serial_port="/dev/null", video_device=0, + vnc_enabled=True, ) driver._shared_device = mock_device video_child = driver.children["video"] @@ -247,8 +248,19 @@ def test_keysym_and_pointer_mapping(): down = kb.printable_down("Slash", frozenset()) assert down[0] & MODIFIER_BITS["ShiftLeft"] == 0 assert down[2] == 0x38 - up = kb.printable_up() + up = kb.printable_up("Slash") assert up[0] & MODIFIER_BITS["ShiftLeft"] + assert 0x38 not in up[2:] + + kb = KeyboardReport() + down_a = kb.printable_down("KeyA", frozenset()) + down_b = kb.printable_down("KeyB", frozenset()) + assert 0x04 in down_a[2:] + assert 0x04 in down_b[2:] + assert 0x05 in down_b[2:] + up_a = kb.printable_up("KeyA") + assert 0x04 not in up_a[2:] + assert 0x05 in up_a[2:] hid, wheel = rfb_buttons_to_hid(0x01) assert hid == MouseButton.LEFT @@ -258,11 +270,11 @@ def test_keysym_and_pointer_mapping(): def test_des_encrypt_nist_vector(): - from .vnc_server import _des_ecb_encrypt, vnc_auth_response + from .des import des_ecb_encrypt, vnc_auth_response key = bytes.fromhex("133457799BBCDFF1") plain = bytes.fromhex("0123456789ABCDEF") - assert _des_ecb_encrypt(plain, key) == bytes.fromhex("85E813540F0AB405") + assert des_ecb_encrypt(plain, key) == bytes.fromhex("85E813540F0AB405") response = vnc_auth_response(b"\x00" * 16, "secret") assert len(response) == 16 @@ -419,3 +431,86 @@ def test_rfb_tcp_handshake(tmp_path): finally: server.stop() pump.stop() + + +def test_pack_rgb_frame_bgrx_matches_rgbx(): + import numpy as np + + from .vnc_server import _bgrx_with_zero_pad, _default_pixel_format, pack_rgb_frame + + rgb = Image.new("RGB", (4, 2), color=(0, 0, 0)) + pixels = rgb.load() + assert pixels is not None + pixels[0, 0] = (255, 0, 0) + pixels[1, 0] = (0, 255, 0) + pixels[2, 0] = (0, 0, 255) + pixels[3, 0] = (1, 2, 3) + + pf = _default_pixel_format() + packed_rgb = pack_rgb_frame(rgb, pf) + rgbx = rgb.convert("RGBX") + packed_rgbx = pack_rgb_frame(rgbx, pf) + packed_rgba = pack_rgb_frame(rgb.convert("RGBA"), pf) + assert packed_rgbx == packed_rgb + assert packed_rgba == packed_rgb + assert packed_rgb[0:4] == bytes([0, 0, 255, 0]) + + arr = np.frombuffer(rgbx.tobytes(), dtype=np.uint8).reshape(2, 4, 4) + packed_bgrx = _bgrx_with_zero_pad(arr[:, :, [2, 1, 0, 3]]) + assert packed_bgrx == packed_rgb + + +def test_rfb_max_clients(tmp_path): + import socket + import time + + from .frame_pump import FramePump + from .vnc_server import RFB_VERSION, RfbServer, _recvexact + + jpeg = _jpeg_bytes(16, 16) + pump = FramePump(lambda: jpeg, fps=10) + pump.start() + sock_path = str(tmp_path / "vnc-max.sock") + server = RfbServer( + sock_path, + pump, + MagicMock(), + width=16, + height=16, + tcp_port=0, + tcp_bind="127.0.0.1", + max_clients=1, + ) + server.start() + first = None + second = None + try: + endpoint = server.tcp_endpoint + assert endpoint is not None + host, port = endpoint + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + try: + first = socket.create_connection((host, port), timeout=2) + break + except OSError: + time.sleep(0.01) + assert first is not None + first.settimeout(5) + assert _recvexact(first, 12) == RFB_VERSION + second = socket.create_connection((host, port), timeout=2) + second.settimeout(2) + try: + leftover = second.recv(12) + except ConnectionResetError: + leftover = b"" + assert leftover == b"" + first.close() + first = None + finally: + if first is not None: + first.close() + if second is not None: + second.close() + server.stop() + pump.stop() diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py index 73776ea6c..fea9a9319 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py @@ -351,8 +351,8 @@ def reset(self) -> list[int]: self._pressed.clear() return self._build_report() - def _build_report(self) -> list[int]: - report = [self._modifier, 0, 0, 0, 0, 0, 0, 0] + def _build_report(self, modifier: int | None = None) -> list[int]: + report = [self._modifier if modifier is None else modifier, 0, 0, 0, 0, 0, 0, 0] for index, keycode in enumerate(self._pressed.values()): if index >= MAX_KEYS: break @@ -364,6 +364,8 @@ def printable_down(self, key: str, combo_mods: frozenset[str]) -> list[int]: keycode = KEYCODE_MAP.get(key) if keycode is None: raise ValueError(f"Unknown key: {key!r}") + if key not in self._pressed and len(self._pressed) < MAX_KEYS: + self._pressed[key] = keycode shift_altgr = MODIFIER_BITS["ShiftLeft"] | MODIFIER_BITS["ShiftRight"] | MODIFIER_BITS["AltRight"] modifier = self._modifier & ~shift_altgr for mod in combo_mods: @@ -371,16 +373,10 @@ def printable_down(self, key: str, combo_mods: frozenset[str]) -> list[int]: if bit is None: raise ValueError(f"Unknown modifier: {mod!r}") modifier |= bit - report = [modifier, 0, keycode, 0, 0, 0, 0, 0] - slot = 3 - for pressed in self._pressed.values(): - if pressed == keycode or slot >= 8: - continue - report[slot] = pressed - slot += 1 - return report + return self._build_report(modifier) - def printable_up(self) -> list[int]: + def printable_up(self, key: str) -> list[int]: + self._pressed.pop(key, None) return self._build_report() def char_to_report(self, ch: str) -> tuple[list[int], list[int]]: diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py index 323036a56..b05668d1f 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py @@ -16,6 +16,7 @@ import numpy as np from PIL import Image +from .des import vnc_auth_response from .frame_pump import FramePump from .mouse import MouseButton from .vnc_keymap import char_combo, is_swallowed_keysym, named_key, normalize_layout @@ -92,149 +93,6 @@ def _recvexact(sock: socket.socket, n: int) -> bytes: return bytes(buf) -def _bitrev8(value: int) -> int: - value = ((value * 0x0202020202) & 0x010884422010) % 1023 - return value - - -def vnc_auth_response(challenge: bytes, password: str) -> bytes: - """Encrypt the 16-byte VNC challenge with the bit-reversed DES password.""" - if len(challenge) != 16: - raise ValueError("VNC challenge must be 16 bytes") - key = password.encode("latin-1", "replace")[:8].ljust(8, b"\x00") - des_key = bytes(_bitrev8(b) for b in key) - return _des_ecb_encrypt(challenge[:8], des_key) + _des_ecb_encrypt(challenge[8:], des_key) - - -# DES encrypt (ECB, one 8-byte block). Tables from FIPS 46-3 / public domain d3des. -# fmt: off -_IP = ( - 58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, - 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, - 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, - 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7, -) -_FP = ( - 40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, - 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, - 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, - 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25, -) -_E = ( - 32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, 12, 13, - 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, 22, 23, 24, 25, - 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1, -) -_P = ( - 16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, - 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25, -) -_PC1 = ( - 57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, - 10, 2, 59, 51, 43, 35, 27, 19, 11, 3, 60, 52, 44, 36, - 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, 46, 38, 30, 22, - 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4, -) -_PC2 = ( - 14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, - 23, 19, 12, 4, 26, 8, 16, 7, 27, 20, 13, 2, - 41, 52, 31, 37, 47, 55, 30, 40, 51, 45, 33, 48, - 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32, -) -_SHIFTS = (1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1) -_SBOX = ( - ( - 14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, - 0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, - 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0, - 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13, - ), - ( - 15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, - 3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, - 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15, - 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9, - ), - ( - 10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, - 13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, - 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7, - 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12, - ), - ( - 7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, - 13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, - 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4, - 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14, - ), - ( - 2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, - 14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, - 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14, - 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3, - ), - ( - 12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, - 10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, - 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6, - 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13, - ), - ( - 4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, - 13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, - 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2, - 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12, - ), - ( - 13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, - 1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, - 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8, - 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11, - ), -) -# fmt: on - - -def _permute(value: int, table: tuple[int, ...], nbits: int) -> int: - out = 0 - for bit in table: - out = (out << 1) | ((value >> (nbits - bit)) & 1) - return out - - -def _des_subkeys(key: bytes) -> list[int]: - k = int.from_bytes(key, "big") - cd = _permute(k, _PC1, 64) - c = (cd >> 28) & 0xFFFFFFF - d = cd & 0xFFFFFFF - keys = [] - for shift in _SHIFTS: - c = ((c << shift) | (c >> (28 - shift))) & 0xFFFFFFF - d = ((d << shift) | (d >> (28 - shift))) & 0xFFFFFFF - keys.append(_permute((c << 28) | d, _PC2, 56)) - return keys - - -def _des_f(r: int, subkey: int) -> int: - er = _permute(r, _E, 32) ^ subkey - s = 0 - for i in range(8): - chunk = (er >> (42 - 6 * i)) & 0x3F - row = ((chunk & 0x20) >> 4) | (chunk & 1) - col = (chunk >> 1) & 0xF - s = (s << 4) | _SBOX[i][row * 16 + col] - return _permute(s, _P, 32) - - -def _des_ecb_encrypt(block: bytes, key: bytes) -> bytes: - ip = _permute(int.from_bytes(block, "big"), _IP, 64) - left, right = ip >> 32, ip & 0xFFFFFFFF - for subkey in _des_subkeys(key): - left, right = right, left ^ _des_f(right, subkey) - preout = ((right & 0xFFFFFFFF) << 32) | (left & 0xFFFFFFFF) - return _permute(preout, _FP, 64).to_bytes(8, "big") - - def _default_pixel_format() -> dict[str, int]: return { "bits_per_pixel": 32, @@ -283,13 +141,49 @@ def _unpack_pixel_format(data: bytes) -> dict[str, int]: return dict(zip(keys, values, strict=True)) +def _is_default_32le(pf: dict[str, int]) -> bool: + return ( + pf.get("bits_per_pixel") == 32 + and pf.get("big_endian") == 0 + and pf.get("true_colour") == 1 + and pf.get("red_max") == 255 + and pf.get("green_max") == 255 + and pf.get("blue_max") == 255 + and pf.get("red_shift") == 16 + and pf.get("green_shift") == 8 + and pf.get("blue_shift") == 0 + ) + + +def _bgrx_with_zero_pad(arr: np.ndarray) -> bytes: + out = np.array(arr, dtype=np.uint8, copy=True, order="C") + out[:, :, 3] = 0 + return out.tobytes() + + +def _pack_default_32le(image: Image.Image) -> bytes: + """Pack to RFB raw 32bpp LE (B, G, R, 0 bytes in memory).""" + mode = image.mode + if mode in ("BGRX", "BGRA"): + arr = np.frombuffer(image.tobytes(), dtype=np.uint8).reshape(image.height, image.width, 4) + return _bgrx_with_zero_pad(arr) + if mode in ("RGBX", "RGBA"): + arr = np.frombuffer(image.tobytes(), dtype=np.uint8).reshape(image.height, image.width, 4) + return _bgrx_with_zero_pad(arr[:, :, [2, 1, 0, 3]]) + rgbx = image.convert("RGBX") + arr = np.frombuffer(rgbx.tobytes(), dtype=np.uint8).reshape(image.height, image.width, 4) + return _bgrx_with_zero_pad(arr[:, :, [2, 1, 0, 3]]) + + def pack_rgb_frame(image: Image.Image, pf: dict[str, int]) -> bytes: """Pack an RGB image into RFB raw pixels for ``pf``.""" - rgb = image.convert("RGB") bpp = pf["bits_per_pixel"] if bpp not in (16, 32): bpp = 32 pf = _default_pixel_format() + if bpp == 32 and _is_default_32le(pf): + return _pack_default_32le(image) + rgb = image.convert("RGB") arr = np.asarray(rgb, dtype=np.uint16 if bpp == 16 else np.uint32) r = arr[:, :, 0].astype(np.uint32) g = arr[:, :, 1].astype(np.uint32) @@ -319,6 +213,7 @@ def __init__( tcp_port: int | None = None, tcp_bind: str = "127.0.0.1", layout: str = "us", + max_clients: int = 2, on_client: Callable[[], None] | None = None, ) -> None: self.path = path @@ -330,6 +225,7 @@ def __init__( self._tcp_port = tcp_port self._tcp_bind = tcp_bind self._layout = normalize_layout(layout) + self._max_clients = max(1, int(max_clients)) self._on_client = on_client self._stop = threading.Event() self._thread: threading.Thread | None = None @@ -346,14 +242,14 @@ def start(self) -> None: os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True) listen = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) listen.bind(self.path) - listen.listen(4) + listen.listen(self._max_clients) listen.setblocking(False) self._listen = listen if self._tcp_port is not None: tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp.bind((self._tcp_bind, self._tcp_port)) - tcp.listen(4) + tcp.listen(self._max_clients) tcp.setblocking(False) self._tcp = tcp self._tcp_port = tcp.getsockname()[1] @@ -414,6 +310,12 @@ def _accept_loop(self) -> None: continue conn.settimeout(_CLIENT_IO_TIMEOUT) with self._clients_lock: + if len(self._clients) >= self._max_clients: + try: + conn.close() + except OSError: + pass + continue self._clients.append(conn) threading.Thread( target=self._session, @@ -478,6 +380,7 @@ def _handshake(self, conn: socket.socket) -> None: def _serve(self, conn: socket.socket) -> None: pf = _default_pixel_format() last_gen = -1 + last_pixels: bytes | None = None want_update = True conn.settimeout(0) while not self._stop.is_set(): @@ -493,9 +396,9 @@ def _serve(self, conn: socket.socket) -> None: if requested: want_update = True if want_update: - sent = self._send_frame(conn, pf, last_gen) + sent = self._send_frame(conn, pf, last_gen, last_pixels) if sent is not None: - last_gen = sent + last_gen, last_pixels = sent want_update = False def _read_client_message( # noqa: C901 @@ -571,7 +474,13 @@ def _handle_pointer(self, mask: int, x: int, y: int) -> None: except Exception: logger.debug("HID pointer event failed", exc_info=True) - def _send_frame(self, conn: socket.socket, pf: dict[str, int], last_gen: int) -> int | None: + def _send_frame( + self, + conn: socket.socket, + pf: dict[str, int], + last_gen: int, + last_pixels: bytes | None, + ) -> tuple[int, bytes | None] | None: pump = self._pump() if callable(self._pump) else self._pump if pump is None: return None @@ -580,17 +489,23 @@ def _send_frame(self, conn: socket.socket, pf: dict[str, int], last_gen: int) -> return None jpeg, gen = got try: - image = Image.open(BytesIO(jpeg)).convert("RGB") + image = Image.open(BytesIO(jpeg)) + image.load() except Exception: logger.debug("failed to decode JPEG for RFB", exc_info=True) - return gen + return gen, last_pixels if image.size != (self._width, self._height): image = image.resize((self._width, self._height)) pixels = pack_rgb_frame(image, pf) - header = struct.pack("!BxH", 0, 1) + struct.pack("!HHHHi", 0, 0, self._width, self._height, ENCODING_RAW) conn.settimeout(_CLIENT_IO_TIMEOUT) try: - conn.sendall(header + pixels) + if last_pixels is not None and pixels == last_pixels: + conn.sendall(struct.pack("!BxH", 0, 0)) + else: + header = struct.pack("!BxH", 0, 1) + struct.pack( + "!HHHHi", 0, 0, self._width, self._height, ENCODING_RAW + ) + conn.sendall(header + pixels) finally: conn.settimeout(0) - return gen + return gen, pixels From 2c34e4596eab9622690107b23e5e11c4d96860ec Mon Sep 17 00:00:00 2001 From: mparram Date: Thu, 24 Sep 2026 16:27:44 +0200 Subject: [PATCH 5/5] fix(nanokvm-usb): keep printable combo mods and honor RFB incremental Overlapping printable keys dropped Shift/AltGr, a bad JPEG cleared want_update so clients hung, and non-incremental FBURs could get 0-rect no-ops instead of a full refresh. Co-authored-by: Cursor --- .../driver_test.py | 121 ++++++++++++++++++ .../keyboard.py | 35 +++-- .../vnc_server.py | 30 +++-- 3 files changed, 167 insertions(+), 19 deletions(-) diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py index f7e671818..032f7a2c1 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/driver_test.py @@ -262,6 +262,16 @@ def test_keysym_and_pointer_mapping(): assert 0x04 not in up_a[2:] assert 0x05 in up_a[2:] + kb = KeyboardReport() + held = kb.printable_down("Digit2", SHIFT) + assert held[0] & MODIFIER_BITS["ShiftLeft"] + assert 0x1F in held[2:] + kb.printable_down("KeyA", frozenset()) + after = kb.printable_up("KeyA") + assert 0x04 not in after[2:] + assert 0x1F in after[2:] + assert after[0] & MODIFIER_BITS["ShiftLeft"] + hid, wheel = rfb_buttons_to_hid(0x01) assert hid == MouseButton.LEFT assert wheel == 0 @@ -514,3 +524,114 @@ def test_rfb_max_clients(tmp_path): second.close() server.stop() pump.stop() + + +def _rfb_connect(sock_path: str): + import socket + import struct + import time + + from .vnc_server import RFB_VERSION, _recvexact + + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not Path(sock_path).exists(): + time.sleep(0.01) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(5) + client.connect(sock_path) + assert _recvexact(client, 12) == RFB_VERSION + client.sendall(RFB_VERSION) + header = _recvexact(client, 2) + assert header == bytes([1, 1]) + client.sendall(bytes([1])) + assert struct.unpack("!I", _recvexact(client, 4))[0] == 0 + client.sendall(b"\x01") + server_init = _recvexact(client, 20) + width, height = struct.unpack("!HH", server_init[:4]) + name_len = struct.unpack("!I", _recvexact(client, 4))[0] + _recvexact(client, name_len) + return client, width, height + + +def _read_framebuffer_update(client, *, expect_pixels: bool): + import struct + + from .vnc_server import _recvexact + + header = _recvexact(client, 4) + assert header[0] == 0 + nrects = struct.unpack("!H", header[2:4])[0] + if expect_pixels: + assert nrects == 1 + rect = _recvexact(client, 12) + _x, _y, width, height, encoding = struct.unpack("!HHHHi", rect) + assert encoding == 0 + _recvexact(client, width * height * 4) + else: + assert nrects == 0 + return nrects + + +def test_rfb_skips_invalid_jpeg_and_sends_next_frame(tmp_path): + import struct + import threading + import time + + from .frame_pump import FramePump + from .vnc_server import RfbServer + + jpeg = _jpeg_bytes(16, 16) + ready = threading.Event() + + def capture() -> bytes: + if not ready.is_set(): + return b"not-a-jpeg" + return jpeg + + pump = FramePump(capture, fps=20) + pump.start() + sock_path = str(tmp_path / "vnc-bad-jpeg.sock") + server = RfbServer(sock_path, pump, MagicMock(), width=16, height=16) + server.start() + client = None + try: + client, width, height = _rfb_connect(sock_path) + assert (width, height) == (16, 16) + client.sendall(b"\x03\x01" + struct.pack("!HHHH", 0, 0, 16, 16)) + time.sleep(0.15) + ready.set() + _read_framebuffer_update(client, expect_pixels=True) + finally: + if client is not None: + client.close() + server.stop() + pump.stop() + + +def test_rfb_non_incremental_sends_full_frame(tmp_path): + import struct + + from .frame_pump import FramePump + from .vnc_server import RfbServer + + jpeg = _jpeg_bytes(16, 16) + pump = FramePump(lambda: jpeg, fps=20) + pump.start() + sock_path = str(tmp_path / "vnc-incremental.sock") + server = RfbServer(sock_path, pump, MagicMock(), width=16, height=16) + server.start() + client = None + try: + client, width, height = _rfb_connect(sock_path) + assert (width, height) == (16, 16) + client.sendall(b"\x03\x01" + struct.pack("!HHHH", 0, 0, 16, 16)) + _read_framebuffer_update(client, expect_pixels=True) + client.sendall(b"\x03\x01" + struct.pack("!HHHH", 0, 0, 16, 16)) + _read_framebuffer_update(client, expect_pixels=False) + client.sendall(b"\x03\x00" + struct.pack("!HHHH", 0, 0, 16, 16)) + _read_framebuffer_update(client, expect_pixels=True) + finally: + if client is not None: + client.close() + server.stop() + pump.stop() diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py index fea9a9319..56ca88caf 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/keyboard.py @@ -329,6 +329,7 @@ class KeyboardReport: def __init__(self) -> None: self._modifier = 0 self._pressed: dict[str, int] = {} + self._printable_mods: dict[str, frozenset[str]] = {} def key_down(self, code: str) -> list[int]: if is_modifier(code): @@ -337,6 +338,8 @@ def key_down(self, code: str) -> list[int]: keycode = KEYCODE_MAP.get(code) if keycode is not None and len(self._pressed) < MAX_KEYS: self._pressed[code] = keycode + if self._printable_mods: + return self._build_report(self._combo_modifier()) return self._build_report() def key_up(self, code: str) -> list[int]: @@ -344,11 +347,15 @@ def key_up(self, code: str) -> list[int]: self._modifier &= ~MODIFIER_BITS[code] else: self._pressed.pop(code, None) + self._printable_mods.pop(code, None) + if self._printable_mods: + return self._build_report(self._combo_modifier()) return self._build_report() def reset(self) -> list[int]: self._modifier = 0 self._pressed.clear() + self._printable_mods.clear() return self._build_report() def _build_report(self, modifier: int | None = None) -> list[int]: @@ -359,24 +366,36 @@ def _build_report(self, modifier: int | None = None) -> list[int]: report[2 + index] = keycode return report + def _combo_modifier(self) -> int: + shift_altgr = MODIFIER_BITS["ShiftLeft"] | MODIFIER_BITS["ShiftRight"] | MODIFIER_BITS["AltRight"] + modifier = self._modifier & ~shift_altgr + for mods in self._printable_mods.values(): + for mod in mods: + bit = MODIFIER_BITS.get(mod) + if bit is None: + raise ValueError(f"Unknown modifier: {mod!r}") + modifier |= bit + return modifier + def printable_down(self, key: str, combo_mods: frozenset[str]) -> list[int]: """HID report for a character: combo modifiers, ignoring client Shift/AltGr.""" keycode = KEYCODE_MAP.get(key) if keycode is None: raise ValueError(f"Unknown key: {key!r}") - if key not in self._pressed and len(self._pressed) < MAX_KEYS: - self._pressed[key] = keycode - shift_altgr = MODIFIER_BITS["ShiftLeft"] | MODIFIER_BITS["ShiftRight"] | MODIFIER_BITS["AltRight"] - modifier = self._modifier & ~shift_altgr for mod in combo_mods: - bit = MODIFIER_BITS.get(mod) - if bit is None: + if mod not in MODIFIER_BITS: raise ValueError(f"Unknown modifier: {mod!r}") - modifier |= bit - return self._build_report(modifier) + if key not in self._pressed and len(self._pressed) < MAX_KEYS: + self._pressed[key] = keycode + if key in self._pressed: + self._printable_mods[key] = combo_mods + return self._build_report(self._combo_modifier()) def printable_up(self, key: str) -> list[int]: self._pressed.pop(key, None) + self._printable_mods.pop(key, None) + if self._printable_mods: + return self._build_report(self._combo_modifier()) return self._build_report() def char_to_report(self, ch: str) -> tuple[list[int], list[int]]: diff --git a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py index b05668d1f..3ac73e4fd 100644 --- a/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py +++ b/python/packages/jumpstarter-driver-nanokvm-usb/jumpstarter_driver_nanokvm_usb/vnc_server.py @@ -382,6 +382,7 @@ def _serve(self, conn: socket.socket) -> None: last_gen = -1 last_pixels: bytes | None = None want_update = True + force_full = True conn.settimeout(0) while not self._stop.is_set(): try: @@ -392,18 +393,22 @@ def _serve(self, conn: socket.socket) -> None: result = self._read_client_message(conn, pf) if result is None: break - pf, requested = result + pf, requested, incremental = result if requested: want_update = True + if not incremental: + force_full = True if want_update: - sent = self._send_frame(conn, pf, last_gen, last_pixels) + sent = self._send_frame(conn, pf, last_gen, last_pixels, force_full) if sent is not None: - last_gen, last_pixels = sent - want_update = False + last_gen, last_pixels, did_send = sent + if did_send: + want_update = False + force_full = False def _read_client_message( # noqa: C901 self, conn: socket.socket, pf: dict[str, int] - ) -> tuple[dict[str, int], bool] | None: + ) -> tuple[dict[str, int], bool, bool] | None: try: header = conn.recv(1) except BlockingIOError: @@ -412,6 +417,7 @@ def _read_client_message( # noqa: C901 return None msg = header[0] requested = False + incremental = True conn.settimeout(_CLIENT_IO_TIMEOUT) try: if msg == MSG_SET_PIXEL_FORMAT: @@ -423,7 +429,8 @@ def _read_client_message( # noqa: C901 if count: _recvexact(conn, 4 * count) elif msg == MSG_FB_UPDATE_REQUEST: - _recvexact(conn, 9) + payload = _recvexact(conn, 9) + incremental = payload[0] != 0 requested = True elif msg == MSG_KEY_EVENT: payload = _recvexact(conn, 7) @@ -443,7 +450,7 @@ def _read_client_message( # noqa: C901 logger.debug("ignoring unknown RFB client message %s", msg) finally: conn.settimeout(0) - return pf, requested + return pf, requested, incremental def _handle_key(self, keysym: int, down: bool) -> None: if is_swallowed_keysym(keysym): @@ -480,7 +487,8 @@ def _send_frame( pf: dict[str, int], last_gen: int, last_pixels: bytes | None, - ) -> tuple[int, bytes | None] | None: + force_full: bool, + ) -> tuple[int, bytes | None, bool] | None: pump = self._pump() if callable(self._pump) else self._pump if pump is None: return None @@ -493,13 +501,13 @@ def _send_frame( image.load() except Exception: logger.debug("failed to decode JPEG for RFB", exc_info=True) - return gen, last_pixels + return gen, last_pixels, False if image.size != (self._width, self._height): image = image.resize((self._width, self._height)) pixels = pack_rgb_frame(image, pf) conn.settimeout(_CLIENT_IO_TIMEOUT) try: - if last_pixels is not None and pixels == last_pixels: + if not force_full and last_pixels is not None and pixels == last_pixels: conn.sendall(struct.pack("!BxH", 0, 0)) else: header = struct.pack("!BxH", 0, 1) + struct.pack( @@ -508,4 +516,4 @@ def _send_frame( conn.sendall(header + pixels) finally: conn.settimeout(0) - return gen, pixels + return gen, pixels, True