From c8bc8cf06bfaba9e6a20f9840952c7a3a617c9cf Mon Sep 17 00:00:00 2001 From: pascd Date: Thu, 12 Nov 2026 20:23:20 +0000 Subject: [PATCH] Inference/train tabs improvements, Review data view and test suite - Full configuration template for LfD data acquisition; DataNormalizer removed - start_demo moved to its own module; code structure refactored - ActionDispatcher and ObsBuilder: stricter message handling and validation - Inference: obs node spun with a SingleThreadedExecutor to hold 20 Hz - robomimic environment installs OpenCV for inference - relay-d-train: dataset paths made absolute so a relative --folder works - AppAPI: delete_demo() rejects a recorded demo like the GUI - Review: every camera as a card, every recorded dataset as a channel, JSON messages decoded and shown readably - Tests for post-processing, recording, settings, training launcher, background processing, env setup, review, run and spin --- .github/workflows/ci.yml | 62 +- CHANGELOG.md | 387 +- CONTRIBUTING.md | 14 +- LICENSE | 5 - README.md | 177 +- THIRD_PARTY_LICENSES.md | 8 +- debug/equivalence_test.py | 318 - debug/replay_parity.py | 177 - docs/diagrams/architecture.md | 16 +- docs/reporting_template.md | 2 +- pyproject.toml | 22 +- relay_d/acquisition/api/api.py | 209 +- relay_d/acquisition/convert_cli.py | 149 + relay_d/acquisition/qt_app/app.py | 184 +- relay_d/acquisition/qt_app/ctrl_c.py | 15 + relay_d/acquisition/qt_app/first_run.py | 20 + relay_d/acquisition/qt_app/resources.qrc | 6 - .../resources/img/relayd_logo_simple.png | Bin 3895259 -> 11507 bytes relay_d/acquisition/qt_app/resources_rc.py | 259021 +-------------- relay_d/acquisition/qt_app/settings_apply.py | 32 + .../qt_app/ui/config/builder/__init__.py | 0 .../qt_app/ui/config/builder/builder_page.py | 629 + .../qt_app/ui/config/builder/config_check.py | 173 + .../qt_app/ui/config/builder/draft.py | 220 + .../qt_app/ui/config/builder/ros_catalog.py | 107 + .../qt_app/ui/config/builder/urdf_limits.py | 37 + .../qt_app/ui/config/config_page.py | 135 +- .../ui/config/templates}/config_template.yaml | 379 +- .../postprocess_config_template.yaml | 35 + .../qt_app/ui/config/yaml_parser.py | 170 +- .../qt_app/ui/containers/mainWindow.ui | 660 +- .../qt_app/ui/containers/mainWindow_ui.py | 354 +- .../qt_app/ui/containers/topic_containers.py | 237 +- relay_d/acquisition/qt_app/ui/page_style.py | 33 + .../qt_app/ui/postprocess/h5_file_parser.py | 55 - .../qt_app/ui/postprocess/lerobot_writer.py | 684 + .../qt_app/ui/postprocess/offline_config.py | 107 + .../postprocess/postprocess_config_loader.py | 257 +- .../postprocess_configs/robomimic.yaml | 259 - .../ui/postprocess/postprocess_options.py | 143 + .../qt_app/ui/postprocess/postprocess_page.py | 699 +- .../postprocess/robomimic_config_handler.py | 35 +- .../ui/postprocess/yaml_driven_converter.py | 1479 +- .../qt_app/ui/record/container_manager.py | 153 +- .../qt_app/ui/record/container_visualizer.py | 72 +- .../qt_app/ui/record/data_acquisition_node.py | 374 - .../qt_app/ui/record/data_normalization.py | 114 - .../qt_app/ui/record/data_recorder.py | 943 +- .../qt_app/ui/record/demo_start.py | 15 + .../qt_app/ui/record/message_encoding.py | 199 + .../qt_app/ui/record/record_page.py | 435 +- .../qt_app/ui/record/recording_controller.py | 245 + .../qt_app/ui/record/topic_subscribers.py | 207 +- .../qt_app/ui/review/data_player.py | 13 +- .../ui/review/demo_playback_reconstructor.py | 12 +- .../qt_app/ui/review/demo_validator.py | 49 +- .../ui/review/playback_process_manager.py | 9 +- .../qt_app/ui/review/review_page.py | 447 +- relay_d/acquisition/qt_app/ui/run/__init__.py | 0 relay_d/acquisition/qt_app/ui/run/run_page.py | 231 + .../qt_app/ui/settings/config_manager.py | 50 +- .../qt_app/ui/settings/settings_page.py | 335 +- .../qt_app/ui/styles/relayd_dark.qss | 148 +- .../qt_app/ui/styles/relayd_light.qss | 148 +- .../acquisition/qt_app/ui/train/__init__.py | 0 .../acquisition/qt_app/ui/train/train_page.py | 471 + .../acquisition/qt_app/utils/background.py | 36 + .../qt_app/utils/default_configs.py | 41 + .../qt_app/utils/interface_menu.py | 263 - .../qt_app/utils/stream_builder.py | 245 +- relay_d/dispatch/__init__.py | 12 +- relay_d/dispatch/action_dispatcher.py | 525 +- relay_d/dispatch/api.py | 255 +- .../action_config_template.yaml | 431 +- .../config_templates/obs_config_template.yaml | 140 +- relay_d/dispatch/inference_node.py | 101 +- relay_d/dispatch/lerobot_runner.py | 239 + relay_d/dispatch/model_runner.py | 1215 - relay_d/dispatch/obs_builder.py | 470 +- relay_d/dispatch/reexec.py | 55 + relay_d/dispatch/robomimic_runner.py | 71 +- .../templates/inference_script_template.py | 91 +- relay_d/training/__init__.py | 4 + relay_d/training/envsetup.py | 176 + relay_d/training/launcher.py | 494 + relay_d/training/runs.py | 106 + relay_d/training/train_cli.py | 117 + relay_d/utils/coloring_logger.py | 152 +- relay_d/utils/numerics.py | 118 + relay_d/utils/qos.py | 25 + relay_d/utils/ros_media_codec.py | 17 +- relay_d/utils/tf_filter.py | 42 + relay_d/utils/tf_topic.py | 10 +- relay_d/utils/user_settings.py | 251 + scripts/extract_joint_limits.py | 105 +- scripts/reprocess_recorded_data.py | 6 +- tests/conftest.py | 84 + tests/helpers.py | 73 + tests/test_action_dispatcher.py | 428 + tests/test_app_api.py | 172 + tests/test_builder_page.py | 81 + tests/test_config_draft.py | 114 + tests/test_config_page_tabs.py | 46 + tests/test_conversion_fixes.py | 225 + tests/test_conversion_paths.py | 198 + tests/test_convert_cli.py | 108 + tests/test_ctrl_c.py | 38 + tests/test_data_recorder_save.py | 139 + tests/test_default_configs.py | 17 + tests/test_demo_validator.py | 23 + tests/test_dispatch_reexec.py | 35 + tests/test_dispatch_templates.py | 77 + tests/test_envsetup.py | 122 + tests/test_first_run_folder.py | 19 + tests/test_format_buttons.py | 54 + tests/test_inference_session.py | 281 + tests/test_lerobot_conversion.py | 221 + tests/test_lerobot_runner.py | 237 + tests/test_lerobot_writer.py | 186 + tests/test_logger.py | 62 + tests/test_numerics.py | 119 + tests/test_offline_config.py | 65 + tests/test_postprocess_options.py | 148 + tests/test_postprocess_page.py | 146 + tests/test_precheck.py | 139 + tests/test_recorder_loop.py | 220 + tests/test_recording_controller.py | 224 + tests/test_review_page.py | 127 + tests/test_ros_catalog.py | 50 + tests/test_run_page.py | 91 + tests/test_settings_apply.py | 31 + tests/test_settings_consumers.py | 138 + tests/test_settings_live.py | 41 + tests/test_settings_page.py | 88 + tests/test_spin.py | 89 + tests/test_tf_filter.py | 21 + tests/test_train_cli.py | 58 + tests/test_train_page.py | 196 + tests/test_training_launcher.py | 308 + tests/test_urdf_limits.py | 21 + tests/test_user_settings.py | 91 + 141 files changed, 21027 insertions(+), 262618 deletions(-) delete mode 100644 debug/equivalence_test.py delete mode 100644 debug/replay_parity.py create mode 100644 relay_d/acquisition/convert_cli.py create mode 100644 relay_d/acquisition/qt_app/ctrl_c.py create mode 100644 relay_d/acquisition/qt_app/first_run.py create mode 100644 relay_d/acquisition/qt_app/settings_apply.py create mode 100644 relay_d/acquisition/qt_app/ui/config/builder/__init__.py create mode 100644 relay_d/acquisition/qt_app/ui/config/builder/builder_page.py create mode 100644 relay_d/acquisition/qt_app/ui/config/builder/config_check.py create mode 100644 relay_d/acquisition/qt_app/ui/config/builder/draft.py create mode 100644 relay_d/acquisition/qt_app/ui/config/builder/ros_catalog.py create mode 100644 relay_d/acquisition/qt_app/ui/config/builder/urdf_limits.py rename {config => relay_d/acquisition/qt_app/ui/config/templates}/config_template.yaml (58%) create mode 100644 relay_d/acquisition/qt_app/ui/config/templates/postprocess_config_template.yaml create mode 100644 relay_d/acquisition/qt_app/ui/page_style.py delete mode 100644 relay_d/acquisition/qt_app/ui/postprocess/h5_file_parser.py create mode 100644 relay_d/acquisition/qt_app/ui/postprocess/lerobot_writer.py create mode 100644 relay_d/acquisition/qt_app/ui/postprocess/offline_config.py delete mode 100644 relay_d/acquisition/qt_app/ui/postprocess/postprocess_configs/robomimic.yaml create mode 100644 relay_d/acquisition/qt_app/ui/postprocess/postprocess_options.py delete mode 100644 relay_d/acquisition/qt_app/ui/record/data_acquisition_node.py delete mode 100644 relay_d/acquisition/qt_app/ui/record/data_normalization.py create mode 100644 relay_d/acquisition/qt_app/ui/record/demo_start.py create mode 100644 relay_d/acquisition/qt_app/ui/record/message_encoding.py create mode 100644 relay_d/acquisition/qt_app/ui/record/recording_controller.py create mode 100644 relay_d/acquisition/qt_app/ui/run/__init__.py create mode 100644 relay_d/acquisition/qt_app/ui/run/run_page.py create mode 100644 relay_d/acquisition/qt_app/ui/train/__init__.py create mode 100644 relay_d/acquisition/qt_app/ui/train/train_page.py create mode 100644 relay_d/acquisition/qt_app/utils/background.py create mode 100644 relay_d/acquisition/qt_app/utils/default_configs.py delete mode 100644 relay_d/acquisition/qt_app/utils/interface_menu.py create mode 100644 relay_d/dispatch/lerobot_runner.py delete mode 100644 relay_d/dispatch/model_runner.py create mode 100644 relay_d/dispatch/reexec.py create mode 100644 relay_d/training/__init__.py create mode 100644 relay_d/training/envsetup.py create mode 100644 relay_d/training/launcher.py create mode 100644 relay_d/training/runs.py create mode 100644 relay_d/training/train_cli.py create mode 100644 relay_d/utils/numerics.py create mode 100644 relay_d/utils/qos.py create mode 100644 relay_d/utils/tf_filter.py create mode 100644 relay_d/utils/user_settings.py create mode 100644 tests/conftest.py create mode 100644 tests/helpers.py create mode 100644 tests/test_action_dispatcher.py create mode 100644 tests/test_app_api.py create mode 100644 tests/test_builder_page.py create mode 100644 tests/test_config_draft.py create mode 100644 tests/test_config_page_tabs.py create mode 100644 tests/test_conversion_fixes.py create mode 100644 tests/test_conversion_paths.py create mode 100644 tests/test_convert_cli.py create mode 100644 tests/test_ctrl_c.py create mode 100644 tests/test_data_recorder_save.py create mode 100644 tests/test_default_configs.py create mode 100644 tests/test_demo_validator.py create mode 100644 tests/test_dispatch_reexec.py create mode 100644 tests/test_dispatch_templates.py create mode 100644 tests/test_envsetup.py create mode 100644 tests/test_first_run_folder.py create mode 100644 tests/test_format_buttons.py create mode 100644 tests/test_inference_session.py create mode 100644 tests/test_lerobot_conversion.py create mode 100644 tests/test_lerobot_runner.py create mode 100644 tests/test_lerobot_writer.py create mode 100644 tests/test_logger.py create mode 100644 tests/test_numerics.py create mode 100644 tests/test_offline_config.py create mode 100644 tests/test_postprocess_options.py create mode 100644 tests/test_postprocess_page.py create mode 100644 tests/test_precheck.py create mode 100644 tests/test_recorder_loop.py create mode 100644 tests/test_recording_controller.py create mode 100644 tests/test_review_page.py create mode 100644 tests/test_ros_catalog.py create mode 100644 tests/test_run_page.py create mode 100644 tests/test_settings_apply.py create mode 100644 tests/test_settings_consumers.py create mode 100644 tests/test_settings_live.py create mode 100644 tests/test_settings_page.py create mode 100644 tests/test_spin.py create mode 100644 tests/test_tf_filter.py create mode 100644 tests/test_train_cli.py create mode 100644 tests/test_train_page.py create mode 100644 tests/test_training_launcher.py create mode 100644 tests/test_urdf_limits.py create mode 100644 tests/test_user_settings.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c45791..bc22393 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,17 +1,20 @@ name: CI -# NOTE: This workflow deliberately does NOT install or exercise ROS 2 (rclpy, -# tf2_ros, cv_bridge, ROS 2 message packages). GitHub-hosted runners don't -# have ROS 2 preinstalled, and this project has no automated test suite for -# those code paths yet (see CONTRIBUTING.md). This workflow only sanity-checks -# that the pip-installable subset of the package builds and installs cleanly, -# plus a non-blocking lint report. +# It does: +# - sanity-check that the pip-installable subset of the package builds and installs, +# - run the ROS-free test suite (config parsing, dataset conversion to robomimic +# and LeRobot formats, the headless converter, ...) on a plain runner, +# - run the whole suite again inside the ros:jazzy image, so the tests that need +# rclpy (action dispatcher, recorder, AppAPI, inference session) run too: they +# skip themselves on the plain runner. No robot, no live topics: unit tests only. +# - produce a non-blocking lint report. +# Only the `main` branch is covered. on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] workflow_dispatch: jobs: @@ -38,6 +41,45 @@ jobs: - name: Install the package (pip-installable dependencies only) run: pip install . + test: + name: Tests (ROS-free) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # The tests need neither ROS nor torch. They do use Qt (offscreen) and OpenCV for the pages and the + # LeRobot viewer: without them those tests skip themselves and CI stays green without running them. + - name: Install test dependencies + run: python -m pip install pytest numpy h5py pyyaml pyarrow av opencv-python-headless pyqt5 pyqtgraph + + - name: Run tests + run: python -m pytest -q + + test-ros: + name: Tests (ROS 2 Jazzy) + runs-on: ubuntu-latest + container: ros:jazzy + steps: + - uses: actions/checkout@v4 + + # Qt and OpenCV from apt, built against the system numpy that ROS uses; pip adds only what apt lacks. + - name: Install test dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends python3-pip python3-pytest python3-opencv python3-pyqt5 ros-jazzy-example-interfaces + pip install --break-system-packages h5py "pyarrow<21" av pyqtgraph # pyarrow 21+ needs NumPy 2 + + - name: Run tests + shell: bash + run: | + source /opt/ros/jazzy/setup.bash + python3 -m pytest -q + lint: name: Lint (non-blocking) runs-on: ubuntu-latest @@ -52,5 +94,9 @@ jobs: - name: Install ruff run: python -m pip install ruff + # An undefined name only crashes when that code path runs, and the ROS / Qt pages are not run in CI. + - name: Undefined names (fails the build) + run: ruff check --select F821 --exclude relay_d/acquisition/qt_app/resources_rc.py . + - name: Run ruff (report-only, does not fail the build) run: ruff check --exit-zero . diff --git a/CHANGELOG.md b/CHANGELOG.md index c1eaa43..56c034d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,339 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Upgrade notes (read first) +- **Re-convert your datasets and retrain.** An absolute action is now the target of the next step (`action[t] = value[t+1]`), see *Changed*. Datasets converted before this release taught the policy to hold still. +- **LeRobot environment:** press **Set up lerobot environment** (Train tab) again: it adds `lerobot[smolvla]`. The `.[lerobot]` extra now needs `pyarrow<21` (pyarrow 21+ needs NumPy 2, ROS Jazzy has 1.26). +- **`relay_d.dispatch.DispatcherHelpers` is gone:** use `save_dispatch_templates(output_dir)`. +- **`lfd-inference-node --hz`** defaults to the rate the LeRobot policy was trained at (its dataset fps), no longer the `infer_hz` setting (10 Hz). robomimic checkpoints still use the setting: pass `--hz `. +- **Action configs copied from the template:** the `robotiq_gripper` entry must map `position: 0`, not `7` (see *Fixed*). +- **Relay-D no longer normalizes** (robomimic and LeRobot already normalize and un-normalize with the dataset's stats). Remove + `normalize` / `normalized_range` / `limits` from acquisition streams and `obs_config.yaml`, and `denormalize` / `normalized_range` / + `apply_limits` from `action_config.yaml`; configs that still have them are refused. `limits` on an action entry now only **clamps** + the real value: an old action config with `limits` and no `denormalize` key (it defaulted to true) used to rescale [-1, 1] and now + clamps, so it is only right for a policy retrained on this release. +- **Removed config keys now stop with "'' was removed: ...; "** instead of being ignored. Acquisition config: + `output_map` and `fill_default` (also refused at Start: they changed the recorded data), `transfer_rate`, `output.prefix_path`, + `config.data`, `metadata.*`, and the old post-process keys (`training_model`, `*_frac`, `lerobot`). `postprocess_config.yaml`: + `robomimic.metadata`, `robomimic.training_model`, `splits.test_frac`. `action_config.yaml`: `command:` entries, `dedupe`, and the + `scale` / `clip` / `vacuum` / `magnetic` transforms. `obs_config.yaml`: `static` inputs. `recording_frequency` is required in the + acquisition YAML, and the `rec_default_frequency` / `rec_default_sync_*` settings are gone (the YAML is the only source). +- **Acquisition template:** the streams that read inputs which are only commented examples (`tf_object`, `ee_pose`, ...) are + commented out too; before, a "Generate Defaults" config failed its own check and inference refused its obs_config. +- **Action template:** only `move_arm_joint` and `robotiq_gripper` are live (`move_eef` became a commented example); before, a copied + template commanded the arm twice per tick, as a pose and as joint deltas. +- **New folder names:** recording sessions end in `_Recording`, Post-process outputs in `_Robomimic` / `_Lerobot`. Existing folders keep their names. + +### Added +- **Review shows everything that was recorded.** + - **Images:** every camera of the demo is shown side by side (raw or compressed), each in its own card titled with its name and resolution. The frame fills the card, keeps its aspect ratio and rescales with the window. One slider drives them all, with the frame number and the time since the start. Before, only the first camera was shown. + - **Every other recorded dataset gets its own channel too:** + - raw input data (e.g. `joint_states/joint_positions`, with the joint names as legend), `timestamps` and `timestamp_spreads` are plotted; + - messages stored as JSON (types without their own encoder, e.g. `std_msgs/Float64MultiArray`) are decoded: their numeric fields are plotted over the demo, and the sample you pick is shown as indented JSON, with readable field names and number lists on one line; + - the cursor of these plots follows the picked sample. +- **`AppAPI.delete_demo(path)`** deletes a recorded demo and drops it from the session, like the GUI's Review > Delete / Reject Demo (same `delete_demo_file()`). A script can now reject a bad demo (e.g. a failed scripted run) without `shutdown()` then converting a missing file. +- **SmolVLA**, a third LeRobot policy (Train tab, `relay-d-train --policy smolvla`). It is a 450M vision-language-action model fine-tuned from the pretrained `lerobot/smolvla_base` (`--policy.path`, downloaded once), not trained from scratch, + and it follows the dataset's task sentence. The dataset's cameras (at most 3) are renamed to the model's `camera1..3` with `--rename_map`, which LeRobot keeps in the checkpoint. "Set up lerobot environment" installs + `lerobot[smolvla]` (`transformers`, `num2words`). Measured on an RTX 4060 Laptop (8 GB): the fine-tune peaks at about 4.2 GB with the default batch size, and inference needs under 1 GB. +- **Task sentence at inference:** `lfd-inference-node --task "..."` / `InferenceSession(task=...)`, by default the training dataset's task (`meta/tasks.parquet`). A language policy without a task stops at startup. For a fine-tuned checkpoint + the state size and cameras are taken from the training dataset, because the checkpoint config keeps the pretrained model's ones. +- **Safe stop:** a command or topic entry of `action_config.yaml` can set `stop: [...]` (e.g. `[0, 0, 0]` for a velocity), sent once when inference stops sending: inputs go stale, the model fails, `on_timeout: raise`, Ctrl+C, the end of + `run()`, or `close()`. Before, the node just went quiet and a controller without a watchdog kept the last command. A broken `stop` (wrong length, a mapping that does not fit, set on an `action: true` entry) stops startup. +- **Recorded-rate check:** a demo recorded more than 10% off `recording_frequency` (a PC that could not keep up) fails the after-recording check and is left out of robomimic and LeRobot datasets: the policy runs at that frequency, so + it would move faster than shown. +- **CI runs the ROS-dependent tests** in a `ros:jazzy` container job (dispatcher, recorder, `AppAPI`, inference session). `pytest` also runs locally with ROS sourced (`pyproject.toml` disables the ROS pytest plugins). +- Verified end to end against LeRobot 0.6.1: `LeRobotDataset` loads a Relay-D dataset (frames, RGB video, `action[t] = state[t+1]`, ACT-style action chunks via `delta_timestamps`), `lerobot_train` trains on it with the Train tab's + command (ACT and SmolVLA), and `LeRobotPolicyRunner` runs the checkpoint, with state and action normalized by the dataset's own stats. + +### Changed +- **Training actions are next-step targets:** an absolute action (a stream not in `delta_actions`) is the stream's value at t+1, as a delta already looked one step ahead; it was the value at t, so a state stream used as an action + taught the policy to stand still. A stream that is both `normalize: true` and in `delta_actions` is refused (conversion and the config check): the dispatcher cannot turn a normalized delta back into real units. +- **LeRobot policies run at their training rate:** `--hz` defaults to the dataset's fps (read through the checkpoint), and `InferenceSession.run()` warns when the rate differs (a 30 fps policy at 10 Hz played its actions 3x slower). +- **Output names:** a recording session folder ends in `_Recording` (after the `session_folder_format` pattern). A Post-process output folder and the dataset inside it end in `_Robomimic` or `_Lerobot` + (`2026-11-02_15-10-00_Robomimic/dataset_