From d0ff59163f2a600c4a2809ff7a9fa5125a5681f7 Mon Sep 17 00:00:00 2001 From: theo-michel Date: Wed, 15 Jul 2026 07:02:31 +0000 Subject: [PATCH 01/19] feat(pick): wrist-camera visual-servo grasp for pick_any_object + tuning panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a wrist-camera fine-alignment stage to the pick_any_object grasp, and the Teleop tuning panel that drives it. After the base parks the object in the head-camera pick box, the arm goes to an operator-posed search position (up high, elbow ~90°, wrist camera looking down), Gemini marks the object, and the arm visual-servos straight down onto it: center-in-box first, then descend one step, repeating until a stop height where the blind ladder finishes the grasp. Tracking is HSV color segmentation + CamShift, not optical flow: during the descent the object grows ~2.5x in the wrist image and fabric deforms, which slides LK patches onto the carpet. A likelihood-ratio color model (object vs surrounding floor) back-projected each frame is scale- and deformation-proof, and its score honestly reports occlusion/loss instead of tracking carpet. The pick_any_object skill and its runtime helpers (arm_rest_position, gripper_open, gripper_close) move from the gitignored workspace/custom_skills/ into workspace/innate_skills/ so the pipeline is tracked and works from a fresh checkout. Bare-name skill chaining still resolves them (local/ then innate-os/). Panel (Teleop, key "o"): live top-down + side grasp views, per-stage sliders that publish overrides the running skill applies mid-run, and a draggable wrist box + detection marker over the arm-camera video for aiming the servo. Topics: skill -> /pick_any_object/debug (stage events), panel -> /pick_any_object/tuning (parameter overrides). --- webapp/css/app.css | 332 +++++ webapp/js/constants.js | 6 + webapp/js/teleop/main.js | 2 + webapp/js/teleop/pickTunePanel.js | 1237 +++++++++++++++++ workspace/innate_skills/arm_rest_position.py | 105 ++ workspace/innate_skills/gripper_close.py | 56 + workspace/innate_skills/gripper_open.py | 53 + workspace/innate_skills/pick_any_object.py | 1296 ++++++++++++++++++ 8 files changed, 3087 insertions(+) create mode 100644 webapp/js/teleop/pickTunePanel.js create mode 100644 workspace/innate_skills/arm_rest_position.py create mode 100644 workspace/innate_skills/gripper_close.py create mode 100644 workspace/innate_skills/gripper_open.py create mode 100644 workspace/innate_skills/pick_any_object.py diff --git a/webapp/css/app.css b/webapp/css/app.css index bf5ae931f..c9398be68 100644 --- a/webapp/css/app.css +++ b/webapp/css/app.css @@ -872,6 +872,268 @@ button { color: var(--accent); } +/* ---- pick-tuning panel (Teleop) ------------------------------------------ */ + +/* Toggle sits under the telemetry card, left edge — same glass pulse-button + language as the profiling toggle. */ +.picktune-toggle { + position: absolute; + top: 92px; + left: 14px; + display: flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + padding: 0; + background: var(--glass); + backdrop-filter: blur(14px); + -webkit-backdrop-filter: blur(14px); + border: 1px solid var(--hairline); + border-radius: 9px; + color: var(--muted); + cursor: pointer; + transition: color 150ms ease, border-color 150ms ease; +} + +.picktune-toggle:hover { + color: var(--text); + border-color: var(--hairline-strong); +} + +.picktune-toggle.active { + color: var(--accent); + border-color: var(--accent-dim); +} + +/* A pick run is streaming debug events — pulse green even while closed. */ +.picktune-toggle.running { + color: var(--ok); + border-color: var(--ok); +} + +.picktune { + position: absolute; + top: 134px; + left: 14px; + z-index: 2; + width: 324px; + max-height: calc(100% - 148px); + overflow-y: auto; + padding: 11px 14px; + background: rgb(0 0 0 / 66%); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + border: 1px solid var(--hairline); + border-radius: 11px; + user-select: none; + -webkit-user-select: none; +} + +.picktune-head { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 10px; +} + +.picktune-head .profiling-hint { + margin-left: auto; +} + +.picktune-sync { + font-size: 10px; + color: var(--muted); +} + +.picktune-sync.live { + color: var(--ok); +} + +.picktune-canvas { + display: block; + width: 296px; + height: 252px; + border: 1px solid var(--hairline); + border-radius: 8px; +} + +/* Side (x–z) grasp view under the top-down view. */ +.picktune-canvas-side { + height: 150px; + margin-top: 8px; +} + +/* Grasp reticle over the live head video: the skill's grasp pixel, centered + on (left,top) set in JS. Lives in .video-stage, drawn above the frame but + below the glass overlays. */ +.picktune-grab { + position: absolute; + z-index: 1; + transform: translate(-50%, -50%); + color: var(--accent); + pointer-events: none; + filter: drop-shadow(0 0 2px rgb(0 0 0 / 70%)); +} + +.picktune-grab-tag { + position: absolute; + top: 50%; + left: calc(100% + 4px); + transform: translateY(-50%); + font-size: 10px; + color: inherit; + white-space: nowrap; +} + +/* Detection marker (box corners at Gemini's pixel) — green to read apart + from the amber grasp target. Its tag hangs left so the two labels don't + collide when detection and grasp target sit centimeters apart (the usual + case: they differ only by the fingertip offset). */ +.picktune-seen { + color: var(--ok); +} + +.picktune-seen .picktune-grab-tag { + left: auto; + right: calc(100% + 4px); +} + +/* The positioning goal square on the video: put the detection inside this and + the base stops. Green while the skill reports the detection inside. + Draggable — moving it re-aims the skill (sweet_x / box_y). */ +.picktune-boxgoal { + position: absolute; + z-index: 1; + transform: translate(-50%, -50%); + border: 1.5px dashed var(--accent-dim); + border-radius: 2px; + pointer-events: auto; + cursor: grab; + touch-action: none; +} + +.picktune-boxgoal.dragging { + cursor: grabbing; + border-color: var(--accent); +} + +/* Inner accept box: the point must land in THIS to stop positioning. Solid + amber guide; greens (with the tag) when the skill reports the point inside. */ +.picktune-boxaccept { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + border: 1.5px solid var(--accent-dim); + border-radius: 2px; + pointer-events: none; +} + +.picktune-boxgoal.inside .picktune-boxaccept { + border-color: var(--ok); +} + +.picktune-boxgoal-tag { + position: absolute; + bottom: calc(100% + 3px); + left: 0; + font-size: 10px; + color: var(--accent-dim); + white-space: nowrap; +} + +.picktune-boxgoal.inside .picktune-boxgoal-tag { + color: var(--ok); +} + +.picktune-group { + margin-top: 10px; + margin-bottom: 4px; +} + +.picktune-param { + display: grid; + grid-template-columns: 82px 1fr 64px; + align-items: center; + gap: 8px; + padding: 1px 0; +} + +.picktune-param-label { + font-size: 11px; + color: var(--muted); + white-space: nowrap; +} + +/* Overridden away from the code default — flag it. */ +.picktune-param.dirty .picktune-param-label { + color: var(--accent); +} + +.picktune-param-value { + font-size: 11px; + color: var(--text); + text-align: right; + white-space: nowrap; +} + +.picktune-slider { + width: 100%; + height: 3px; + appearance: none; + -webkit-appearance: none; + background: var(--hairline-strong); + border-radius: 2px; + outline: none; + cursor: pointer; +} + +.picktune-slider::-webkit-slider-thumb { + appearance: none; + -webkit-appearance: none; + width: 11px; + height: 11px; + border-radius: 50%; + background: var(--text); + border: none; +} + +.picktune-slider::-moz-range-thumb { + width: 11px; + height: 11px; + border-radius: 50%; + background: var(--text); + border: none; +} + +.picktune-param.dirty .picktune-slider::-webkit-slider-thumb { + background: var(--accent); +} + +.picktune-param.dirty .picktune-slider::-moz-range-thumb { + background: var(--accent); +} + +.picktune-footer { + display: flex; + justify-content: flex-end; + margin: 10px 0; +} + +.picktune-log { + margin-top: 4px; + max-height: 108px; + overflow-y: auto; + font-size: 10px; + line-height: 1.5; + color: var(--muted); +} + +.picktune-log-line.warn { + color: var(--danger); +} + /* ---- record HUD (Collect page) ------------------------------------------ */ .overlay-record { @@ -1563,6 +1825,76 @@ button { margin: 2px 0; } +/* Live follower-joint readout — a subtle click-to-copy array literal. */ +.arm-follower { + position: relative; + display: flex; + align-items: center; + gap: 6px; +} + +/* Copy icon sitting beside the readout. */ +.arm-follower-copy { + flex: none; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 2px; + color: var(--muted); + background: transparent; + border: 0; + cursor: pointer; +} + +.arm-follower-copy:hover:not(:disabled) { + color: var(--text); +} + +.arm-follower-copy:disabled { + opacity: 0.4; + cursor: default; +} + +/* "Copied" confirmation, floated above the readout on click. */ +.arm-follower-tip { + position: absolute; + bottom: calc(100% + 3px); + left: 0; + font-size: 9px; + color: var(--ok); + background: var(--panel); + border: 1px solid var(--ok); + border-radius: 3px; + padding: 1px 5px; + pointer-events: none; +} + +.arm-follower-value { + flex: 1; + min-width: 0; + text-align: left; + font-size: 9px; + line-height: 1.3; + color: var(--muted); + background: transparent; + border: 0; + padding: 0; + cursor: pointer; + word-break: break-word; +} + +.arm-follower-value:hover:not(:disabled) { + color: var(--text); +} + +.arm-follower-value:disabled { + cursor: default; +} + +.arm-follower-value.copied { + color: var(--ok); +} + .arm-services { display: flex; flex-direction: column; diff --git a/webapp/js/constants.js b/webapp/js/constants.js index c52d6e2d1..4873163e3 100644 --- a/webapp/js/constants.js +++ b/webapp/js/constants.js @@ -132,6 +132,12 @@ export const PINNED_SKILLS = ["navigate with vision", "navigate with position", export const EXECUTE_SKILL_ACTION = "/execute_skill"; export const EXECUTE_SKILL_ACTION_TYPE = "brain_messages/action/ExecuteSkill"; +// Pick-tuning panel (Teleop) <-> pick_any_object skill. DEBUG streams the +// skill's stage events (std_msgs/String JSON: {ev, t, ...}); TUNING carries +// live parameter overrides the same way ({param_key: number, ...}). +export const PICK_DEBUG_TOPIC = "/pick_any_object/debug"; +export const PICK_TUNING_TOPIC = "/pick_any_object/tuning"; + export const HEAD_MIN_DEG = -40; export const HEAD_MAX_DEG = 70; diff --git a/webapp/js/teleop/main.js b/webapp/js/teleop/main.js index 37d636718..7c3213852 100644 --- a/webapp/js/teleop/main.js +++ b/webapp/js/teleop/main.js @@ -23,6 +23,7 @@ import { createTtsBar } from "./ttsBar.js"; import { createTelemetry } from "./telemetry.js"; import { createArmPanel } from "./armPanel.js"; import { createProfilingPanel } from "./profilingPanel.js"; +import { createPickTunePanel } from "./pickTunePanel.js"; import { createSkillsMenu } from "./skillsMenu.js"; import { createCameraSwitch } from "./cameraSwitch.js"; @@ -92,6 +93,7 @@ function buildCockpit(root) { createSkillsMenu(ttsOverlay, ros), createArmPanel(armOverlay, ros, { hideServices: !!config.simControls }), ...(config.simControls ? [] : [createProfilingPanel(root, session)]), + createPickTunePanel(root, ros, session), createCameraSwitch(root, session, ros), keyboard, ); diff --git a/webapp/js/teleop/pickTunePanel.js b/webapp/js/teleop/pickTunePanel.js new file mode 100644 index 000000000..c95ea1107 --- /dev/null +++ b/webapp/js/teleop/pickTunePanel.js @@ -0,0 +1,1237 @@ +// @ts-check +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2026 Innate Inc +// Pick-tuning panel — a live workbench for iterating on the pick_any_object +// skill's POSITION and GRASP stages. A crosshair toggle under the telemetry +// card (or "o") opens it; hidden by default so the cockpit stays clean. +// +// Top: a top-down base_link view (robot at the bottom, +x forward/up) that +// plots every metric localization the skill reports on /pick_any_object/debug, +// with the sweet-spot ring, bearing/approach tolerance geometry, and the arm's +// grasp-reach clamp box drawn from the *current* slider values — so you see a +// run converge (or not) against the exact thresholds you set. Under it: a +// side (x–z) view of the grasp — the planned hover→descend ladder from the +// sliders, overlaid with the actual EE positions each stage reports, so a +// missed grasp shows exactly where the descent diverged. Below: sliders +// that publish overrides to /pick_any_object/tuning (applied live, mid-run, +// by the skill) and a compact event log. +// +// The skill also reprojects its grasp target back onto the head frame and +// sends the pixel (grab_px) on the debug topic. We draw a reticle over the +// live video at that pixel — the actual spot on the floor the fingers will +// close on — so you can watch the aim on the real image, not just the plot. +// The detection pixel from each localize event gets its own marker (green box +// corners) so you see where Gemini found the object vs where the arm will +// grab; it clears whenever the base starts moving, since a stored pixel no +// longer matches the shifted view, and reappears on the next localization. +// The head video is object-fit:contain (letterboxed, never cropped), so the +// 640×480 pixel maps to the screen with a single uniform scale + centering. +// When the ARM (wrist) camera is on the big stage instead, the same stage +// hosts the wrist-align overlays: a draggable goal box in raw wrist-image +// pixels (wrist_box_u/v) and the skill's wrist detection marker. +// +// Sync protocol: overrides persist in localStorage and are re-published every +// time the skill reports run_start, which closes the cold-start gap (the +// skill only subscribes once its first run begins). The skill acks every +// tuning message by echoing its live params; "live" in the header means the +// last echo matches our sliders. + +import { PICK_DEBUG_TOPIC, PICK_TUNING_TOPIC } from "../constants.js"; + +const TOGGLE_KEY = "KeyO"; +const STORAGE_KEY = "innate.pickTune.overrides"; +const PUBLISH_DEBOUNCE_MS = 120; +const LOG_CAP = 40; + +// Grasp-target clamp in pick_any_object.py's _grasp_at — drawn as the reach box. +const REACH = { xMin: 0.22, xMax: 0.4, yMax: 0.1 }; + +/** + * @typedef {{ key: string, label: string, unit: string, min: number, max: number, step: number, def: number }} ParamSpec + */ + +// Mirrors TUNABLE in workspace/custom_skills/pick_any_object.py — keep the +// keys and defaults in sync (defaults here are what Reset publishes). +/** @type {{ label: string, params: ParamSpec[] }[]} */ +const PARAM_GROUPS = [ + { + // Position first: box x (forward) + box y (lateral) are the two coords the + // video drag writes, kept adjacent so the pair is obvious. box x can aim + // past the arm's ~0.43 m grasp reach — beyond that the grasp saturates at + // the reach clamp (positioning still works, the fingers just stop short). + label: "pick box (image goal)", + params: [ + { key: "sweet_x", label: "box x", unit: "m", min: 0.2, max: 1.0, step: 0.005, def: 0.3 }, + { key: "box_y", label: "box y", unit: "m", min: -0.12, max: 0.12, step: 0.005, def: 0 }, + { key: "box_half_px", label: "box half", unit: "px", min: 15, max: 120, step: 5, def: 40 }, + { key: "accept_frac", label: "accept", unit: "×", min: 0.2, max: 1, step: 0.05, def: 0.5 }, + { key: "box_steps", label: "max steps", unit: "", min: 2, max: 10, step: 1, def: 6 }, + { key: "bearing_go_deg", label: "bearing go", unit: "°", min: 1, max: 15, step: 0.5, def: 4 }, + ], + }, + { + // Pixel-servo gains for the optical-flow follower: rad/s (or m/s) per 100 px + // of tracked-point error from the box centre. + label: "follow (px servo)", + params: [ + { key: "follow_gain_ang", label: "turn gain", unit: "", min: 0.05, max: 1.0, step: 0.05, def: 0.3 }, + { key: "follow_gain_lin", label: "drive gain", unit: "", min: 0.01, max: 0.2, step: 0.01, def: 0.06 }, + ], + }, + { + label: "detect", + params: [ + { key: "tilt_deg", label: "head tilt", unit: "°", min: -35, max: -10, step: 1, def: -20 }, + { key: "settle_s", label: "settle", unit: "s", min: 0.2, max: 3, step: 0.1, def: 1.2 }, + ], + }, + { + label: "rotate loop", + params: [ + { key: "rot_tol_deg", label: "tol", unit: "°", min: 0.5, max: 6, step: 0.25, def: 2.5 }, + { key: "rot_kp", label: "kp", unit: "", min: 0.3, max: 3, step: 0.1, def: 1.2 }, + { key: "rot_wz_max", label: "wz max", unit: "rad/s", min: 0.15, max: 0.8, step: 0.05, def: 0.5 }, + { key: "rot_wz_min", label: "wz min", unit: "rad/s", min: 0.05, max: 0.3, step: 0.01, def: 0.15 }, + ], + }, + { + label: "drive loop", + params: [ + { key: "drive_tol_m", label: "tol", unit: "m", min: 0.005, max: 0.05, step: 0.005, def: 0.015 }, + { key: "drive_kp", label: "kp", unit: "", min: 0.1, max: 1, step: 0.05, def: 0.3 }, + { key: "drive_v_max", label: "v max", unit: "m/s", min: 0.04, max: 0.2, step: 0.01, def: 0.1 }, + { key: "drive_v_min", label: "v min", unit: "m/s", min: 0.02, max: 0.08, step: 0.005, def: 0.04 }, + ], + }, + { + // Wrist visual-servo descent: the hover goes to the operator's posed + // search position (up high, wrist camera on the object), Gemini seeds the + // object pixel there, then optical flow tracks it on wrist frames while + // the arm steps down, correcting x/y each cycle; z drops only while the + // point is inside the wrist box (drag it on the arm-camera view). Gains + // are SIGNED — the wrist cam is mirrored — so flip one live if the servo + // diverges instead of centering. + label: "wrist servo (arm cam)", + params: [ + { key: "wrist_steps", label: "gemini seeds", unit: "", min: 0, max: 4, step: 1, def: 2 }, + { key: "wrist_stop_z", label: "stop z", unit: "m", min: 0.02, max: 0.12, step: 0.005, def: 0.04 }, + { key: "wrist_z_step", label: "z step", unit: "m", min: 0.005, max: 0.04, step: 0.005, def: 0.01 }, + { key: "wrist_move_s", label: "step t", unit: "s", min: 0.2, max: 1.5, step: 0.1, def: 0.5 }, + { key: "wrist_pitch", label: "pitch", unit: "rad", min: 0.6, max: 1.55, step: 0.01, def: 0.82 }, + { key: "wrist_box_u", label: "box u", unit: "px", min: 0, max: 640, step: 5, def: 320 }, + { key: "wrist_box_v", label: "box v", unit: "px", min: 0, max: 480, step: 5, def: 240 }, + { key: "wrist_half_px", label: "box half", unit: "px", min: 20, max: 160, step: 5, def: 60 }, + { key: "wrist_kx", label: "fwd gain", unit: "", min: -0.12, max: 0.12, step: 0.005, def: -0.04 }, + { key: "wrist_ky", label: "lat gain", unit: "", min: -0.12, max: 0.12, step: 0.005, def: -0.04 }, + { key: "wrist_step_max", label: "step max", unit: "m", min: 0.01, max: 0.08, step: 0.005, def: 0.04 }, + { key: "wrist_settle_s", label: "settle", unit: "s", min: 0.2, max: 2, step: 0.1, def: 0.8 }, + ], + }, + { + label: "grasp descent", + params: [ + { key: "grasp_x_off", label: "x offset", unit: "m", min: 0, max: 0.08, step: 0.005, def: 0.03 }, + { key: "hover_z", label: "hover z", unit: "m", min: 0.08, max: 0.25, step: 0.005, def: 0.15 }, + { key: "hover_s", label: "hover t", unit: "s", min: 0.8, max: 4, step: 0.1, def: 2 }, + { key: "descend_z1", label: "step z1", unit: "m", min: 0.02, max: 0.14, step: 0.005, def: 0.1 }, + { key: "descend_z2", label: "step z2", unit: "m", min: 0.01, max: 0.12, step: 0.005, def: 0.06 }, + { key: "descend_z3", label: "step z3", unit: "m", min: 0, max: 0.1, step: 0.005, def: 0.03 }, + { key: "floor_z", label: "floor z", unit: "m", min: 0, max: 0.05, step: 0.002, def: 0.01 }, + { key: "descend_s", label: "step t", unit: "s", min: 0.4, max: 3, step: 0.1, def: 1.2 }, + { key: "descend_abort_z", label: "abort z", unit: "m", min: 0.05, max: 0.2, step: 0.005, def: 0.12 }, + { key: "arm_pitch", label: "pitch", unit: "rad", min: 1, max: 1.55, step: 0.01, def: 1.3 }, + ], + }, + { + label: "grasp close · lift", + params: [ + { key: "close_strength", label: "strength", unit: "", min: 0.2, max: 1, step: 0.05, def: 0.6 }, + { key: "close_s", label: "close t", unit: "s", min: 0.5, max: 3, step: 0.1, def: 1.5 }, + { key: "close_settle_s", label: "settle", unit: "s", min: 0, max: 2, step: 0.1, def: 0.8 }, + { key: "twist_rad", label: "twist", unit: "rad", min: 0, max: 1.4, step: 0.05, def: 0.6 }, + { key: "lift_rad", label: "lift", unit: "rad", min: 0.2, max: 1.4, step: 0.05, def: 0.6 }, + ], + }, +]; + +/** @type {ParamSpec[]} */ +const ALL_PARAMS = PARAM_GROUPS.flatMap((g) => g.params); + +// Top-down view window in base_link meters: x (forward) up, y (left) left. +// xMax spans the full box x range so the goal stays on-canvas at max reach-out. +const VIEW = { xMin: -0.08, xMax: 1.08, canvasW: 296, canvasH: 252 }; + +// Head-camera frame size the skill's grasp pixel is expressed in (IMG_W/IMG_H +// in pick_any_object.py). Only a fallback — the live