mirror of
https://github.com/huggingface/lerobot.git
synced 2026-08-01 05:59:44 +00:00
Add Unitree Go2 robot + spatial-memory navigation base
unitree_go2 robot: high-level sport-mode body-velocity control over DDS (unitree_sdk2py) straight from the host — no companion computer, unlike the G1's low-level ZMQ bridge. Actions x.vel/y.vel/theta.vel; observation is planar odometry (rt/sportmodestate) + the built-in front camera via VideoClient. SDK imported only in connect() so configs/features/tests work without it; 20 tests via mocks. unitree_go2 pyproject extra. lerobot.navigation package: BaseController protocol + StubBaseController + SafeBaseController (velocity clamp, occupancy gate, keyframe watchdog, e-stop latch) + RobotBaseController wrapping any Robot on the standard REP-103 mobile-base contract, carrying the world<->body velocity and odometry<->world pose frame math. Robot-agnostic: the SDK lives only in the robot class. 29 tests, SDK/torch-free. First step of consolidating the dyna360 DynaMem navigation stack into lerobot; dyna360 is a source to copy from, not a runtime dependency. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
# `lerobot.navigation` — spatial-memory navigation
|
||||
|
||||
Online spatio-semantic mapping (DynaMem-style), A* planning, obstacle
|
||||
avoidance and open-vocabulary goto/explore for LeRobot mobile bases.
|
||||
Ported from the dyna360 research stack; the physical robot layer lives in
|
||||
`lerobot.robots` (e.g. [`unitree_go2`](../robots/unitree_go2)).
|
||||
|
||||
## Idea
|
||||
|
||||
Drive any LeRobot `Robot` on the standard REP-103 mobile-base contract —
|
||||
body-velocity actions `x.vel`/`y.vel`/`theta.vel` and planar odometry
|
||||
`x.pos`/`y.pos`/`theta.pos` — from a spatial memory that is built and
|
||||
updated online from the robot's camera. With no prompt the base explores
|
||||
autonomously; given a text prompt it queries the map and navigates to the
|
||||
matching object, or explores to find it if it isn't there (or has moved).
|
||||
|
||||
## Architecture
|
||||
|
||||
The navigation layer talks to hardware only through LeRobot's own `Robot`
|
||||
interface, so it is robot-agnostic and carries no SDK dependency.
|
||||
|
||||
```
|
||||
BaseController (protocol) world-frame move()/pose() seam
|
||||
├── StubBaseController kinematic integrator (sim, tests)
|
||||
├── RobotBaseController wraps any Robot; world<->body +
|
||||
│ odometry<->world frame math
|
||||
└── SafeBaseController velocity clamp, occupancy gate,
|
||||
keyframe watchdog, e-stop latch
|
||||
```
|
||||
|
||||
World frame is OpenCV (x right, y down, z forward); the base moves in the
|
||||
XZ plane. `RobotBaseController.feed_observation(obs)` updates pose from
|
||||
the observation the navigation loop already fetches (closed-loop
|
||||
odometry), avoiding an extra camera read; absent odometry it integrates
|
||||
open-loop so sim matches hardware.
|
||||
|
||||
## Status (branch `feat/unitree-go2`)
|
||||
|
||||
Implemented:
|
||||
- `base_controller.py` — the controller seam above. SDK/torch-free;
|
||||
tests in `tests/navigation/test_base_controller.py`.
|
||||
|
||||
Planned (porting from dyna360, everything lands here — dyna360 is a
|
||||
source to copy from, not a runtime dependency):
|
||||
- `geometry.py` — LingBot-Map streaming reconstruction runner
|
||||
(`{points, local_points, conf, camera_poses}`), scale-anchored to
|
||||
odometry. (Pi3X is not carried over.)
|
||||
- `voxel_map.py` — 5 cm `SegmentVoxelMap`, no point-cloud retention;
|
||||
one SigLIP2 feature per segment.
|
||||
- `occupancy.py` — occupancy grid + A* + frontiers; `value_map.py`.
|
||||
- `features.py` (SigLIP2), `segmenter.py` (SAM2) — run offboard.
|
||||
- `agent.py`, `skills.py`, `dog_cli.py` — the interactive explore/query
|
||||
REPL (the deliverable).
|
||||
|
||||
## Target platform
|
||||
|
||||
Unitree Go2 EDU, no companion computer: the workstation (single RTX 5090)
|
||||
talks DDS straight to the dog; geometry is monocular LingBot-Map from the
|
||||
built-in front camera, scale-anchored to sport-mode odometry; the map is
|
||||
5 cm voxels. See [`robots/unitree_go2`](../robots/unitree_go2).
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Spatial-memory navigation for LeRobot mobile bases.
|
||||
|
||||
Online spatio-semantic mapping (DynaMem-style), A* planning, obstacle
|
||||
avoidance and open-vocabulary goto/explore, driving any LeRobot ``Robot``
|
||||
that exposes body-velocity actions and planar odometry. Ported from the
|
||||
dyna360 research stack; the physical robot layer lives in
|
||||
``lerobot.robots`` (e.g. ``unitree_go2``).
|
||||
"""
|
||||
|
||||
from .base_controller import (
|
||||
BaseController,
|
||||
RobotBaseController,
|
||||
SafeBaseController,
|
||||
StubBaseController,
|
||||
odometry_to_world_pose,
|
||||
world_velocity_to_body,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"BaseController",
|
||||
"RobotBaseController",
|
||||
"SafeBaseController",
|
||||
"StubBaseController",
|
||||
"odometry_to_world_pose",
|
||||
"world_velocity_to_body",
|
||||
]
|
||||
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Base controller for spatial-memory navigation.
|
||||
|
||||
The navigation/skills layer commands motion in a single **world frame**
|
||||
(OpenCV convention: x right, y down, z forward — the base lives in the XZ
|
||||
plane, y is gravity) and reads back an SE(3) pose. :class:`BaseController`
|
||||
is that seam. Three implementations:
|
||||
|
||||
- :class:`StubBaseController` — kinematic integrator, no hardware; sim +
|
||||
unit tests.
|
||||
- :class:`RobotBaseController` — drives any LeRobot :class:`Robot` whose
|
||||
action space is body-frame velocities ``x.vel`` (forward, m/s),
|
||||
``y.vel`` (left, m/s), ``theta.vel`` (CCW yaw, rad/s) and whose
|
||||
observation carries planar odometry ``x.pos``/``y.pos``/``theta.pos``
|
||||
(REP-103: x forward, y left, yaw CCW). The Unitree Go2 satisfies this
|
||||
out of the box; so would a LeKiwi base.
|
||||
- :class:`SafeBaseController` — wraps any of the above with velocity
|
||||
clamping, an optional occupancy gate, a keyframe watchdog and an
|
||||
e-stop latch.
|
||||
|
||||
All frame conversions between the world frame and a robot's body/odometry
|
||||
frame live in :func:`world_velocity_to_body` and
|
||||
:func:`odometry_to_world_pose`; nothing else needs to know the mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
|
||||
import numpy as np
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lerobot.robots import Robot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# BaseController protocol
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class BaseController(Protocol):
|
||||
"""Mobile-base interface used by the navigation/skills layer.
|
||||
|
||||
Velocities are in **world** frame XZ (m/s); ``yaw_rate`` is rad/s
|
||||
about the world's −Y axis (turning around the up vector). ``pose``
|
||||
is 4×4 SE(3) camera-to-world (OpenCV).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def move(self, vx: float, vz: float, yaw_rate: float = 0.0, dt: float = 0.05) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def stop(self) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
def pose(self) -> np.ndarray: ...
|
||||
|
||||
@abstractmethod
|
||||
def position(self) -> tuple[float, float, float]: ...
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Frame math (pure functions)
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def world_velocity_to_body(
|
||||
vx_world: float,
|
||||
vz_world: float,
|
||||
yaw_rate_rad_s: float,
|
||||
heading_rad: float,
|
||||
) -> tuple[float, float, float]:
|
||||
"""World-frame velocity → body-frame ``(x.vel, y.vel, theta.vel)``.
|
||||
|
||||
Returns ``(vx_forward, vy_left, vyaw)`` in m/s, m/s, rad/s — the
|
||||
action a REP-103 base expects. At heading ``h`` the body axes in the
|
||||
world XZ plane are forward = (sin h, cos h), left = (−cos h, sin h)
|
||||
(left = up × forward, up = −y). The navigation world's positive yaw
|
||||
is clockwise about the up vector; a REP-103 base's ``theta.vel`` is
|
||||
counter-clockwise, hence the sign flip.
|
||||
"""
|
||||
s, c = math.sin(heading_rad), math.cos(heading_rad)
|
||||
vx_fwd = vx_world * s + vz_world * c
|
||||
vy_left = -vx_world * c + vz_world * s
|
||||
return vx_fwd, vy_left, -yaw_rate_rad_s
|
||||
|
||||
|
||||
def odometry_to_world_pose(
|
||||
x_fwd: float,
|
||||
y_left: float,
|
||||
yaw: float,
|
||||
origin: tuple[float, float, float],
|
||||
) -> tuple[np.ndarray, float]:
|
||||
"""Planar odometry ``(x_fwd, y_left, yaw)`` → world pose + heading.
|
||||
|
||||
``origin`` is the ``(x_fwd, y_left, yaw)`` sample captured when the
|
||||
controller first saw odometry, so the run starts at identity
|
||||
regardless of where the robot's odometry origin sits. The result is
|
||||
the OpenCV world convention, planarized: height Y is 0 and only yaw
|
||||
survives of the orientation — pitch/roll gait wobble is the camera's
|
||||
concern, not the base's.
|
||||
|
||||
Odometry frame is REP-103 (x forward, y left, yaw CCW about z-up).
|
||||
Mapping to OpenCV world: ``x_world = −y_odom``, ``z_world = x_odom``,
|
||||
``heading = −yaw``.
|
||||
"""
|
||||
ox, oy, oyaw = origin
|
||||
dx, dy = x_fwd - ox, y_left - oy
|
||||
c0, s0 = math.cos(-oyaw), math.sin(-oyaw)
|
||||
x_rel = c0 * dx - s0 * dy
|
||||
y_rel = s0 * dx + c0 * dy
|
||||
yaw_rel = yaw - oyaw
|
||||
|
||||
x_world, z_world = -y_rel, x_rel
|
||||
heading = -yaw_rel
|
||||
|
||||
ch, sh = math.cos(heading), math.sin(heading)
|
||||
pose = np.eye(4, dtype=np.float64)
|
||||
pose[0, 0], pose[0, 2] = ch, sh
|
||||
pose[2, 0], pose[2, 2] = -sh, ch
|
||||
pose[0, 3], pose[2, 3] = x_world, z_world
|
||||
return pose, heading
|
||||
|
||||
|
||||
def _heading_pose(x: float, z: float, heading: float) -> np.ndarray:
|
||||
"""Build a planar world pose from position + heading."""
|
||||
c, s = math.cos(heading), math.sin(heading)
|
||||
pose = np.eye(4, dtype=np.float64)
|
||||
pose[0, 0], pose[0, 2] = c, s
|
||||
pose[2, 0], pose[2, 2] = -s, c
|
||||
pose[0, 3], pose[2, 3] = x, z
|
||||
return pose
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Stub controller (kinematic, no hardware)
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@dataclass
|
||||
class StubBaseController:
|
||||
"""Kinematic stub: integrates each ``move()`` into pose exactly.
|
||||
|
||||
No latency, slip or dynamics — for sim and skill-layer unit tests.
|
||||
"""
|
||||
|
||||
initial_pose: np.ndarray | None = None
|
||||
max_lin_speed: float = 1.0
|
||||
max_yaw_rate: float = 1.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._pose = (
|
||||
np.asarray(self.initial_pose, dtype=np.float64).copy()
|
||||
if self.initial_pose is not None
|
||||
else np.eye(4, dtype=np.float64)
|
||||
)
|
||||
if self._pose.shape != (4, 4):
|
||||
raise ValueError(f"initial_pose must be (4, 4); got {self._pose.shape}")
|
||||
self._heading = 0.0
|
||||
self._stopped = False
|
||||
|
||||
def move(self, vx: float, vz: float, yaw_rate: float = 0.0, dt: float = 0.05) -> None:
|
||||
vx = float(np.clip(vx, -self.max_lin_speed, self.max_lin_speed))
|
||||
vz = float(np.clip(vz, -self.max_lin_speed, self.max_lin_speed))
|
||||
yaw_rate = float(np.clip(yaw_rate, -self.max_yaw_rate, self.max_yaw_rate))
|
||||
if dt <= 0:
|
||||
return
|
||||
self._pose[0, 3] += vx * dt
|
||||
self._pose[2, 3] += vz * dt
|
||||
if yaw_rate != 0.0:
|
||||
self._heading += yaw_rate * dt
|
||||
self._pose = _heading_pose(self._pose[0, 3], self._pose[2, 3], self._heading)
|
||||
self._stopped = False
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stopped = True
|
||||
|
||||
def pose(self) -> np.ndarray:
|
||||
return self._pose.copy()
|
||||
|
||||
def position(self) -> tuple[float, float, float]:
|
||||
p = self._pose[:3, 3]
|
||||
return float(p[0]), float(p[1]), float(p[2])
|
||||
|
||||
@property
|
||||
def is_stopped(self) -> bool:
|
||||
return self._stopped
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Robot-backed controller
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RobotBaseControllerConfig:
|
||||
"""Behaviour knobs for :class:`RobotBaseController`."""
|
||||
|
||||
max_lin_speed: float = 0.6
|
||||
"""Hard cap on per-axis world linear velocity (m/s)."""
|
||||
|
||||
max_yaw_rate: float = 1.2
|
||||
"""Hard cap on yaw rate (rad/s)."""
|
||||
|
||||
pose_from_odometry: bool = True
|
||||
"""Report pose from the robot's odometry (closed-loop). When False,
|
||||
integrate pose open-loop from commanded velocities."""
|
||||
|
||||
|
||||
class RobotBaseController(BaseController):
|
||||
""":class:`BaseController` over any LeRobot :class:`Robot`.
|
||||
|
||||
The robot must accept body-velocity actions ``x.vel`` (forward),
|
||||
``y.vel`` (left), ``theta.vel`` (CCW yaw) and — for closed-loop pose
|
||||
— report odometry ``x.pos``/``y.pos``/``theta.pos`` in its
|
||||
observation. This is the standard REP-103 mobile-base contract, which
|
||||
``UnitreeGo2`` implements.
|
||||
|
||||
Pose is refreshed from observations the navigation loop already
|
||||
fetches: call :meth:`feed_observation` each keyframe rather than
|
||||
having the controller poll the robot (which would trigger an extra
|
||||
camera read). Absent any fed observation, pose falls back to
|
||||
open-loop integration so sim/dry-run behaves like the stub.
|
||||
"""
|
||||
|
||||
def __init__(self, robot: Robot, cfg: RobotBaseControllerConfig | None = None) -> None:
|
||||
self.robot = robot
|
||||
self.cfg = cfg or RobotBaseControllerConfig()
|
||||
self._pose = np.eye(4, dtype=np.float64)
|
||||
self._heading = 0.0
|
||||
self._stopped = False
|
||||
self._odom_origin: tuple[float, float, float] | None = None
|
||||
self._have_odom = False
|
||||
|
||||
# ----- odometry feed --------------------------------------------------
|
||||
|
||||
def feed_observation(self, obs: dict) -> None:
|
||||
"""Update pose from an observation the nav loop already fetched."""
|
||||
if not self.cfg.pose_from_odometry:
|
||||
return
|
||||
if not {"x.pos", "y.pos", "theta.pos"} <= obs.keys():
|
||||
return
|
||||
sample = (float(obs["x.pos"]), float(obs["y.pos"]), float(obs["theta.pos"]))
|
||||
if self._odom_origin is None:
|
||||
self._odom_origin = sample
|
||||
self._pose, self._heading = odometry_to_world_pose(*sample, self._odom_origin)
|
||||
self._have_odom = True
|
||||
|
||||
# ----- BaseController API --------------------------------------------
|
||||
|
||||
def move(self, vx: float, vz: float, yaw_rate: float = 0.0, dt: float = 0.05) -> None:
|
||||
vx = float(np.clip(vx, -self.cfg.max_lin_speed, self.cfg.max_lin_speed))
|
||||
vz = float(np.clip(vz, -self.cfg.max_lin_speed, self.cfg.max_lin_speed))
|
||||
yaw_rate = float(np.clip(yaw_rate, -self.cfg.max_yaw_rate, self.cfg.max_yaw_rate))
|
||||
if dt <= 0:
|
||||
return
|
||||
|
||||
vx_fwd, vy_left, vyaw = world_velocity_to_body(vx, vz, yaw_rate, self._heading)
|
||||
self.robot.send_action({"x.vel": vx_fwd, "y.vel": vy_left, "theta.vel": vyaw})
|
||||
|
||||
# Open-loop pose only when we have no odometry to trust.
|
||||
if not (self.cfg.pose_from_odometry and self._have_odom):
|
||||
self._pose[0, 3] += vx * dt
|
||||
self._pose[2, 3] += vz * dt
|
||||
if yaw_rate != 0.0:
|
||||
self._heading += yaw_rate * dt
|
||||
self._pose = _heading_pose(self._pose[0, 3], self._pose[2, 3], self._heading)
|
||||
self._stopped = False
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stopped = True
|
||||
try:
|
||||
self.robot.send_action({"x.vel": 0.0, "y.vel": 0.0, "theta.vel": 0.0})
|
||||
except Exception:
|
||||
logger.exception("stop(): failed to send zero-velocity action")
|
||||
|
||||
def pose(self) -> np.ndarray:
|
||||
return self._pose.copy()
|
||||
|
||||
def position(self) -> tuple[float, float, float]:
|
||||
p = self._pose[:3, 3]
|
||||
return float(p[0]), float(p[1]), float(p[2])
|
||||
|
||||
@property
|
||||
def is_stopped(self) -> bool:
|
||||
return self._stopped
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Safety wrapper
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@dataclass
|
||||
class SafeBaseController(BaseController):
|
||||
"""Wrap any :class:`BaseController` with safety layers:
|
||||
|
||||
- **velocity clamp** on every ``move()``;
|
||||
- **occupancy gate**: when ``occupancy_provider`` is set, predict
|
||||
the next position and refuse (latch e-stop) if it lands in an
|
||||
obstacle cell. The provider returns an object exposing
|
||||
``world_to_cell(x, z) -> (iz, ix)`` and an ``is_obstacle(iz, ix)
|
||||
-> bool`` predicate; ``None`` means "no map yet, allow";
|
||||
- **watchdog**: if no keyframe has been fed in
|
||||
``watchdog_timeout_s`` (caller ticks :meth:`feed_watchdog` per
|
||||
map update), ``move()`` latches stop until :meth:`reset_watchdog`.
|
||||
"""
|
||||
|
||||
inner: BaseController
|
||||
max_lin_speed: float = 0.6
|
||||
max_yaw_rate: float = 1.2
|
||||
occupancy_provider: object = None # callable[[], grid | None] when set
|
||||
watchdog_timeout_s: float = 2.0
|
||||
e_stop_latched: bool = False
|
||||
_last_keyframe_walltime: float = field(default_factory=time.monotonic, init=False)
|
||||
|
||||
def feed_watchdog(self) -> None:
|
||||
self._last_keyframe_walltime = time.monotonic()
|
||||
|
||||
def reset_watchdog(self) -> None:
|
||||
self.e_stop_latched = False
|
||||
self._last_keyframe_walltime = time.monotonic()
|
||||
|
||||
def latch_estop(self, reason: str = "external") -> None:
|
||||
logger.warning("SafeBaseController e-stop latched: %s", reason)
|
||||
self.e_stop_latched = True
|
||||
self.inner.stop()
|
||||
|
||||
def move(self, vx: float, vz: float, yaw_rate: float = 0.0, dt: float = 0.05) -> None:
|
||||
if self.e_stop_latched:
|
||||
return
|
||||
if (time.monotonic() - self._last_keyframe_walltime) > self.watchdog_timeout_s:
|
||||
self.latch_estop(f"watchdog: no keyframe in last {self.watchdog_timeout_s:.2f}s")
|
||||
return
|
||||
|
||||
vx = float(np.clip(vx, -self.max_lin_speed, self.max_lin_speed))
|
||||
vz = float(np.clip(vz, -self.max_lin_speed, self.max_lin_speed))
|
||||
yaw_rate = float(np.clip(yaw_rate, -self.max_yaw_rate, self.max_yaw_rate))
|
||||
|
||||
if self.occupancy_provider is not None:
|
||||
try:
|
||||
grid = self.occupancy_provider()
|
||||
except Exception:
|
||||
logger.exception("occupancy_provider raised; refusing move")
|
||||
return
|
||||
if grid is not None and self._would_enter_obstacle(grid, vx, vz, dt):
|
||||
self.latch_estop("about to enter obstacle cell")
|
||||
return
|
||||
|
||||
self.inner.move(vx, vz, yaw_rate, dt)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.inner.stop()
|
||||
|
||||
def pose(self) -> np.ndarray:
|
||||
return self.inner.pose()
|
||||
|
||||
def position(self) -> tuple[float, float, float]:
|
||||
return self.inner.position()
|
||||
|
||||
def _would_enter_obstacle(self, grid, vx: float, vz: float, dt: float) -> bool:
|
||||
pos = self.inner.position()
|
||||
next_x = pos[0] + vx * dt
|
||||
next_z = pos[2] + vz * dt
|
||||
iz, ix = grid.world_to_cell(next_x, next_z)
|
||||
return bool(grid.is_obstacle(iz, ix))
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from .config_unitree_go2 import UnitreeGo2Config
|
||||
from .unitree_go2 import UnitreeGo2
|
||||
|
||||
__all__ = ["UnitreeGo2", "UnitreeGo2Config"]
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from lerobot.cameras import CameraConfig
|
||||
|
||||
from ..config import RobotConfig
|
||||
|
||||
|
||||
@RobotConfig.register_subclass("unitree_go2")
|
||||
@dataclass
|
||||
class UnitreeGo2Config(RobotConfig):
|
||||
"""Configuration for the Unitree Go2 quadruped (EDU).
|
||||
|
||||
The host machine talks DDS directly to the dog over Ethernet/WiFi via
|
||||
``unitree_sdk2py`` — no onboard companion computer is required. Actions
|
||||
are high-level sport-mode body velocities; observations are sport-mode
|
||||
odometry plus the dog's built-in front camera.
|
||||
"""
|
||||
|
||||
# Network interface on the host that is wired/bridged to the Go2
|
||||
# (the dog lives on 192.168.123.x when connected over Ethernet).
|
||||
network_interface: str = "eth0"
|
||||
|
||||
# DDS domain id (0 for a stock Go2).
|
||||
domain_id: int = 0
|
||||
|
||||
# Safety clamps applied in send_action() before commands reach the dog.
|
||||
# The Go2 accepts far more (vx up to ~3.7 m/s) — keep indoor-sane defaults.
|
||||
max_x_vel: float = 1.0 # m/s, body forward
|
||||
max_y_vel: float = 0.5 # m/s, body left
|
||||
max_theta_vel: float = 1.5 # rad/s, CCW about z-up
|
||||
|
||||
# Built-in front camera, served through the SDK VideoClient.
|
||||
use_front_camera: bool = True
|
||||
front_camera_width: int = 1280
|
||||
front_camera_height: int = 720
|
||||
|
||||
# Send BalanceStand once on connect so the dog is ready to walk.
|
||||
stand_on_connect: bool = True
|
||||
|
||||
# Additional external cameras (standard LeRobot camera configs).
|
||||
cameras: dict[str, CameraConfig] = field(default_factory=dict)
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unitree Go2 quadruped (EDU) — high-level sport-mode integration.
|
||||
|
||||
Unlike :class:`~lerobot.robots.unitree_g1.UnitreeG1` (low-level joint
|
||||
control at 250 Hz through an on-robot ZMQ bridge), the Go2 is driven with
|
||||
sport-mode **body velocity commands** at tens of Hz, which work fine over
|
||||
plain DDS from any Linux host on the dog's network — no bridge server, no
|
||||
onboard companion computer.
|
||||
|
||||
Setup:
|
||||
1. Connect the host to the Go2 via Ethernet (dog is on 192.168.123.x)
|
||||
or put both on the same WiFi network.
|
||||
2. ``pip install unitree_sdk2py`` (Linux only — rides on cyclonedds).
|
||||
3. Find your interface name (``ip link``), then e.g.::
|
||||
|
||||
lerobot-teleoperate \
|
||||
--robot.type=unitree_go2 \
|
||||
--robot.network_interface=enp2s0 \
|
||||
--teleop.type=gamepad
|
||||
|
||||
Actions are body-frame velocities ``x.vel`` (forward, m/s), ``y.vel``
|
||||
(left, m/s), ``theta.vel`` (CCW yaw, rad/s) — the exact arguments of the
|
||||
SDK's ``SportClient.Move``. Observations are planar sport-mode odometry
|
||||
(``*.pos`` pose + ``*.vel`` body velocities) and the built-in front
|
||||
camera, plus any extra configured cameras.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from functools import cached_property
|
||||
from typing import Any
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from lerobot.cameras import make_cameras_from_configs
|
||||
from lerobot.types import RobotAction, RobotObservation
|
||||
from lerobot.utils.import_utils import require_package
|
||||
|
||||
from ..robot import Robot
|
||||
from .config_unitree_go2 import UnitreeGo2Config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DDS topic names follow Unitree SDK naming conventions
|
||||
SPORT_MODE_STATE_TOPIC = "rt/sportmodestate"
|
||||
|
||||
|
||||
class UnitreeGo2(Robot):
|
||||
"""LeRobot interface to a Unitree Go2 over unitree_sdk2py sport mode."""
|
||||
|
||||
config_class = UnitreeGo2Config
|
||||
name = "unitree_go2"
|
||||
|
||||
def __init__(self, config: UnitreeGo2Config):
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
self._cameras = make_cameras_from_configs(config.cameras)
|
||||
|
||||
# SDK handles — populated in connect(); the SDK import lives there
|
||||
# too so that configs, features and tests work on SDK-less hosts.
|
||||
self._sport = None
|
||||
self._video = None
|
||||
self._state_subscriber = None
|
||||
|
||||
self._state_lock = threading.Lock()
|
||||
self._latest_state = None # last SportModeState_ message
|
||||
self._connected = False
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Features
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@cached_property
|
||||
def _odom_ft(self) -> dict[str, type]:
|
||||
return {
|
||||
"x.pos": float,
|
||||
"y.pos": float,
|
||||
"theta.pos": float,
|
||||
"x.vel": float,
|
||||
"y.vel": float,
|
||||
"theta.vel": float,
|
||||
}
|
||||
|
||||
@property
|
||||
def _cameras_ft(self) -> dict[str, tuple]:
|
||||
ft: dict[str, tuple] = {}
|
||||
if self.config.use_front_camera:
|
||||
ft["front"] = (self.config.front_camera_height, self.config.front_camera_width, 3)
|
||||
for name, cam in self._cameras.items():
|
||||
ft[name] = (cam.height, cam.width, 3)
|
||||
return ft
|
||||
|
||||
@property
|
||||
def observation_features(self) -> dict:
|
||||
return {**self._odom_ft, **self._cameras_ft}
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
return {"x.vel": float, "y.vel": float, "theta.vel": float}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
if self._connected:
|
||||
return
|
||||
require_package("unitree-sdk2py", extra="unitree_go2", import_name="unitree_sdk2py")
|
||||
|
||||
from unitree_sdk2py.core.channel import ChannelFactoryInitialize, ChannelSubscriber
|
||||
from unitree_sdk2py.go2.sport.sport_client import SportClient
|
||||
from unitree_sdk2py.idl.unitree_go.msg.dds_ import SportModeState_
|
||||
|
||||
ChannelFactoryInitialize(self.config.domain_id, self.config.network_interface)
|
||||
|
||||
sport = SportClient()
|
||||
sport.SetTimeout(5.0)
|
||||
sport.Init()
|
||||
self._sport = sport
|
||||
|
||||
subscriber = ChannelSubscriber(SPORT_MODE_STATE_TOPIC, SportModeState_)
|
||||
subscriber.Init(self._on_sport_state, 10)
|
||||
self._state_subscriber = subscriber
|
||||
|
||||
if self.config.use_front_camera:
|
||||
from unitree_sdk2py.go2.video.video_client import VideoClient
|
||||
|
||||
video = VideoClient()
|
||||
video.SetTimeout(3.0)
|
||||
video.Init()
|
||||
self._video = video
|
||||
|
||||
for cam in self._cameras.values():
|
||||
cam.connect()
|
||||
|
||||
if self.config.stand_on_connect:
|
||||
self._sport.BalanceStand()
|
||||
|
||||
self._connected = True
|
||||
self.configure()
|
||||
logger.info(
|
||||
"%s connected (iface=%s, domain=%d)",
|
||||
self,
|
||||
self.config.network_interface,
|
||||
self.config.domain_id,
|
||||
)
|
||||
|
||||
def disconnect(self) -> None:
|
||||
if self._sport is not None:
|
||||
try:
|
||||
self._sport.StopMove()
|
||||
except Exception:
|
||||
logger.exception("StopMove on disconnect failed")
|
||||
for cam in self._cameras.values():
|
||||
try:
|
||||
cam.disconnect()
|
||||
except Exception:
|
||||
logger.exception("camera disconnect failed")
|
||||
self._sport = None
|
||||
self._video = None
|
||||
self._state_subscriber = None
|
||||
self._connected = False
|
||||
|
||||
# Sport mode needs no calibration.
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
return True
|
||||
|
||||
def calibrate(self) -> None:
|
||||
pass
|
||||
|
||||
def configure(self) -> None:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# I/O
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def get_observation(self) -> RobotObservation:
|
||||
if not self._connected:
|
||||
raise ConnectionError(f"{self} is not connected.")
|
||||
|
||||
obs: dict[str, Any] = dict.fromkeys(self._odom_ft, 0.0)
|
||||
with self._state_lock:
|
||||
state = self._latest_state
|
||||
if state is not None:
|
||||
obs["x.pos"] = float(state.position[0])
|
||||
obs["y.pos"] = float(state.position[1])
|
||||
obs["theta.pos"] = float(state.imu_state.rpy[2])
|
||||
obs["x.vel"] = float(state.velocity[0])
|
||||
obs["y.vel"] = float(state.velocity[1])
|
||||
obs["theta.vel"] = float(state.yaw_speed)
|
||||
|
||||
if self.config.use_front_camera:
|
||||
obs["front"] = self._read_front_camera()
|
||||
|
||||
for name, cam in self._cameras.items():
|
||||
obs[name] = cam.async_read()
|
||||
|
||||
return obs
|
||||
|
||||
def send_action(self, action: RobotAction) -> RobotAction:
|
||||
if not self._connected:
|
||||
raise ConnectionError(f"{self} is not connected.")
|
||||
|
||||
vx = float(np.clip(action.get("x.vel", 0.0), -self.config.max_x_vel, self.config.max_x_vel))
|
||||
vy = float(np.clip(action.get("y.vel", 0.0), -self.config.max_y_vel, self.config.max_y_vel))
|
||||
vyaw = float(
|
||||
np.clip(action.get("theta.vel", 0.0), -self.config.max_theta_vel, self.config.max_theta_vel)
|
||||
)
|
||||
|
||||
self._sport.Move(vx, vy, vyaw)
|
||||
return {"x.vel": vx, "y.vel": vy, "theta.vel": vyaw}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Internals
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _on_sport_state(self, msg) -> None:
|
||||
with self._state_lock:
|
||||
self._latest_state = msg
|
||||
|
||||
def _read_front_camera(self) -> np.ndarray:
|
||||
"""Fetch one frame from the built-in front camera (RGB, HxWx3)."""
|
||||
h, w = self.config.front_camera_height, self.config.front_camera_width
|
||||
code, data = self._video.GetImageSample()
|
||||
if code != 0 or data is None:
|
||||
logger.warning("front camera GetImageSample failed (code=%s)", code)
|
||||
return np.zeros((h, w, 3), dtype=np.uint8)
|
||||
frame = cv2.imdecode(np.frombuffer(bytes(data), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
if frame is None:
|
||||
logger.warning("front camera frame failed to decode")
|
||||
return np.zeros((h, w, 3), dtype=np.uint8)
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
if frame.shape[:2] != (h, w):
|
||||
frame = cv2.resize(frame, (w, h), interpolation=cv2.INTER_AREA)
|
||||
return frame
|
||||
Reference in New Issue
Block a user