admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-03 08:07:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(viz): add support for foxglove in rollout + add to viz tag

Browse Source
This commit is contained in:
Steven Palma
2026-07-01 11:25:49 +02:00
parent b77d0cd40c
commit 70b2cf2b0a
8 changed files with 39 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+1 -3
View File
@@ -125,9 +125,7 @@ hardware = [
]
viz = [
"rerun-sdk>=0.24.0,<0.34.0",
]
foxglove = [
"foxglove-sdk>=0.25.1,<1.0.0",
"foxglove-sdk>=0.25.1,<0.26.0",
]
# ── User-facing composite extras (map to CLI scripts) ─────
# lerobot-record, lerobot-replay, lerobot-calibrate, lerobot-teleoperate, etc.
src/lerobot/rollout/configs.py
+6 -3
View File
@@ -226,11 +226,14 @@ class RolloutConfig:
device: str | None = None
task: str = ""
display_data: bool = False
# Display data on a remote Rerun server
# Visualization backend used when display_data is True: "rerun" or "foxglove".
display_mode: str = "rerun"
# For "rerun": IP of a remote server to send to. For "foxglove": interface to bind the WebSocket
# server to (127.0.0.1 for local only, 0.0.0.0 for all interfaces).
display_ip: str | None = None
# Port of the remote Rerun server
# For "rerun": port of the remote server. For "foxglove": port to bind the WebSocket server to.
display_port: int | None = None
# Whether to display compressed images in Rerun
# Whether to display compressed (JPEG) images instead of raw frames
display_compressed_images: bool = False
# Use vocal synthesis to read events
play_sounds: bool = True
src/lerobot/rollout/strategies/core.py
+4 -3
View File
@@ -26,7 +26,7 @@ from lerobot.utils.action_interpolator import ActionInterpolator
from lerobot.utils.constants import OBS_STR
from lerobot.utils.feature_utils import build_dataset_frame
from lerobot.utils.robot_utils import precise_sleep
from lerobot.utils.visualization_utils import log_rerun_data
from lerobot.utils.visualization_utils import log_visualization_data
from ..inference import InferenceEngine
@@ -162,11 +162,12 @@ class RolloutStrategy(abc.ABC):
action_dict: dict | None,
runtime_ctx: RuntimeContext,
) -> None:
"""Log observation/action telemetry to Rerun if display_data is enabled."""
"""Log observation/action telemetry to the visualization backend if display_data is enabled."""
cfg = runtime_ctx.cfg
if not cfg.display_data:
return
log_rerun_data(
log_visualization_data(
cfg.display_mode,
observation=obs_processed,
action=action_dict,
compress_images=cfg.display_compressed_images,
src/lerobot/rollout/strategies/episodic.py
+5 -2
View File
@@ -44,7 +44,7 @@ from lerobot.utils.feature_utils import build_dataset_frame
from lerobot.utils.keyboard_input import init_keyboard_listener
from lerobot.utils.robot_utils import precise_sleep
from lerobot.utils.utils import log_say
from lerobot.utils.visualization_utils import log_rerun_data
from lerobot.utils.visualization_utils import log_visualization_data
from ..configs import EpisodicStrategyConfig
from ..context import RolloutContext
@@ -171,6 +171,7 @@ class EpisodicStrategy(RolloutStrategy):
fps=fps,
control_time_s=reset_time_s,
display_data=cfg.display_data,
display_mode=cfg.display_mode,
display_compressed=display_compressed,
)
@@ -259,6 +260,7 @@ class EpisodicStrategy(RolloutStrategy):
fps: float,
control_time_s: float,
display_data: bool,
display_mode: str,
display_compressed: bool,
) -> None:
"""Reset-phase loop: teleop drives the robot if available, no recording."""
@@ -288,7 +290,8 @@ class EpisodicStrategy(RolloutStrategy):
if display_data:
obs_processed = processors.robot_observation_processor(obs)
log_rerun_data(
log_visualization_data(
display_mode,
observation=obs_processed,
action=act_teleop,
compress_images=display_compressed,
src/lerobot/scripts/lerobot_dataset_viz.py
+1
View File
@@ -70,6 +70,7 @@ local$ lerobot-dataset-viz \
```
This starts a Foxglove WebSocket server that serves the episode on demand from the on-disk dataset,
so you can play/pause and scrub anywhere in the episode using Foxglove's playback controls.
Requires the Foxglove extra: ``pip install 'lerobot[foxglove]'``.
"""
src/lerobot/scripts/lerobot_rollout.py
+13 -3
View File
@@ -145,6 +145,9 @@ Usage examples
--dataset.rgb_encoder.vcodec=h264 \\
--dataset.rgb_encoder.preset=fast \\
--dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2}
# Stream to Foxglove instead of Rerun:
# add --display_mode=foxglove, then connect the Foxglove app to ws://127.0.0.1:8765.
"""
import logging
@@ -190,7 +193,7 @@ from lerobot.teleoperators import ( # noqa: F401
from lerobot.utils.import_utils import register_third_party_plugins
from lerobot.utils.process import ProcessSignalHandler
from lerobot.utils.utils import init_logging
from lerobot.utils.visualization_utils import init_rerun
from lerobot.utils.visualization_utils import init_visualization, shutdown_visualization
logger = logging.getLogger(__name__)
@@ -201,8 +204,13 @@ def rollout(cfg: RolloutConfig):
init_logging()
if cfg.display_data:
logger.info("Initializing Rerun visualization (ip=%s, port=%s)", cfg.display_ip, cfg.display_port)
init_rerun(session_name="rollout", ip=cfg.display_ip, port=cfg.display_port)
logger.info(
"Initializing %s visualization (ip=%s, port=%s)",
cfg.display_mode,
cfg.display_ip,
cfg.display_port,
)
init_visualization(cfg.display_mode, session_name="rollout", ip=cfg.display_ip, port=cfg.display_port)
signal_handler = ProcessSignalHandler(use_threads=True, display_pid=False)
shutdown_event = signal_handler.shutdown_event
@@ -227,6 +235,8 @@ def rollout(cfg: RolloutConfig):
logger.info("Interrupted by user")
finally:
strategy.teardown(ctx)
if cfg.display_data:
shutdown_visualization(cfg.display_mode)
logger.info("Rollout finished")
src/lerobot/scripts/lerobot_teleoperate.py
+2 -2
View File
@@ -31,8 +31,8 @@ lerobot-teleoperate \
--display_data=true
```
To stream the data to Foxglove instead of Rerun, add ``--display_mode=foxglove`` (then connect the
Foxglove app to ``ws://127.0.0.1:8765``; override the port with ``--display_port=<port>``):
To stream the data to Foxglove instead of Rerun, add ``--display_mode=foxglove``
(then connect the Foxglove app to ``ws://127.0.0.1:8765``; override the port with ``--display_port=<port>``):
```shell
lerobot-teleoperate \
uv.lock
Generated
+7 -6
View File
@@ -1,5 +1,5 @@
version = 1
revision = 2
revision = 3
requires-python = ">=3.12"
resolution-markers = [
"(python_full_version >= '3.15' and platform_machine == 'AMD64' and sys_platform == 'linux') or (python_full_version >= '3.15' and platform_machine == 'x86_64' and sys_platform == 'linux')",
@@ -2831,6 +2831,7 @@ all = [
{ name = "faker" },
{ name = "fastapi" },
{ name = "feetech-servo-sdk" },
{ name = "foxglove-sdk" },
{ name = "grpcio" },
{ name = "grpcio-tools" },
{ name = "gym-aloha" },
@@ -2915,6 +2916,7 @@ core-scripts = [
{ name = "av" },
{ name = "datasets" },
{ name = "deepdiff" },
{ name = "foxglove-sdk" },
{ name = "jsonlines" },
{ name = "pandas" },
{ name = "pyarrow" },
@@ -2937,6 +2939,7 @@ dataset = [
dataset-viz = [
{ name = "av" },
{ name = "datasets" },
{ name = "foxglove-sdk" },
{ name = "jsonlines" },
{ name = "pandas" },
{ name = "pyarrow" },
@@ -2980,9 +2983,6 @@ feetech = [
{ name = "feetech-servo-sdk" },
{ name = "pyserial" },
]
foxglove = [
{ name = "foxglove-sdk" },
]
gamepad = [
{ name = "hidapi" },
{ name = "pygame" },
@@ -3206,6 +3206,7 @@ video-benchmark = [
{ name = "scikit-image" },
]
viz = [
{ name = "foxglove-sdk" },
{ name = "rerun-sdk" },
]
vla-jepa = [
@@ -3245,7 +3246,7 @@ requires-dist = [
{ name = "fastapi", marker = "extra == 'phone'", specifier = "<1.0" },
{ name = "feetech-servo-sdk", marker = "extra == 'feetech'", specifier = ">=1.0.0,<2.0.0" },
{ name = "flash-attn", marker = "sys_platform != 'darwin' and extra == 'groot'", specifier = ">=2.5.9,<3.0.0" },
{ name = "foxglove-sdk", marker = "extra == 'foxglove'", specifier = ">=0.25.1,<1.0.0" },
{ name = "foxglove-sdk", marker = "extra == 'viz'", specifier = ">=0.25.1,<0.26.0" },
{ name = "grpcio", marker = "extra == 'grpcio-dep'", specifier = ">=1.73.1,<2.0.0" },
{ name = "grpcio", marker = "extra == 'reachy2'", specifier = "<=1.73.1" },
{ name = "grpcio-tools", marker = "extra == 'dev'", specifier = ">=1.73.1,<2.0.0" },
@@ -3441,7 +3442,7 @@ requires-dist = [
{ name = "transformers", marker = "extra == 'transformers-dep'", specifier = ">=5.4.0,<5.6.0" },
{ name = "wandb", marker = "extra == 'training'", specifier = ">=0.24.0,<0.28.0" },
]
provides-extras = ["dataset", "training", "hardware", "viz", "foxglove", "core-scripts", "evaluation", "dataset-viz", "av-dep", "pygame-dep", "placo-dep", "transformers-dep", "grpcio-dep", "accelerate-dep", "can-dep", "peft-dep", "scipy-dep", "diffusers-dep", "qwen-vl-utils-dep", "matplotlib-dep", "pyserial-dep", "deepdiff-dep", "pynput-dep", "pyzmq-dep", "motorbridge-dep", "motorbridge-smart-servo-dep", "feetech", "dynamixel", "damiao", "robstride", "openarms", "gamepad", "hopejr", "lekiwi", "unitree-g1", "reachy2", "rebot", "kinematics", "intelrealsense", "phone", "diffusion", "wallx", "pi", "molmoact2", "smolvla", "multi-task-dit", "groot", "sarm", "robometer", "topreward", "xvla", "eo1", "hilserl", "vla-jepa", "async", "peft", "annotations", "dev", "notebook", "test", "video-benchmark", "aloha", "pusht", "libero", "metaworld", "all"]
provides-extras = ["dataset", "training", "hardware", "viz", "core-scripts", "evaluation", "dataset-viz", "av-dep", "pygame-dep", "placo-dep", "transformers-dep", "grpcio-dep", "accelerate-dep", "can-dep", "peft-dep", "scipy-dep", "diffusers-dep", "qwen-vl-utils-dep", "matplotlib-dep", "pyserial-dep", "deepdiff-dep", "pynput-dep", "pyzmq-dep", "motorbridge-dep", "motorbridge-smart-servo-dep", "feetech", "dynamixel", "damiao", "robstride", "openarms", "gamepad", "hopejr", "lekiwi", "unitree-g1", "reachy2", "rebot", "kinematics", "intelrealsense", "phone", "diffusion", "wallx", "pi", "molmoact2", "smolvla", "multi-task-dit", "groot", "sarm", "robometer", "topreward", "xvla", "eo1", "hilserl", "vla-jepa", "async", "peft", "annotations", "dev", "notebook", "test", "video-benchmark", "aloha", "pusht", "libero", "metaworld", "all"]
[[package]]
name = "librt"