diff --git a/pyproject.toml b/pyproject.toml index 5b3ba9b53..b61f20933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,9 +125,7 @@ hardware = [ ] viz = [ "rerun-sdk>=0.24.0,<0.34.0", -] -foxglove = [ - "foxglove-sdk>=0.25.1,<1.0.0", + "foxglove-sdk>=0.25.1,<0.26.0", ] # ── User-facing composite extras (map to CLI scripts) ───── # lerobot-record, lerobot-replay, lerobot-calibrate, lerobot-teleoperate, etc. diff --git a/src/lerobot/rollout/configs.py b/src/lerobot/rollout/configs.py index 60c47cfba..639e2ba29 100644 --- a/src/lerobot/rollout/configs.py +++ b/src/lerobot/rollout/configs.py @@ -226,11 +226,14 @@ class RolloutConfig: device: str | None = None task: str = "" display_data: bool = False - # Display data on a remote Rerun server + # Visualization backend used when display_data is True: "rerun" or "foxglove". + display_mode: str = "rerun" + # For "rerun": IP of a remote server to send to. For "foxglove": interface to bind the WebSocket + # server to (127.0.0.1 for local only, 0.0.0.0 for all interfaces). display_ip: str | None = None - # Port of the remote Rerun server + # For "rerun": port of the remote server. For "foxglove": port to bind the WebSocket server to. display_port: int | None = None - # Whether to display compressed images in Rerun + # Whether to display compressed (JPEG) images instead of raw frames display_compressed_images: bool = False # Use vocal synthesis to read events play_sounds: bool = True diff --git a/src/lerobot/rollout/strategies/core.py b/src/lerobot/rollout/strategies/core.py index 9c897522f..460ad12e5 100644 --- a/src/lerobot/rollout/strategies/core.py +++ b/src/lerobot/rollout/strategies/core.py @@ -26,7 +26,7 @@ from lerobot.utils.action_interpolator import ActionInterpolator from lerobot.utils.constants import OBS_STR from lerobot.utils.feature_utils import build_dataset_frame from lerobot.utils.robot_utils import precise_sleep -from lerobot.utils.visualization_utils import log_rerun_data +from lerobot.utils.visualization_utils import log_visualization_data from ..inference import InferenceEngine @@ -162,11 +162,12 @@ class RolloutStrategy(abc.ABC): action_dict: dict | None, runtime_ctx: RuntimeContext, ) -> None: - """Log observation/action telemetry to Rerun if display_data is enabled.""" + """Log observation/action telemetry to the visualization backend if display_data is enabled.""" cfg = runtime_ctx.cfg if not cfg.display_data: return - log_rerun_data( + log_visualization_data( + cfg.display_mode, observation=obs_processed, action=action_dict, compress_images=cfg.display_compressed_images, diff --git a/src/lerobot/rollout/strategies/episodic.py b/src/lerobot/rollout/strategies/episodic.py index e70e66787..15b9bb971 100644 --- a/src/lerobot/rollout/strategies/episodic.py +++ b/src/lerobot/rollout/strategies/episodic.py @@ -44,7 +44,7 @@ from lerobot.utils.feature_utils import build_dataset_frame from lerobot.utils.keyboard_input import init_keyboard_listener from lerobot.utils.robot_utils import precise_sleep from lerobot.utils.utils import log_say -from lerobot.utils.visualization_utils import log_rerun_data +from lerobot.utils.visualization_utils import log_visualization_data from ..configs import EpisodicStrategyConfig from ..context import RolloutContext @@ -171,6 +171,7 @@ class EpisodicStrategy(RolloutStrategy): fps=fps, control_time_s=reset_time_s, display_data=cfg.display_data, + display_mode=cfg.display_mode, display_compressed=display_compressed, ) @@ -259,6 +260,7 @@ class EpisodicStrategy(RolloutStrategy): fps: float, control_time_s: float, display_data: bool, + display_mode: str, display_compressed: bool, ) -> None: """Reset-phase loop: teleop drives the robot if available, no recording.""" @@ -288,7 +290,8 @@ class EpisodicStrategy(RolloutStrategy): if display_data: obs_processed = processors.robot_observation_processor(obs) - log_rerun_data( + log_visualization_data( + display_mode, observation=obs_processed, action=act_teleop, compress_images=display_compressed, diff --git a/src/lerobot/scripts/lerobot_dataset_viz.py b/src/lerobot/scripts/lerobot_dataset_viz.py index 2c94736ee..2f1e978af 100644 --- a/src/lerobot/scripts/lerobot_dataset_viz.py +++ b/src/lerobot/scripts/lerobot_dataset_viz.py @@ -70,6 +70,7 @@ local$ lerobot-dataset-viz \ ``` This starts a Foxglove WebSocket server that serves the episode on demand from the on-disk dataset, so you can play/pause and scrub anywhere in the episode using Foxglove's playback controls. +Requires the Foxglove extra: ``pip install 'lerobot[foxglove]'``. """ diff --git a/src/lerobot/scripts/lerobot_rollout.py b/src/lerobot/scripts/lerobot_rollout.py index daee87bbe..879070721 100644 --- a/src/lerobot/scripts/lerobot_rollout.py +++ b/src/lerobot/scripts/lerobot_rollout.py @@ -145,6 +145,9 @@ Usage examples --dataset.rgb_encoder.vcodec=h264 \\ --dataset.rgb_encoder.preset=fast \\ --dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} + + # Stream to Foxglove instead of Rerun: + # add --display_mode=foxglove, then connect the Foxglove app to ws://127.0.0.1:8765. """ import logging @@ -190,7 +193,7 @@ from lerobot.teleoperators import ( # noqa: F401 from lerobot.utils.import_utils import register_third_party_plugins from lerobot.utils.process import ProcessSignalHandler from lerobot.utils.utils import init_logging -from lerobot.utils.visualization_utils import init_rerun +from lerobot.utils.visualization_utils import init_visualization, shutdown_visualization logger = logging.getLogger(__name__) @@ -201,8 +204,13 @@ def rollout(cfg: RolloutConfig): init_logging() if cfg.display_data: - logger.info("Initializing Rerun visualization (ip=%s, port=%s)", cfg.display_ip, cfg.display_port) - init_rerun(session_name="rollout", ip=cfg.display_ip, port=cfg.display_port) + logger.info( + "Initializing %s visualization (ip=%s, port=%s)", + cfg.display_mode, + cfg.display_ip, + cfg.display_port, + ) + init_visualization(cfg.display_mode, session_name="rollout", ip=cfg.display_ip, port=cfg.display_port) signal_handler = ProcessSignalHandler(use_threads=True, display_pid=False) shutdown_event = signal_handler.shutdown_event @@ -227,6 +235,8 @@ def rollout(cfg: RolloutConfig): logger.info("Interrupted by user") finally: strategy.teardown(ctx) + if cfg.display_data: + shutdown_visualization(cfg.display_mode) logger.info("Rollout finished") diff --git a/src/lerobot/scripts/lerobot_teleoperate.py b/src/lerobot/scripts/lerobot_teleoperate.py index 0c7f49010..30f13987e 100644 --- a/src/lerobot/scripts/lerobot_teleoperate.py +++ b/src/lerobot/scripts/lerobot_teleoperate.py @@ -31,8 +31,8 @@ lerobot-teleoperate \ --display_data=true ``` -To stream the data to Foxglove instead of Rerun, add ``--display_mode=foxglove`` (then connect the -Foxglove app to ``ws://127.0.0.1:8765``; override the port with ``--display_port=``): +To stream the data to Foxglove instead of Rerun, add ``--display_mode=foxglove`` +(then connect the Foxglove app to ``ws://127.0.0.1:8765``; override the port with ``--display_port=``): ```shell lerobot-teleoperate \ diff --git a/uv.lock b/uv.lock index 2b784cab9..bd60319cf 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "(python_full_version >= '3.15' and platform_machine == 'AMD64' and sys_platform == 'linux') or (python_full_version >= '3.15' and platform_machine == 'x86_64' and sys_platform == 'linux')", @@ -2831,6 +2831,7 @@ all = [ { name = "faker" }, { name = "fastapi" }, { name = "feetech-servo-sdk" }, + { name = "foxglove-sdk" }, { name = "grpcio" }, { name = "grpcio-tools" }, { name = "gym-aloha" }, @@ -2915,6 +2916,7 @@ core-scripts = [ { name = "av" }, { name = "datasets" }, { name = "deepdiff" }, + { name = "foxglove-sdk" }, { name = "jsonlines" }, { name = "pandas" }, { name = "pyarrow" }, @@ -2937,6 +2939,7 @@ dataset = [ dataset-viz = [ { name = "av" }, { name = "datasets" }, + { name = "foxglove-sdk" }, { name = "jsonlines" }, { name = "pandas" }, { name = "pyarrow" }, @@ -2980,9 +2983,6 @@ feetech = [ { name = "feetech-servo-sdk" }, { name = "pyserial" }, ] -foxglove = [ - { name = "foxglove-sdk" }, -] gamepad = [ { name = "hidapi" }, { name = "pygame" }, @@ -3206,6 +3206,7 @@ video-benchmark = [ { name = "scikit-image" }, ] viz = [ + { name = "foxglove-sdk" }, { name = "rerun-sdk" }, ] vla-jepa = [ @@ -3245,7 +3246,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'phone'", specifier = "<1.0" }, { name = "feetech-servo-sdk", marker = "extra == 'feetech'", specifier = ">=1.0.0,<2.0.0" }, { name = "flash-attn", marker = "sys_platform != 'darwin' and extra == 'groot'", specifier = ">=2.5.9,<3.0.0" }, - { name = "foxglove-sdk", marker = "extra == 'foxglove'", specifier = ">=0.25.1,<1.0.0" }, + { name = "foxglove-sdk", marker = "extra == 'viz'", specifier = ">=0.25.1,<0.26.0" }, { name = "grpcio", marker = "extra == 'grpcio-dep'", specifier = ">=1.73.1,<2.0.0" }, { name = "grpcio", marker = "extra == 'reachy2'", specifier = "<=1.73.1" }, { name = "grpcio-tools", marker = "extra == 'dev'", specifier = ">=1.73.1,<2.0.0" }, @@ -3441,7 +3442,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'transformers-dep'", specifier = ">=5.4.0,<5.6.0" }, { name = "wandb", marker = "extra == 'training'", specifier = ">=0.24.0,<0.28.0" }, ] -provides-extras = ["dataset", "training", "hardware", "viz", "foxglove", "core-scripts", "evaluation", "dataset-viz", "av-dep", "pygame-dep", "placo-dep", "transformers-dep", "grpcio-dep", "accelerate-dep", "can-dep", "peft-dep", "scipy-dep", "diffusers-dep", "qwen-vl-utils-dep", "matplotlib-dep", "pyserial-dep", "deepdiff-dep", "pynput-dep", "pyzmq-dep", "motorbridge-dep", "motorbridge-smart-servo-dep", "feetech", "dynamixel", "damiao", "robstride", "openarms", "gamepad", "hopejr", "lekiwi", "unitree-g1", "reachy2", "rebot", "kinematics", "intelrealsense", "phone", "diffusion", "wallx", "pi", "molmoact2", "smolvla", "multi-task-dit", "groot", "sarm", "robometer", "topreward", "xvla", "eo1", "hilserl", "vla-jepa", "async", "peft", "annotations", "dev", "notebook", "test", "video-benchmark", "aloha", "pusht", "libero", "metaworld", "all"] +provides-extras = ["dataset", "training", "hardware", "viz", "core-scripts", "evaluation", "dataset-viz", "av-dep", "pygame-dep", "placo-dep", "transformers-dep", "grpcio-dep", "accelerate-dep", "can-dep", "peft-dep", "scipy-dep", "diffusers-dep", "qwen-vl-utils-dep", "matplotlib-dep", "pyserial-dep", "deepdiff-dep", "pynput-dep", "pyzmq-dep", "motorbridge-dep", "motorbridge-smart-servo-dep", "feetech", "dynamixel", "damiao", "robstride", "openarms", "gamepad", "hopejr", "lekiwi", "unitree-g1", "reachy2", "rebot", "kinematics", "intelrealsense", "phone", "diffusion", "wallx", "pi", "molmoact2", "smolvla", "multi-task-dit", "groot", "sarm", "robometer", "topreward", "xvla", "eo1", "hilserl", "vla-jepa", "async", "peft", "annotations", "dev", "notebook", "test", "video-benchmark", "aloha", "pusht", "libero", "metaworld", "all"] [[package]] name = "librt"