mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-03 16:17:15 +00:00
Steven Palma
052d329470
* Add Foxglove display mode for teleoperate
Add a --display_mode flag (rerun|foxglove) to lerobot-teleoperate. When set
to foxglove, stream observations/actions over a Foxglove WebSocket server:
images as RawImage/CompressedImage, scalars as typed JSON channels with
schemas generated from the feature names (sanitized so paths don't need
quoting). Adds a `foxglove` extra.
* Add Foxglove display mode to lerobot-record
Wire the --display_mode flag (rerun|foxglove) into lerobot-record, matching
lerobot-teleoperate: route init/log through the backend-agnostic dispatchers
and stop the visualization backend on exit.
* update foxglove-sdk to 0.25.1
* Use static lerobot.Scalars schema for Foxglove state topics
Replace the per-topic JSON schema derived from feature names with a single
static lerobot.Scalars schema: a scalars array of {label, value} objects. The
same schema fits any robot regardless of which observation/action features it
reports, and the label field lets Foxglove name each series automatically so
one filtered path plots every feature.
* add foxglove option to dataset viz
* Make Foxglove dataset playback loop the sole frame emitter
Address review: the listener no longer emits frames, it only mutates
playback state and queues a one-shot seek index that the playback loop
services. The loop is now the only caller of emit_frame, so concurrent
random access into the on-disk dataset / video decoder never overlaps.
Also remove the dead server_holder and tighten the _foxglove_safe_name
docstring to state what it does and why.
* Label Foxglove dataset scalars with feature dimension names
Use the dataset's per-dimension feature names (e.g. joint names) as the
Foxglove series labels for /observation/state and /action/state instead
of bare indices. LeRobot stores `names` inconsistently (flat list,
{category: [...]}, or {name: index}), so _feature_dim_names handles each
and falls back to indices on any unknown format or length mismatch.
* Make Foxglove server host bindable and refactor topic/channel handling
Pass display_ip through as the Foxglove WebSocket bind host (127.0.0.1
for local only, 0.0.0.0 for all interfaces) instead of always binding
locally. In lerobot-dataset-viz, fold the separate --port into --web-port
so one flag covers both the Rerun web viewer and the Foxglove server port.
Add a _foxglove_topic() helper and thread a per-topic channel cache
through the log helpers so dataset playback stays self-contained instead
of mutating the module-global cache. Promote SUCCESS to constants.py.
* feat(viz): add support for foxglove in rollout + add to viz tag
* fix(docs): remove misleading installation note
* fix(visualization): no duplicated prefix, consolidated norm + warnings log
* chore(viz): minor improvements
* refactor(viz): split files + autoplay + updated docs + added minimal tests
* fix(viz): right tags + warning
* feat(deprecated ws-port): removing rerun's depreacted ws-port parameter in dataset visualization
* chore(web ports): adding global variables for default foxglove/rerun web ports
* feat(depth): adding depth support to foxglove visualizer. Because of foxglove limitations (min and max values on RawImage cannot be set from the SDK), depth is normalized between [0,1] when a depth range is provided.
* fix(rerun depth range): making rerun depth range computation safe against missing stats
* chore(foxglove depth): make it simple, and make it work.
* fix(scaling): fixing depth frames scaling
---------
Co-authored-by: Roman Shtylman <roman@foxglove.dev>
Co-authored-by: Caroline Pascal <caroline8.pascal@gmail.com>
80 lines
3.2 KiB
Python
80 lines
3.2 KiB
Python
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Backend-agnostic visualization dispatch.
|
|
|
|
Selects a visualization backend at runtime via a display-mode string (e.g. a ``--display_mode`` CLI
|
|
flag) so callers never branch on the backend. The concrete implementations live in
|
|
:mod:`lerobot.utils.rerun_visualization` and :mod:`lerobot.utils.foxglove_visualization`; importing
|
|
this module does not import ``rerun`` or ``foxglove`` (each backend imports its SDK lazily behind a
|
|
``require_package`` guard).
|
|
"""
|
|
|
|
from lerobot.types import RobotAction, RobotObservation
|
|
|
|
from .foxglove_visualization import init_foxglove, log_foxglove_data, shutdown_foxglove
|
|
from .rerun_visualization import init_rerun, log_rerun_data, shutdown_rerun
|
|
|
|
# Visualization backends selectable at runtime via a display-mode string (e.g. a --display_mode flag).
|
|
VISUALIZATION_MODES = ("rerun", "foxglove")
|
|
|
|
|
|
def init_visualization(
|
|
display_mode: str,
|
|
*,
|
|
session_name: str = "lerobot_control_loop",
|
|
ip: str | None = None,
|
|
port: int | None = None,
|
|
) -> None:
|
|
"""Initializes the visualization backend selected by ``display_mode``.
|
|
|
|
For ``"rerun"``, ``ip``/``port`` point at an optional remote Rerun server. For ``"foxglove"``,
|
|
``ip`` is the interface to bind the WebSocket server to (``127.0.0.1`` for local only, ``0.0.0.0``
|
|
for all interfaces) and ``port`` is its port.
|
|
"""
|
|
|
|
if display_mode == "rerun":
|
|
init_rerun(session_name=session_name, ip=ip, port=port)
|
|
elif display_mode == "foxglove":
|
|
init_foxglove(host=ip or "127.0.0.1", port=port)
|
|
else:
|
|
raise ValueError(f"Unknown display_mode '{display_mode}'. Expected one of {VISUALIZATION_MODES}.")
|
|
|
|
|
|
def log_visualization_data(
|
|
display_mode: str,
|
|
observation: RobotObservation | None = None,
|
|
action: RobotAction | None = None,
|
|
compress_images: bool = False,
|
|
) -> None:
|
|
"""Logs observation/action data to the backend selected by ``display_mode``."""
|
|
|
|
if display_mode == "rerun":
|
|
log_rerun_data(observation=observation, action=action, compress_images=compress_images)
|
|
elif display_mode == "foxglove":
|
|
log_foxglove_data(observation=observation, action=action, compress_images=compress_images)
|
|
else:
|
|
raise ValueError(f"Unknown display_mode '{display_mode}'. Expected one of {VISUALIZATION_MODES}.")
|
|
|
|
|
|
def shutdown_visualization(display_mode: str) -> None:
|
|
"""Shuts down the backend selected by ``display_mode``."""
|
|
|
|
if display_mode == "rerun":
|
|
shutdown_rerun()
|
|
elif display_mode == "foxglove":
|
|
shutdown_foxglove()
|
|
else:
|
|
raise ValueError(f"Unknown display_mode '{display_mode}'. Expected one of {VISUALIZATION_MODES}.")
|