mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-03 16:17:15 +00:00
Steven Palma
052d329470
* Add Foxglove display mode for teleoperate
Add a --display_mode flag (rerun|foxglove) to lerobot-teleoperate. When set
to foxglove, stream observations/actions over a Foxglove WebSocket server:
images as RawImage/CompressedImage, scalars as typed JSON channels with
schemas generated from the feature names (sanitized so paths don't need
quoting). Adds a `foxglove` extra.
* Add Foxglove display mode to lerobot-record
Wire the --display_mode flag (rerun|foxglove) into lerobot-record, matching
lerobot-teleoperate: route init/log through the backend-agnostic dispatchers
and stop the visualization backend on exit.
* update foxglove-sdk to 0.25.1
* Use static lerobot.Scalars schema for Foxglove state topics
Replace the per-topic JSON schema derived from feature names with a single
static lerobot.Scalars schema: a scalars array of {label, value} objects. The
same schema fits any robot regardless of which observation/action features it
reports, and the label field lets Foxglove name each series automatically so
one filtered path plots every feature.
* add foxglove option to dataset viz
* Make Foxglove dataset playback loop the sole frame emitter
Address review: the listener no longer emits frames, it only mutates
playback state and queues a one-shot seek index that the playback loop
services. The loop is now the only caller of emit_frame, so concurrent
random access into the on-disk dataset / video decoder never overlaps.
Also remove the dead server_holder and tighten the _foxglove_safe_name
docstring to state what it does and why.
* Label Foxglove dataset scalars with feature dimension names
Use the dataset's per-dimension feature names (e.g. joint names) as the
Foxglove series labels for /observation/state and /action/state instead
of bare indices. LeRobot stores `names` inconsistently (flat list,
{category: [...]}, or {name: index}), so _feature_dim_names handles each
and falls back to indices on any unknown format or length mismatch.
* Make Foxglove server host bindable and refactor topic/channel handling
Pass display_ip through as the Foxglove WebSocket bind host (127.0.0.1
for local only, 0.0.0.0 for all interfaces) instead of always binding
locally. In lerobot-dataset-viz, fold the separate --port into --web-port
so one flag covers both the Rerun web viewer and the Foxglove server port.
Add a _foxglove_topic() helper and thread a per-topic channel cache
through the log helpers so dataset playback stays self-contained instead
of mutating the module-global cache. Promote SUCCESS to constants.py.
* feat(viz): add support for foxglove in rollout + add to viz tag
* fix(docs): remove misleading installation note
* fix(visualization): no duplicated prefix, consolidated norm + warnings log
* chore(viz): minor improvements
* refactor(viz): split files + autoplay + updated docs + added minimal tests
* fix(viz): right tags + warning
* feat(deprecated ws-port): removing rerun's depreacted ws-port parameter in dataset visualization
* chore(web ports): adding global variables for default foxglove/rerun web ports
* feat(depth): adding depth support to foxglove visualizer. Because of foxglove limitations (min and max values on RawImage cannot be set from the SDK), depth is normalized between [0,1] when a depth range is provided.
* fix(rerun depth range): making rerun depth range computation safe against missing stats
* chore(foxglove depth): make it simple, and make it work.
* fix(scaling): fixing depth frames scaling
---------
Co-authored-by: Roman Shtylman <roman@foxglove.dev>
Co-authored-by: Caroline Pascal <caroline8.pascal@gmail.com>
102 lines
4.5 KiB
Python
102 lines
4.5 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Tests for the Foxglove backend's pure helpers.
|
|
|
|
These cover topic naming, series labelling and feature-name parsing. They import
|
|
``foxglove_visualization`` directly and need NO ``foxglove`` extra: the SDK is imported lazily inside
|
|
the functions that talk to the server, so the helpers below run in the base test tier.
|
|
"""
|
|
|
|
import numpy as np
|
|
|
|
from lerobot.utils import foxglove_visualization as fv
|
|
from lerobot.utils.constants import ACTION, OBS_STATE
|
|
|
|
|
|
def test_foxglove_safe_name_collapses_dots():
|
|
assert fv._foxglove_safe_name("observation.images.front") == "observation_images_front"
|
|
assert fv._foxglove_safe_name("plain") == "plain"
|
|
|
|
|
|
def test_foxglove_topic_image_strips_prefix_without_doubling_images():
|
|
# Fully-qualified camera key -> single clean segment (no doubled "images").
|
|
assert fv._foxglove_topic("observation.images.front", is_image=True) == "/observation/images/front"
|
|
# A nested camera name keeps its structure via safe-name collapsing.
|
|
assert (
|
|
fv._foxglove_topic("observation.images.wrist.left", is_image=True) == "/observation/images/wrist_left"
|
|
)
|
|
# Bare camera name (as real robots emit).
|
|
assert fv._foxglove_topic("front", is_image=True) == "/observation/images/front"
|
|
|
|
|
|
def test_foxglove_topic_scalar_sources():
|
|
assert fv._foxglove_topic(OBS_STATE) == "/observation/state"
|
|
assert fv._foxglove_topic("observation.environment_state") == "/observation/state"
|
|
assert fv._foxglove_topic(ACTION) == "/action/state"
|
|
assert fv._foxglove_topic("action.delta") == "/action/state"
|
|
|
|
|
|
def test_labeled_scalars_uses_labels_then_index_fallback():
|
|
assert fv._labeled_scalars("state", np.array([1.0, 2.0, 3.0])) == {
|
|
"state_0": 1.0,
|
|
"state_1": 2.0,
|
|
"state_2": 3.0,
|
|
}
|
|
assert fv._labeled_scalars("state", [1.0, 2.0], ["pan", "lift"]) == {"pan": 1.0, "lift": 2.0}
|
|
# Wrong-length labels fall back to index naming (never silently mislabels).
|
|
assert fv._labeled_scalars("q", [1.0, 2.0], ["only_one"]) == {"q_0": 1.0, "q_1": 2.0}
|
|
|
|
|
|
def test_frame_to_scalars_matches_live_labeling_and_handles_scalar():
|
|
frame = {OBS_STATE: np.array([1.0, 2.0])}
|
|
# No metadata -> {short_name}_{i}, identical to the live-stream fallback.
|
|
assert fv._frame_to_scalars(frame, OBS_STATE) == fv._labeled_scalars("state", np.array([1.0, 2.0]))
|
|
assert fv._frame_to_scalars(frame, OBS_STATE) == {"state_0": 1.0, "state_1": 2.0}
|
|
# Metadata labels are honored.
|
|
assert fv._frame_to_scalars(frame, OBS_STATE, ["pan", "lift"]) == {"pan": 1.0, "lift": 2.0}
|
|
# A 0-d scalar becomes a single entry named by the short feature name.
|
|
assert fv._frame_to_scalars({ACTION: np.array(5.0)}, ACTION) == {"action": 5.0}
|
|
# A missing feature yields an empty mapping.
|
|
assert fv._frame_to_scalars({}, OBS_STATE) == {}
|
|
|
|
|
|
def test_feature_dim_names_formats():
|
|
# Flat list of names.
|
|
assert fv._feature_dim_names({"shape": [2], "names": ["x", "y"]}) == ["x", "y"]
|
|
# Category mapping (dict of lists).
|
|
assert fv._feature_dim_names({"shape": [2], "names": {"motors": ["m0", "m1"]}}) == ["m0", "m1"]
|
|
# name -> index mapping (returned sorted by index).
|
|
assert fv._feature_dim_names({"shape": [2], "names": {"delta_x": 0, "delta_y": 1}}) == [
|
|
"delta_x",
|
|
"delta_y",
|
|
]
|
|
# Bool values must NOT be treated as an index map (bool is a subclass of int).
|
|
assert fv._feature_dim_names({"shape": [2], "names": {"a": True, "b": False}}) is None
|
|
# Mismatched length -> None (won't silently mislabel).
|
|
assert fv._feature_dim_names({"shape": [3], "names": ["x", "y"]}) is None
|
|
# Missing / absent names -> None.
|
|
assert fv._feature_dim_names(None) is None
|
|
assert fv._feature_dim_names({"shape": [2]}) is None
|
|
|
|
|
|
def test_is_scalar():
|
|
assert fv._is_scalar(1.0)
|
|
assert fv._is_scalar(np.float32(2.0))
|
|
assert fv._is_scalar(np.array(3.0)) # 0-d array
|
|
assert not fv._is_scalar(np.array([1.0, 2.0]))
|
|
assert not fv._is_scalar("x")
|