faster canbus

.github/workflows/fast_tests.yml

@@ -60,17 +60,12 @@ jobs:
     runs-on: ubuntu-latest
     env:
       MUJOCO_GL: egl
-      HF_HOME: /mnt/cache/.cache/huggingface
-      HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot
     steps:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
           lfs: true
 
-      - name: Setup /mnt storage
-        run: sudo chown -R $USER:$USER /mnt
-
       # TODO(Steven): Evaluate the need of these dependencies
       - name: Install apt dependencies
         run: |
@@ -85,14 +80,8 @@ jobs:
           version: ${{ env.UV_VERSION }}
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Check disk usage
-        run: df -h
-
       - name: Install lerobot with test extras
         run: uv sync --extra "test"
 
-      - name: Check disk usage
-        run: df -h
-
       - name: Run pytest
         run: uv run pytest tests -vv --maxfail=10

.github/workflows/full_tests.yml

@@ -58,17 +58,12 @@ jobs:
       github.event_name == 'workflow_dispatch'
     env:
       MUJOCO_GL: egl
-      HF_HOME: /mnt/cache/.cache/huggingface
-      HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot
     steps:
       - uses: actions/checkout@v4
         with:
           lfs: true
           persist-credentials: false
 
-      - name: Setup /mnt storage
-        run: sudo chown -R $USER:$USER /mnt
-
       - name: Install apt dependencies
         run: |
           sudo apt-get update && sudo apt-get install -y build-essential \
@@ -85,21 +80,12 @@ jobs:
       - name: Install lerobot with all extras
         run: uv sync --all-extras --no-extra groot # TODO(Steven): Make flash-attn optional
 
-      - name: Check disk usage
-        run: df -h
-
       - name: Run pytest (all extras)
         run: uv run pytest tests -vv --maxfail=10
 
-      - name: Check disk usage
-        run: df -h
-
       - name: Run end-to-end tests
         run: uv run make test-end-to-end
 
-      - name: Check disk usage
-        run: df -h
-
   # This job builds a GPU enabled image for testing
   # It runs everytime a PR is approved or a push to main
   # TODO(Steven): For now we skip this job for community PRs

.github/workflows/release.yml

@@ -83,11 +83,11 @@ jobs:
           fi
 
       - name: Remove Tags with Git dependencies
-        # TODO(Steven): Temporary patch to remove pi from PyPi 0.4.0 release due to its reliance on git dependencies.
+        # TODO(Steven): Temporary patch to remove libero and pi from PyPi 0.4.0 release due to its reliance on git dependencies.
         run: |
           echo "::info:: Checking for Git dependencies to remove from pyproject.toml..."
-          grep -E '@ git\+https|lerobot\[pi\]' pyproject.toml | sed 's/^/::warning:: Removing line: /' || true
-          sed -E -i '/@ git\+https|lerobot\[pi\]/d' pyproject.toml
+          grep -E '@ git\+https|lerobot\[pi\]|lerobot\[libero\]' pyproject.toml | sed 's/^/::warning:: Removing line: /' || true
+          sed -E -i '/@ git\+https|lerobot\[pi\]|lerobot\[libero\]/d' pyproject.toml
           echo "::info:: Git dependencies removed. Proceeding with build."
 
       - name: Install build dependencies

.github/workflows/unbound_deps_tests.yml

@@ -45,15 +45,11 @@ jobs:
     runs-on: ubuntu-latest
     env:
       MUJOCO_GL: egl
-      HF_HOME: /mnt/cache/.cache/huggingface
-      HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot
     steps:
       - uses: actions/checkout@v4
         with:
           lfs: true
           persist-credentials: false
-      - name: Setup /mnt storage
-        run: sudo chown -R $USER:$USER /mnt
 
       - name: Install apt dependencies
         run: |
@@ -74,7 +70,7 @@ jobs:
           echo "Dependencies unbound:" && cat pyproject.toml
 
       - name: Install lerobot with all extras
-        run: uv sync --all-extras --no-extra groot # TODO(Steven): Make flash-attn optional
+        run: uv sync --all-extras
 
       - name: Run pytest (all extras)
         run: uv run pytest tests -vv

README.md

@@ -186,7 +186,7 @@ For a full list of optional dependencies, see:
 https://pypi.org/project/lerobot/
 
 > [!NOTE]
-> For lerobot 0.4.0, if you want to install pi tags, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`.
+> For lerobot 0.4.0, if you want to install libero or pi tags, you will have to do: `pip install "lerobot[pi,libero]@git+https://github.com/huggingface/lerobot.git"`.
 >
 > This will be solved in the next patch release
 

benchmarks/video/benchmark.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import threading
+import time
+from contextlib import ContextDecorator
+
+
+class TimeBenchmark(ContextDecorator):
+    """
+    Measures execution time using a context manager or decorator.
+
+    This class supports both context manager and decorator usage, and is thread-safe for multithreaded
+    environments.
+
+    Args:
+        print: If True, prints the elapsed time upon exiting the context or completing the function. Defaults
+        to False.
+
+    Examples:
+
+        Using as a context manager:
+
+        >>> benchmark = TimeBenchmark()
+        >>> with benchmark:
+        ...     time.sleep(1)
+        >>> print(f"Block took {benchmark.result:.4f} seconds")
+        Block took approximately 1.0000 seconds
+
+        Using with multithreading:
+
+        ```python
+        import threading
+
+        benchmark = TimeBenchmark()
+
+
+        def context_manager_example():
+            with benchmark:
+                time.sleep(0.01)
+            print(f"Block took {benchmark.result_ms:.2f} milliseconds")
+
+
+        threads = []
+        for _ in range(3):
+            t1 = threading.Thread(target=context_manager_example)
+            threads.append(t1)
+
+        for t in threads:
+            t.start()
+
+        for t in threads:
+            t.join()
+        ```
+        Expected output:
+        Block took approximately 10.00 milliseconds
+        Block took approximately 10.00 milliseconds
+        Block took approximately 10.00 milliseconds
+    """
+
+    def __init__(self, print=False):
+        self.local = threading.local()
+        self.print_time = print
+
+    def __enter__(self):
+        self.local.start_time = time.perf_counter()
+        return self
+
+    def __exit__(self, *exc):
+        self.local.end_time = time.perf_counter()
+        self.local.elapsed_time = self.local.end_time - self.local.start_time
+        if self.print_time:
+            print(f"Elapsed time: {self.local.elapsed_time:.4f} seconds")
+        return False
+
+    @property
+    def result(self):
+        return getattr(self.local, "elapsed_time", None)
+
+    @property
+    def result_ms(self):
+        return self.result * 1e3

benchmarks/video/capture_camera_feed.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+
+# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Capture video feed from a camera as raw images."""
+
+import argparse
+import datetime as dt
+import os
+import time
+from pathlib import Path
+
+import cv2
+import rerun as rr
+
+# see https://rerun.io/docs/howto/visualization/limit-ram
+RERUN_MEMORY_LIMIT = os.getenv("LEROBOT_RERUN_MEMORY_LIMIT", "5%")
+
+
+def display_and_save_video_stream(output_dir: Path, fps: int, width: int, height: int, duration: int):
+    rr.init("lerobot_capture_camera_feed")
+    rr.spawn(memory_limit=RERUN_MEMORY_LIMIT)
+
+    now = dt.datetime.now()
+    capture_dir = output_dir / f"{now:%Y-%m-%d}" / f"{now:%H-%M-%S}"
+    if not capture_dir.exists():
+        capture_dir.mkdir(parents=True, exist_ok=True)
+
+    # Opens the default webcam
+    cap = cv2.VideoCapture(0)
+    if not cap.isOpened():
+        print("Error: Could not open video stream.")
+        return
+
+    cap.set(cv2.CAP_PROP_FPS, fps)
+    cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
+    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
+
+    frame_index = 0
+    start_time = time.time()
+    while time.time() - start_time < duration:
+        ret, frame = cap.read()
+
+        if not ret:
+            print("Error: Could not read frame.")
+            break
+        rr.log("video/stream", rr.Image(frame), static=True)
+        cv2.imwrite(str(capture_dir / f"frame_{frame_index:06d}.png"), frame)
+        frame_index += 1
+
+    # Release the capture
+    cap.release()
+
+    # TODO(Steven): Add a graceful shutdown via a close() method for the Viewer context, though not currently supported in the Rerun API.
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path("outputs/cam_capture/"),
+        help="Directory where the capture images are written. A subfolder named with the current date & time will be created inside it for each capture.",
+    )
+    parser.add_argument(
+        "--fps",
+        type=int,
+        default=30,
+        help="Frames Per Second of the capture.",
+    )
+    parser.add_argument(
+        "--width",
+        type=int,
+        default=1280,
+        help="Width of the captured images.",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        default=720,
+        help="Height of the captured images.",
+    )
+    parser.add_argument(
+        "--duration",
+        type=int,
+        default=20,
+        help="Duration in seconds for which the video stream should be captured.",
+    )
+    args = parser.parse_args()
+    display_and_save_video_stream(**vars(args))

benchmarks/video/run_video_benchmark.py

@@ -21,13 +21,11 @@ See the provided README.md or run `python benchmark/video/run_video_benchmark.py
 
 import argparse
 import datetime as dt
-import itertools
 import random
 import shutil
 from collections import OrderedDict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
-from threading import Lock
 
 import einops
 import numpy as np
@@ -37,13 +35,13 @@ import torch
 from skimage.metrics import mean_squared_error, peak_signal_noise_ratio, structural_similarity
 from tqdm import tqdm
 
+from benchmarks.video.benchmark import TimeBenchmark
 from lerobot.datasets.lerobot_dataset import LeRobotDataset
 from lerobot.datasets.video_utils import (
-    decode_video_frames,
+    decode_video_frames_torchvision,
     encode_video_frames,
 )
 from lerobot.utils.constants import OBS_IMAGE
-from lerobot.utils.utils import TimerManager
 
 BASE_ENCODING = OrderedDict(
     [
@@ -88,7 +86,7 @@ def load_original_frames(imgs_dir: Path, timestamps: list[float], fps: int) -> t
     frames = []
     for ts in timestamps:
         idx = int(ts * fps)
-        frame = PIL.Image.open(imgs_dir / f"frame-{idx:06d}.png")
+        frame = PIL.Image.open(imgs_dir / f"frame_{idx:06d}.png")
         frame = torch.from_numpy(np.array(frame))
         frame = frame.type(torch.float32) / 255
         frame = einops.rearrange(frame, "h w c -> c h w")
@@ -99,21 +97,21 @@ def load_original_frames(imgs_dir: Path, timestamps: list[float], fps: int) -> t
 def save_decoded_frames(
     imgs_dir: Path, save_dir: Path, frames: torch.Tensor, timestamps: list[float], fps: int
 ) -> None:
-    if save_dir.exists() and len(list(save_dir.glob("frame-*.png"))) == len(timestamps):
+    if save_dir.exists() and len(list(save_dir.glob("frame_*.png"))) == len(timestamps):
         return
 
     save_dir.mkdir(parents=True, exist_ok=True)
     for i, ts in enumerate(timestamps):
         idx = int(ts * fps)
         frame_hwc = (frames[i].permute((1, 2, 0)) * 255).type(torch.uint8).cpu().numpy()
-        PIL.Image.fromarray(frame_hwc).save(save_dir / f"frame-{idx:06d}_decoded.png")
-        shutil.copyfile(imgs_dir / f"frame-{idx:06d}.png", save_dir / f"frame-{idx:06d}_original.png")
+        PIL.Image.fromarray(frame_hwc).save(save_dir / f"frame_{idx:06d}_decoded.png")
+        shutil.copyfile(imgs_dir / f"frame_{idx:06d}.png", save_dir / f"frame_{idx:06d}_original.png")
 
 
 def save_first_episode(imgs_dir: Path, dataset: LeRobotDataset) -> None:
     episode_index = 0
     ep_num_images = dataset.meta.episodes["length"][episode_index]
-    if imgs_dir.exists() and len(list(imgs_dir.glob("frame-*.png"))) == ep_num_images:
+    if imgs_dir.exists() and len(list(imgs_dir.glob("frame_*.png"))) == ep_num_images:
         return
 
     imgs_dir.mkdir(parents=True, exist_ok=True)
@@ -127,7 +125,7 @@ def save_first_episode(imgs_dir: Path, dataset: LeRobotDataset) -> None:
         tqdm(imgs_dataset, desc=f"saving {dataset.repo_id} first episode images", leave=False)
     ):
         img = item[img_keys[0]]
-        img.save(str(imgs_dir / f"frame-{i:06d}.png"), quality=100)
+        img.save(str(imgs_dir / f"frame_{i:06d}.png"), quality=100)
 
         if i >= ep_num_images - 1:
             break
@@ -151,6 +149,18 @@ def sample_timestamps(timestamps_mode: str, ep_num_images: int, fps: int) -> lis
     return [idx / fps for idx in frame_indexes]
 
 
+def decode_video_frames(
+    video_path: str,
+    timestamps: list[float],
+    tolerance_s: float,
+    backend: str,
+) -> torch.Tensor:
+    if backend in ["pyav", "video_reader"]:
+        return decode_video_frames_torchvision(video_path, timestamps, tolerance_s, backend)
+    else:
+        raise NotImplementedError(backend)
+
+
 def benchmark_decoding(
     imgs_dir: Path,
     video_path: Path,
@@ -162,8 +172,8 @@ def benchmark_decoding(
     num_workers: int = 4,
     save_frames: bool = False,
 ) -> dict:
-    def process_sample(sample: int, lock: Lock):
-        time_benchmark = TimerManager(log=False)
+    def process_sample(sample: int):
+        time_benchmark = TimeBenchmark()
         timestamps = sample_timestamps(timestamps_mode, ep_num_images, fps)
         num_frames = len(timestamps)
         result = {
@@ -172,13 +182,13 @@ def benchmark_decoding(
             "mse_values": [],
         }
 
-        with time_benchmark, lock:
+        with time_benchmark:
             frames = decode_video_frames(video_path, timestamps=timestamps, tolerance_s=5e-1, backend=backend)
-        result["load_time_video_ms"] = (time_benchmark.last * 1000) / num_frames
+        result["load_time_video_ms"] = time_benchmark.result_ms / num_frames
 
         with time_benchmark:
             original_frames = load_original_frames(imgs_dir, timestamps, fps)
-        result["load_time_images_ms"] = (time_benchmark.last * 1000) / num_frames
+        result["load_time_images_ms"] = time_benchmark.result_ms / num_frames
 
         frames_np, original_frames_np = frames.numpy(), original_frames.numpy()
         for i in range(num_frames):
@@ -205,10 +215,8 @@ def benchmark_decoding(
     # A sample is a single set of decoded frames specified by timestamps_mode (e.g. a single frame, 2 frames, etc.).
     # For each sample, we record metrics (loading time and quality metrics) which are then averaged over all samples.
     # As these samples are independent, we run them in parallel threads to speed up the benchmark.
-    # Use a single shared lock for all worker threads
-    shared_lock = Lock()
     with ThreadPoolExecutor(max_workers=num_workers) as executor:
-        futures = [executor.submit(process_sample, i, shared_lock) for i in range(num_samples)]
+        futures = [executor.submit(process_sample, i) for i in range(num_samples)]
         for future in tqdm(as_completed(futures), total=num_samples, desc="samples", leave=False):
             result = future.result()
             load_times_video_ms.append(result["load_time_video_ms"])
@@ -350,27 +358,24 @@ def main(
                 imgs_dir = output_dir / "images" / dataset.repo_id.replace("/", "_")
                 # We only use the first episode
                 save_first_episode(imgs_dir, dataset)
-                for duet in [
-                    dict(zip(encoding_benchmarks.keys(), unique_combination, strict=False))
-                    for unique_combination in itertools.product(*encoding_benchmarks.values())
-                ]:
-                    encoding_cfg = BASE_ENCODING.copy()
-                    encoding_cfg["vcodec"] = video_codec
-                    encoding_cfg["pix_fmt"] = pixel_format
-                    for key, value in duet.items():
+                for key, values in tqdm(encoding_benchmarks.items(), desc="encodings (g, crf)", leave=False):
+                    for value in tqdm(values, desc=f"encodings ({key})", leave=False):
+                        encoding_cfg = BASE_ENCODING.copy()
+                        encoding_cfg["vcodec"] = video_codec
+                        encoding_cfg["pix_fmt"] = pixel_format
                         encoding_cfg[key] = value
-                    args_path = Path("_".join(str(value) for value in encoding_cfg.values()))
-                    video_path = output_dir / "videos" / args_path / f"{repo_id.replace('/', '_')}.mp4"
-                    benchmark_table += benchmark_encoding_decoding(
-                        dataset,
-                        video_path,
-                        imgs_dir,
-                        encoding_cfg,
-                        decoding_benchmarks,
-                        num_samples,
-                        num_workers,
-                        save_frames,
-                    )
+                        args_path = Path("_".join(str(value) for value in encoding_cfg.values()))
+                        video_path = output_dir / "videos" / args_path / f"{repo_id.replace('/', '_')}.mp4"
+                        benchmark_table += benchmark_encoding_decoding(
+                            dataset,
+                            video_path,
+                            imgs_dir,
+                            encoding_cfg,
+                            decoding_benchmarks,
+                            num_samples,
+                            num_workers,
+                            save_frames,
+                        )
 
             # Save intermediate results
             benchmark_df = pd.DataFrame(benchmark_table, columns=headers)
@@ -404,9 +409,9 @@ if __name__ == "__main__":
         nargs="*",
         default=[
             "lerobot/pusht_image",
-            "lerobot/aloha_mobile_shrimp_image",
-            "lerobot/paris_street",
-            "lerobot/kitchen",
+            "aliberts/aloha_mobile_shrimp_image",
+            "aliberts/paris_street",
+            "aliberts/kitchen",
         ],
         help="Datasets repo-ids to test against. First episodes only are used. Must be images.",
     )
@@ -414,7 +419,7 @@ if __name__ == "__main__":
         "--vcodec",
         type=str,
         nargs="*",
-        default=["h264", "hevc", "libsvtav1"],
+        default=["libx264", "hevc", "libsvtav1"],
         help="Video codecs to be tested",
     )
     parser.add_argument(
@@ -463,7 +468,7 @@ if __name__ == "__main__":
         "--backends",
         type=str,
         nargs="*",
-        default=["torchcodec", "pyav"],
+        default=["pyav", "video_reader"],
         help="Torchvision decoding backend to be tested.",
     )
     parser.add_argument(

docs/source/_toctree.yml

@@ -15,6 +15,8 @@
     title: Train a Robot with RL
   - local: hilserl_sim
     title: Train RL in Simulation
+  - local: async
+    title: Use Async Inference
   - local: multi_gpu_training
     title: Multi GPU training
   title: "Tutorials"
@@ -37,20 +39,10 @@
     title: π₀.₅ (Pi05)
   - local: groot
     title: NVIDIA GR00T N1.5
-  - local: xvla
-    title: X-VLA
   title: "Policies"
 - sections:
-  - local: async
-    title: Use Async Inference
-  - local: rtc
-    title: Real-Time Chunking (RTC)
-  title: "Inference"
-- sections:
-  - local: envhub
-    title: Environments from the Hub
-  - local: envhub_leisaac
-    title: Control & Train Robots in Sim (LeIsaac)
+  - local: il_sim
+    title: Imitation Learning in Sim
   - local: libero
     title: Using Libero
   - local: metaworld
@@ -65,8 +57,6 @@
     title: Implement your own processor
   - local: processors_robots_teleop
     title: Processors for Robots and Teleoperators
-  - local: env_processor
-    title: Environment Processors
   title: "Robot Processors"
 - sections:
   - local: so101

docs/source/async.mdx

@@ -196,7 +196,7 @@ client_cfg = RobotClientConfig(
     server_address="localhost:8080",
     policy_device="mps",
     policy_type="smolvla",
-    pretrained_name_or_path="<user>/smolvla_async",
+    pretrained_name_or_path="fracapuano/smolvla_async",
     chunk_size_threshold=0.5,
     actions_per_chunk=50,  # make sure this is less than the max actions of the policy
 )

docs/source/env_processor.mdx

@@ -1,418 +0,0 @@
-# Environment Processors
-
-Environment processors are a critical layer in LeRobot's data processing architecture that handle **environment-specific** transformations, separate from policy-specific processing. This separation of concerns enables cleaner code, better modularity, and easier experimentation with different environments and policies.
-
-## Why Environment Processors?
-
-When working with different robot environments (LIBERO, MetaWorld, Aloha, etc.), each environment often has unique data formats, coordinate systems, and conventions that need standardization **before** policy processing. Without environment processors, these transformations would be:
-
-1. **Hardcoded in environment code** - Making it difficult to experiment with different state representations
-2. **Duplicated across policies** - Each policy would need to handle environment-specific quirks
-3. **Mixed with policy logic** - Violating separation of concerns and making debugging harder
-
-Environment processors solve this by providing a **dedicated processing layer** between raw environment observations and policy inputs.
-
-## The Processing Pipeline
-
-Here's how data flows through the complete processing pipeline during evaluation:
-
-```python
-# In lerobot_eval.py rollout() function:
-
-# 1. Raw environment observation (numpy arrays, various formats)
-raw_observation = env.step(action)
-
-# 2. Convert numpy to torch, normalize images [0,1]
-observation = preprocess_observation(raw_observation)
-
-# 3. Add task metadata (for multi-task environments)
-observation = add_envs_task(env, observation)
-
-# 4. ENVIRONMENT-SPECIFIC preprocessing (NEW!)
-#    - Flatten robot states
-#    - Rotate images to match dataset conventions
-#    - Handle environment-specific coordinate systems
-observation = env_preprocessor(observation)
-
-# 5. POLICY-SPECIFIC preprocessing
-#    - Normalize with dataset statistics
-#    - Add batch dimensions
-#    - Move to GPU
-#    - Tokenize language instructions
-observation = preprocessor(observation)
-
-# 6. Policy inference
-action = policy.select_action(observation)
-
-# 7. POLICY-SPECIFIC postprocessing
-#    - Unnormalize actions
-#    - Remove batch dimensions
-action = postprocessor(action)
-
-# 8. ENVIRONMENT-SPECIFIC postprocessing (NEW!)
-#    - Convert action formats if needed
-#    - Apply environment-specific constraints
-action_transition = {"action": action}
-action_transition = env_postprocessor(action_transition)
-action = action_transition["action"]
-
-# 9. Execute in environment
-env.step(action)
-```
-
-## The Benefits
-
-### 1. **Separation of Concerns**
-
-Environment processors handle transformations specific to the **environment's data format**, while policy processors handle transformations specific to the **model's requirements**.
-
-```python
-# ❌ Before: Mixed concerns
-class LiberoVLAPolicy:
-    def preprocess(self, obs):
-        # Environment-specific: Flatten robot state (shouldn't be in policy!)
-        state = self._flatten_robot_state(obs["robot_state"])
-        # Policy-specific: Normalize with dataset stats
-        state = self.normalizer(state)
-        return state
-
-# ✅ After: Clear separation
-# Environment processor: Handles LIBERO's nested robot state
-env_preprocessor = LiberoProcessorStep()  # Flattens robot_state
-
-# Policy processor: Handles model requirements
-policy_preprocessor = NormalizerProcessorStep(stats=dataset_stats)
-```
-
-### 2. **Flexibility and Reusability**
-
-The same policy can work with different environment processors, and the same environment processor can work with different policies:
-
-```python
-# Use SmolVLA policy with LIBERO environment
-libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(libero_cfg)
-smolvla_preprocessor, smolvla_postprocessor = make_pre_post_processors(smolvla_cfg)
-
-# Or use ACT policy with the same LIBERO environment
-libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(libero_cfg)
-act_preprocessor, act_postprocessor = make_pre_post_processors(act_cfg)
-```
-
-### 3. **Easier Experimentation**
-
-Want to try different state representations for LIBERO? Just create a new processor:
-
-```python
-# Original: 8D state (pos + quat→axisangle + gripper)
-@ProcessorStepRegistry.register("libero_processor")
-class LiberoProcessorStep(ObservationProcessorStep):
-    def _process_observation(self, obs):
-        eef_pos = robot_state["eef"]["pos"]          # 3D
-        eef_axisangle = quat2axisangle(quat)         # 3D
-        gripper = robot_state["gripper"]["qpos"]     # 2D
-        state = torch.cat([eef_pos, eef_axisangle, gripper], dim=-1)  # 8D
-        return state
-
-# Experiment: Add velocity for better control
-@ProcessorStepRegistry.register("libero_velocity_processor")
-class LiberoVelocityProcessorStep(ObservationProcessorStep):
-    def _process_observation(self, obs):
-        # Include velocities for 14D state
-        eef_pos = robot_state["eef"]["pos"]          # 3D
-        eef_axisangle = quat2axisangle(quat)         # 3D
-        eef_vel = robot_state["eef"]["vel"]          # 3D  (NEW)
-        gripper_pos = robot_state["gripper"]["qpos"] # 2D
-        gripper_vel = robot_state["gripper"]["qvel"] # 3D  (NEW)
-        state = torch.cat([eef_pos, eef_axisangle, eef_vel,
-                          gripper_pos, gripper_vel], dim=-1)  # 14D
-        return state
-```
-
-### 4. **Cleaner Environment Code**
-
-Environments expose **all available data** without needing to know what downstream models will use:
-
-```python
-# LIBERO environment exposes full robot state
-observation = {
-    "pixels": {"image": img, "image2": img2},
-    "robot_state": {
-        "eef": {"pos": ..., "quat": ..., "vel": ..., "mat": ..., "axisangle": ...},
-        "gripper": {"qpos": ..., "qvel": ...},
-        "joints": {"pos": ..., "vel": ...}
-    }
-}
-
-# Environment processor decides what to use
-# Policy processor handles model-specific transformations
-```
-
-## Using Environment Processors
-
-### Factory Function
-
-The `make_env_pre_post_processors` function follows the same pattern as `make_pre_post_processors` for policies:
-
-```python
-from lerobot.envs.factory import make_env_pre_post_processors
-from lerobot.envs.configs import LiberoEnv, PushtEnv
-
-# For LIBERO: Returns LiberoProcessorStep in preprocessor
-libero_cfg = LiberoEnv(task="libero_spatial", camera_name=["agentview"])
-env_preprocessor, env_postprocessor = make_env_pre_post_processors(libero_cfg)
-
-# For other environments: Returns identity processors (no-op)
-pusht_cfg = PushtEnv()
-env_preprocessor, env_postprocessor = make_env_pre_post_processors(pusht_cfg)
-```
-
-### Implementation in `envs/factory.py`
-
-```python
-def make_env_pre_post_processors(
-    env_cfg: EnvConfig,
-) -> tuple[
-    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
-    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
-]:
-    """
-    Create preprocessor and postprocessor pipelines for environment observations.
-
-    Args:
-        env_cfg: The configuration of the environment.
-
-    Returns:
-        A tuple containing:
-            - preprocessor: Pipeline that processes environment observations
-            - postprocessor: Pipeline that processes environment outputs
-    """
-    # For LIBERO environments, add the LiberoProcessorStep to preprocessor
-    if isinstance(env_cfg, LiberoEnv) or "libero" in env_cfg.type:
-        preprocessor = PolicyProcessorPipeline(steps=[LiberoProcessorStep()])
-    else:
-        # For all other environments, return an identity preprocessor
-        preprocessor = PolicyProcessorPipeline(steps=[])
-
-    # Postprocessor is currently identity for all environments
-    # Future: Could add environment-specific action transformations
-    postprocessor = PolicyProcessorPipeline(steps=[])
-
-    return preprocessor, postprocessor
-```
-
-### Integration in Evaluation
-
-In `lerobot_eval.py`, the environment processors are created once and used throughout:
-
-```python
-def eval_main(cfg: EvalPipelineConfig):
-    # Create environment
-    envs = make_env(cfg.env, n_envs=cfg.eval.batch_size)
-
-    # Create policy
-    policy = make_policy(cfg=cfg.policy, env_cfg=cfg.env)
-
-    # Create policy processors
-    preprocessor, postprocessor = make_pre_post_processors(
-        policy_cfg=cfg.policy,
-        pretrained_path=cfg.policy.pretrained_path,
-    )
-
-    # Create environment processors (NEW!)
-    env_preprocessor, env_postprocessor = make_env_pre_post_processors(env_cfg=cfg.env)
-
-    # Run evaluation with both processor types
-    eval_policy_all(
-        envs=envs,
-        policy=policy,
-        env_preprocessor=env_preprocessor,      # Environment-specific
-        env_postprocessor=env_postprocessor,    # Environment-specific
-        preprocessor=preprocessor,              # Policy-specific
-        postprocessor=postprocessor,            # Policy-specific
-        n_episodes=cfg.eval.n_episodes,
-    )
-```
-
-## Example: LIBERO Environment Processor
-
-The `LiberoProcessorStep` demonstrates a real-world environment processor:
-
-```python
-from lerobot.processor.pipeline import ObservationProcessorStep
-
-@dataclass
-@ProcessorStepRegistry.register(name="libero_processor")
-class LiberoProcessorStep(ObservationProcessorStep):
-    """
-    Processes LIBERO observations into the LeRobot format.
-
-    **State Processing:**
-    - Extracts end-effector position (3D)
-    - Converts quaternion to axis-angle representation (3D)
-    - Extracts gripper joint positions (2D)
-    - Concatenates into 8D state vector
-
-    **Image Processing:**
-    - Rotates images 180° to match HuggingFaceVLA/libero convention
-    """
-
-    def _process_observation(self, observation):
-        processed_obs = observation.copy()
-
-        # Process images: Flip 180° for camera convention
-        for key in list(processed_obs.keys()):
-            if key.startswith("observation.images."):
-                img = processed_obs[key]
-                img = torch.flip(img, dims=[2, 3])  # Flip H and W
-                processed_obs[key] = img
-
-        # Process robot_state: Flatten to 8D vector
-        if "observation.robot_state" in processed_obs:
-            robot_state = processed_obs.pop("observation.robot_state")
-
-            eef_pos = robot_state["eef"]["pos"]           # (B, 3)
-            eef_quat = robot_state["eef"]["quat"]         # (B, 4)
-            gripper_qpos = robot_state["gripper"]["qpos"] # (B, 2)
-
-            # Convert quaternion to axis-angle
-            eef_axisangle = self._quat2axisangle(eef_quat)  # (B, 3)
-
-            # Concatenate into single state vector
-            state = torch.cat((eef_pos, eef_axisangle, gripper_qpos), dim=-1)
-            state = state.float()
-
-            processed_obs["observation.state"] = state
-
-        return processed_obs
-```
-
-### Why These Transformations?
-
-1. **Image Rotation**: The HuggingFaceVLA/libero dataset has images rotated 180° from the raw LIBERO simulator. The processor handles this convention mismatch so policies trained on the dataset work seamlessly.
-
-2. **State Flattening**: The raw LIBERO environment exposes nested dictionaries with all available state information (position, quaternion, velocity, matrix representation, etc.). The processor:
-   - Selects the relevant components (pos, quat, gripper)
-   - Converts quaternion to axis-angle (more suitable for learning)
-   - Flattens to a single 8D vector that policies expect
-
-3. **Flexibility**: The environment still exposes **all** raw data. If you want to try different state representations (e.g., including velocities, using matrix representation instead of axis-angle), you can create a new processor without modifying the environment code.
-
-## Adding Environment Processors for New Environments
-
-To add environment processors for a new environment:
-
-### 1. Create the Processor Step
-
-```python
-# In src/lerobot/processor/env_processor.py
-
-@dataclass
-@ProcessorStepRegistry.register(name="myenv_processor")
-class MyEnvProcessorStep(ObservationProcessorStep):
-    """Process observations from MyEnv."""
-
-    def _process_observation(self, observation):
-        processed = observation.copy()
-
-        # Your environment-specific transformations
-        if "myenv.specific.state" in processed:
-            state = processed.pop("myenv.specific.state")
-            # Transform to standard format
-            processed["observation.state"] = self._transform_state(state)
-
-        return processed
-```
-
-### 2. Update the Factory
-
-```python
-# In src/lerobot/envs/factory.py
-
-def make_env_pre_post_processors(env_cfg: EnvConfig):
-    if isinstance(env_cfg, LiberoEnv) or "libero" in env_cfg.type:
-        preprocessor = PolicyProcessorPipeline(steps=[LiberoProcessorStep()])
-    elif isinstance(env_cfg, MyEnvConfig) or "myenv" in env_cfg.type:
-        preprocessor = PolicyProcessorPipeline(steps=[MyEnvProcessorStep()])
-    else:
-        preprocessor = PolicyProcessorPipeline(steps=[])
-
-    postprocessor = PolicyProcessorPipeline(steps=[])
-    return preprocessor, postprocessor
-```
-
-### 3. Use in Evaluation
-
-No changes needed! The evaluation script automatically uses the appropriate processor:
-
-```bash
-lerobot-eval \
-    --policy.path=lerobot/my_policy \
-    --env.type=myenv \  # Automatically uses MyEnvProcessorStep
-    --eval.n_episodes=10
-```
-
-## Future: Environment Postprocessors
-
-Currently, postprocessors are identity (no-op) for all environments. Future use cases include:
-
-### Action Space Transformations
-
-```python
-@dataclass
-class MyEnvActionPostprocessor(ProcessorStep):
-    """Convert policy actions to environment-specific format."""
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        action = transition["action"]
-
-        # Example: Convert from Cartesian to joint space
-        if self.action_space == "joint":
-            action = self.ik_solver(action)
-
-        # Example: Apply environment-specific safety limits
-        action = torch.clamp(action, self.min_action, self.max_action)
-
-        transition["action"] = action
-        return transition
-```
-
-### Coordinate System Conversions
-
-```python
-@dataclass
-class CoordinateTransformPostprocessor(ProcessorStep):
-    """Transform actions between coordinate systems."""
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        action = transition["action"]
-
-        # Example: Policy outputs in world frame, env expects base frame
-        action = self.world_to_base_transform(action)
-
-        transition["action"] = action
-        return transition
-```
-
-## Best Practices
-
-1. **Keep environment processors simple**: They should only handle environment-specific data format issues, not complex learning-related transformations.
-
-2. **Use policy processors for model requirements**: Normalization, batching, device placement, and tokenization belong in policy processors.
-
-3. **Expose all data from environments**: Let processors decide what to use rather than hardcoding choices in the environment.
-
-4. **Document conventions**: Clearly document any coordinate system conventions, camera orientations, or data formats that your processor handles.
-
-5. **Test independently**: Environment processors should be testable without loading full policies or environments.
-
-## Summary
-
-Environment processors provide a **clean separation** between environment-specific data transformations and policy-specific model requirements. This architecture:
-
-- ✅ Enables easy experimentation with different state representations
-- ✅ Allows policies to work seamlessly across different environments
-- ✅ Keeps environment code focused on simulation/hardware interface
-- ✅ Makes processor pipelines more maintainable and debuggable
-- ✅ Follows the single responsibility principle
-
-The key insight: **Environments define data formats, processors standardize them, policies consume standardized data.** Each layer has a clear, focused responsibility.

docs/source/envhub.mdx

@@ -1,424 +0,0 @@
-# Loading Environments from the Hub
-
-The **EnvHub** feature allows you to load simulation environments directly from the Hugging Face Hub with a single line of code. This unlocks a powerful new model for collaboration: instead of environments being locked away inside monolithic libraries, anyone can publish custom environments and share them with the community.
-
-## Overview
-
-With EnvHub, you can:
-
-- Load environments from the Hub instantly
-- Share your custom simulation tasks with the community
-- Version control your environments using Git
-- Distribute complex physics simulations without packaging hassles
-
-## Quick Start
-
-Loading an environment from the Hub is as simple as:
-
-```python
-from lerobot.envs.factory import make_env
-
-# Load a hub environment (requires explicit consent to run remote code)
-env = make_env("lerobot/cartpole-env", trust_remote_code=True)
-```
-
-<Tip warning={true}>
-  **Security Notice**: Loading environments from the Hub executes Python code
-  from third-party repositories. Only use `trust_remote_code=True` with
-  repositories you trust. We strongly recommend pinning to a specific commit
-  hash for reproducibility and security.
-</Tip>
-
-## What is EnvHub?
-
-EnvHub is a framework that allows researchers and developers to:
-
-1. **Publish environments** to the Hugging Face Hub as Git repositories
-2. **Load environments** dynamically without installing them as packages
-3. **Version and track** environment changes using Git semantics
-4. **Discover** new simulation tasks shared by the community
-
-This design means you can go from discovering an interesting environment on the Hub to running experiments in seconds, without worrying about dependency conflicts or complex installation procedures.
-
-## Repository Structure
-
-To make your environment loadable from the Hub, your repository must contain at minimum:
-
-### Required Files
-
-**`env.py`** (or custom Python file)
-
-- Must expose a `make_env(n_envs: int, use_async_envs: bool)` function
-- This function should return one of:
-  - A `gym.vector.VectorEnv` (most common)
-  - A single `gym.Env` (will be automatically wrapped)
-  - A dict mapping `{suite_name: {task_id: VectorEnv}}` (for multi-task benchmarks)
-
-### Optional Files
-
-**`requirements.txt`**
-
-- List any additional dependencies your environment needs
-- Users will need to install these manually before loading your environment
-
-**`README.md`**
-
-- Document your environment: what task it implements, observation/action spaces, rewards, etc.
-- Include usage examples and any special setup instructions
-
-**`.gitignore`**
-
-- Exclude unnecessary files from your repository
-
-### Example Repository Structure
-
-```
-my-environment-repo/
-├── env.py                 # Main environment definition (required)
-├── requirements.txt       # Dependencies (optional)
-├── README.md             # Documentation (recommended)
-├── assets/               # Images, videos, etc. (optional)
-│   └── demo.gif
-└── configs/              # Config files if needed (optional)
-    └── task_config.yaml
-```
-
-## Creating Your Environment Repository
-
-### Step 1: Define Your Environment
-
-Create an `env.py` file with a `make_env` function:
-
-```python
-# env.py
-import gymnasium as gym
-
-def make_env(n_envs: int = 1, use_async_envs: bool = False):
-    """
-    Create vectorized environments for your custom task.
-
-    Args:
-        n_envs: Number of parallel environments
-        use_async_envs: Whether to use AsyncVectorEnv or SyncVectorEnv
-
-    Returns:
-        gym.vector.VectorEnv or dict mapping suite names to vectorized envs
-    """
-    def _make_single_env():
-        # Create your custom environment
-        return gym.make("CartPole-v1")
-
-    # Choose vector environment type
-    env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv
-
-    # Create vectorized environment
-    vec_env = env_cls([_make_single_env for _ in range(n_envs)])
-
-    return vec_env
-```
-
-### Step 2: Test Locally
-
-Before uploading, test your environment locally:
-
-```python
-from lerobot.envs.utils import _load_module_from_path, _call_make_env, _normalize_hub_result
-
-# Load your module
-module = _load_module_from_path("./env.py")
-
-# Test the make_env function
-result = _call_make_env(module, n_envs=2, use_async_envs=False)
-normalized = _normalize_hub_result(result)
-
-# Verify it works
-suite_name = next(iter(normalized))
-env = normalized[suite_name][0]
-obs, info = env.reset()
-print(f"Observation shape: {obs.shape if hasattr(obs, 'shape') else type(obs)}")
-env.close()
-```
-
-### Step 3: Upload to the Hub
-
-Upload your repository to Hugging Face:
-
-```bash
-# Install huggingface_hub if needed
-pip install huggingface_hub
-
-# Login to Hugging Face
-huggingface-cli login
-
-# Create a new repository
-huggingface-cli repo create my-custom-env --type space --org my-org
-
-# Initialize git and push
-git init
-git add .
-git commit -m "Initial environment implementation"
-git remote add origin https://huggingface.co/my-org/my-custom-env
-git push -u origin main
-```
-
-Alternatively, use the `huggingface_hub` Python API:
-
-```python
-from huggingface_hub import HfApi
-
-api = HfApi()
-
-# Create repository
-api.create_repo("my-custom-env", repo_type="space")
-
-# Upload files
-api.upload_folder(
-    folder_path="./my-env-folder",
-    repo_id="username/my-custom-env",
-    repo_type="space",
-)
-```
-
-## Loading Environments from the Hub
-
-### Basic Usage
-
-```python
-from lerobot.envs.factory import make_env
-
-# Load from the hub
-envs_dict = make_env(
-    "username/my-custom-env",
-    n_envs=4,
-    trust_remote_code=True
-)
-
-# Access the environment
-suite_name = next(iter(envs_dict))
-env = envs_dict[suite_name][0]
-
-# Use it like any gym environment
-obs, info = env.reset()
-action = env.action_space.sample()
-obs, reward, terminated, truncated, info = env.step(action)
-```
-
-### Advanced: Pinning to Specific Versions
-
-For reproducibility and security, pin to a specific Git revision:
-
-```python
-# Pin to a specific branch
-env = make_env("username/my-env@main", trust_remote_code=True)
-
-# Pin to a specific commit (recommended for papers/experiments)
-env = make_env("username/my-env@abc123def456", trust_remote_code=True)
-
-# Pin to a tag
-env = make_env("username/my-env@v1.0.0", trust_remote_code=True)
-```
-
-### Custom File Paths
-
-If your environment definition is not in `env.py`:
-
-```python
-# Load from a custom file
-env = make_env("username/my-env:custom_env.py", trust_remote_code=True)
-
-# Combine with version pinning
-env = make_env("username/my-env@v1.0:envs/task_a.py", trust_remote_code=True)
-```
-
-### Async Environments
-
-For better performance with multiple environments:
-
-```python
-envs_dict = make_env(
-    "username/my-env",
-    n_envs=8,
-    use_async_envs=True,  # Use AsyncVectorEnv for parallel execution
-    trust_remote_code=True
-)
-```
-
-## URL Format Reference
-
-The hub URL format supports several patterns:
-
-| Pattern              | Description                    | Example                                |
-| -------------------- | ------------------------------ | -------------------------------------- |
-| `user/repo`          | Load `env.py` from main branch | `make_env("lerobot/pusht-env")`        |
-| `user/repo@revision` | Load from specific revision    | `make_env("lerobot/pusht-env@main")`   |
-| `user/repo:path`     | Load custom file               | `make_env("lerobot/envs:pusht.py")`    |
-| `user/repo@rev:path` | Revision + custom file         | `make_env("lerobot/envs@v1:pusht.py")` |
-
-## Multi-Task Environments
-
-For benchmarks with multiple tasks (like LIBERO), return a nested dictionary:
-
-```python
-def make_env(n_envs: int = 1, use_async_envs: bool = False):
-    env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv
-
-    # Return dict: {suite_name: {task_id: VectorEnv}}
-    return {
-        "suite_1": {
-            0: env_cls([lambda: gym.make("Task1-v0") for _ in range(n_envs)]),
-            1: env_cls([lambda: gym.make("Task2-v0") for _ in range(n_envs)]),
-        },
-        "suite_2": {
-            0: env_cls([lambda: gym.make("Task3-v0") for _ in range(n_envs)]),
-        }
-    }
-```
-
-## Security Considerations
-
-<Tip warning={true}>
-  **Important**: The `trust_remote_code=True` flag is required to execute
-  environment code from the Hub. This is by design for security.
-</Tip>
-
-When loading environments from the Hub:
-
-1. **Review the code first**: Visit the repository and inspect `env.py` before loading
-2. **Pin to commits**: Use specific commit hashes for reproducibility
-3. **Check dependencies**: Review `requirements.txt` for suspicious packages
-4. **Use trusted sources**: Prefer official organizations or well-known researchers
-5. **Sandbox if needed**: Run untrusted code in isolated environments (containers, VMs)
-
-Example of safe usage:
-
-```python
-# ❌ BAD: Loading without inspection
-env = make_env("random-user/untrusted-env", trust_remote_code=True)
-
-# ✅ GOOD: Review code, then pin to specific commit
-# 1. Visit https://huggingface.co/trusted-org/verified-env
-# 2. Review the env.py file
-# 3. Copy the commit hash
-env = make_env("trusted-org/verified-env@a1b2c3d4", trust_remote_code=True)
-```
-
-## Example: CartPole from the Hub
-
-Here's a complete example using the reference CartPole environment:
-
-```python
-from lerobot.envs.factory import make_env
-import numpy as np
-
-# Load the environment
-envs_dict = make_env("lerobot/cartpole-env", n_envs=4, trust_remote_code=True)
-
-# Get the vectorized environment
-suite_name = next(iter(envs_dict))
-env = envs_dict[suite_name][0]
-
-# Run a simple episode
-obs, info = env.reset()
-done = np.zeros(env.num_envs, dtype=bool)
-total_reward = np.zeros(env.num_envs)
-
-while not done.all():
-    # Random policy
-    action = env.action_space.sample()
-    obs, reward, terminated, truncated, info = env.step(action)
-    total_reward += reward
-    done = terminated | truncated
-
-print(f"Average reward: {total_reward.mean():.2f}")
-env.close()
-```
-
-## Benefits of EnvHub
-
-### For Environment Authors
-
-- **Easy distribution**: No PyPI packaging required
-- **Version control**: Use Git for environment versioning
-- **Rapid iteration**: Push updates instantly
-- **Documentation**: Hub README renders beautifully
-- **Community**: Reach LeRobot users directly
-
-### For Researchers
-
-- **Quick experiments**: Load any environment in one line
-- **Reproducibility**: Pin to specific commits
-- **Discovery**: Browse environments on the Hub
-- **No conflicts**: No need to install conflicting packages
-
-### For the Community
-
-- **Growing ecosystem**: More diverse simulation tasks
-- **Standardization**: Common `make_env` API
-- **Collaboration**: Fork and improve existing environments
-- **Accessibility**: Lower barrier to sharing research
-
-## Troubleshooting
-
-### "Refusing to execute remote code"
-
-You must explicitly pass `trust_remote_code=True`:
-
-```python
-env = make_env("user/repo", trust_remote_code=True)
-```
-
-### "Module X not found"
-
-The hub environment has dependencies you need to install:
-
-```bash
-# Check the repo's requirements.txt and install dependencies
-pip install gymnasium numpy
-```
-
-### "make_env not found in module"
-
-Your `env.py` must expose a `make_env` function:
-
-```python
-def make_env(n_envs: int, use_async_envs: bool):
-    # Your implementation
-    pass
-```
-
-### Environment returns wrong type
-
-The `make_env` function must return:
-
-- A `gym.vector.VectorEnv`, or
-- A single `gym.Env`, or
-- A dict `{suite_name: {task_id: VectorEnv}}`
-
-## Best Practices
-
-1. **Document your environment**: Include observation/action space descriptions, reward structure, and termination conditions in your README
-2. **Add requirements.txt**: List all dependencies with versions
-3. **Test thoroughly**: Verify your environment works locally before pushing
-4. **Use semantic versioning**: Tag releases with version numbers
-5. **Add examples**: Include usage examples in your README
-6. **Keep it simple**: Minimize dependencies when possible
-7. **License your work**: Add a LICENSE file to clarify usage terms
-
-## Future Directions
-
-The EnvHub ecosystem enables exciting possibilities:
-
-- **GPU-accelerated physics**: Share Isaac Gym or Brax environments
-- **Photorealistic rendering**: Distribute environments with advanced graphics
-- **Multi-agent scenarios**: Complex interaction tasks
-- **Real-world simulators**: Digital twins of physical setups
-- **Procedural generation**: Infinite task variations
-- **Domain randomization**: Pre-configured DR pipelines
-
-As more researchers and developers contribute, the diversity and quality of available environments will grow, benefiting the entire robotics learning community.
-
-## See Also
-
-- [Hugging Face Hub Documentation](https://huggingface.co/docs/hub/en/index)
-- [Gymnasium Documentation](https://gymnasium.farama.org/index.html)
-- [Example Hub Environment](https://huggingface.co/lerobot/cartpole-env)

docs/source/envhub_leisaac.mdx

@@ -1,301 +0,0 @@
-# LeIsaac × LeRobot EnvHub
-
-LeRobot EnvHub now supports **imitation learning in simulation** with LeIsaac.
-Spin up everyday manipulation tasks, teleoperate the robot, collect demos, push them to the Hub, and train policies in LeRobot — all in one loop.
-
-[LeIsaac](https://github.com/LightwheelAI/leisaac) integrates with IsaacLab and the SO101 Leader/Follower setup to provide:
-
-- 🕹️ **Teleoperation-first workflows** for data collection
-- 📦 **Built-in data conversion** ready for LeRobot training
-- 🤖 **Everyday skills** like picking oranges, lifting cubes, cleaning tables, and folding cloth
-- ☁️ **Ongoing upgrades** from [LightWheel](https://lightwheel.ai/): cloud simulation, EnvHub support, Sim2Real tooling, and more
-
-Below you’ll find the currently supported LeIsaac tasks exposed through LeRobot EnvHub.
-
-# Available Environments
-
-The following table lists all available tasks and environments in LeIsaac x LeRobot Envhub. You can also get the latest list of environments by running the following command:
-
-```bash
-python scripts/environments/list_envs.py
-```
-
-| Task                                                                                                                                                            | Environment ID                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Task Description                                                                                                           | Related Robot                                              |
-| :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------- |
-| <video src="https://github.com/user-attachments/assets/466eddff-f720-4f99-94d5-5e123e4c302c" autoplay loop muted playsinline style="max-width: 300px;"></video> | [LeIsaac-SO101-PickOrange-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/pick_orange/pick_orange_env_cfg.py)<br /><br />[LeIsaac-SO101-PickOrange-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/pick_orange/direct/pick_orange_env.py)                                                                                                                                                                                                                        | Pick three oranges and put them into the plate, then reset the arm to rest state.                                          | Single-Arm SO101 Follower                                  |
-| <video src="https://github.com/user-attachments/assets/1e4eb83a-0b38-40fb-a0b2-ddb0fe201e6d" autoplay loop muted playsinline style="max-width: 300px;"></video> | [LeIsaac-SO101-LiftCube-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/lift_cube/lift_cube_env_cfg.py)<br /><br />[LeIsaac-SO101-LiftCube-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/lift_cube/direct/lift_cube_env.py)                                                                                                                                                                                                                                    | Lift the red cube up.                                                                                                      | Single-Arm SO101 Follower                                  |
-| <video src="https://github.com/user-attachments/assets/e49d8f1c-dcc9-412b-a88f-100680d8a45b" autoplay loop muted playsinline style="max-width: 300px;"></video> | [LeIsaac-SO101-CleanToyTable-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/clean_toy_table/clean_toy_table_env_cfg.py)<br /><br />[LeIsaac-SO101-CleanToyTable-BiArm-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/clean_toy_table/clean_toy_table_bi_arm_env_cfg.py)<br /><br />[LeIsaac-SO101-CleanToyTable-BiArm-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/clean_toy_table/direct/clean_toy_table_bi_arm_env.py) | Pick two letter e objects into the box, and reset the arm to rest state.                                                   | Single-Arm SO101 Follower<br /><br />Bi-Arm SO101 Follower |
-| <video src="https://github.com/user-attachments/assets/e29a0f8a-9286-4ce6-b45d-342c3d3ba754" autoplay loop muted playsinline style="max-width: 300px;"></video> | [LeIsaac-SO101-FoldCloth-BiArm-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/fold_cloth/fold_cloth_bi_arm_env_cfg.py)<br /><br />[LeIsaac-SO101-FoldCloth-BiArm-Direct-v0](https://github.com/LightwheelAI/leisaac/blob/main/source/leisaac/leisaac/tasks/fold_cloth/direct/fold_cloth_bi_arm_env.py)                                                                                                                                                                                                    | Fold the cloth, and reset the arm to rest state.<br /><br />_Note: Only the DirectEnv support check_success in this task._ | Bi-Arm SO101 Follower                                      |
-
-# Load LeIsaac directly in LeRobot with one line of code
-
-> EnvHub: Share LeIsaac environments through HuggingFace
-
-[EnvHub](https://huggingface.co/docs/lerobot/envhub) is our reproducible environment hub, spin up a packaged simulation with one line, experiment immediately, and publish your own tasks for the community.
-
-LeIsaac offers EnvHub support so you can consume or share tasks with only a few commands.
-
-<video
-  controls
-  src="https://github.com/user-attachments/assets/687666f5-ebe0-421d-84a0-eb86116ac5f8"
-  style={{ width: "100%", maxWidth: "960px", borderRadius: "8px" }}
-/>
-
-## How to get started, environment Setup
-
-Run the following commands to setup your code environments:
-
-```bash
-# Refer to Getting Started/Installation to install leisaac firstly
-conda create -n leisaac_envhub python=3.11
-conda activate leisaac_envhub
-
-conda install -c "nvidia/label/cuda-12.8.1" cuda-toolkit
-pip install -U torch==2.7.0 torchvision==0.22.0 --index-url https://download.pytorch.org/whl/cu128
-pip install 'leisaac[isaaclab] @ git+https://github.com/LightwheelAI/leisaac.git#subdirectory=source/leisaac' --extra-index-url https://pypi.nvidia.com
-
-# Install lerobot
-pip install lerobot==0.4.1
-
-# Fix numpy version
-pip install numpy==1.26.0
-```
-
-## Usage Example
-
-EnvHub exposes every LeIsaac-supported task in a uniform interface. The examples below load `so101_pick_orange` and demonstrate a random-action rollout and an interactive teleoperation.
-
-### Random Action
-
-<details>
-<summary>Click to expand code example</summary>
-
-```python
-# envhub_random_action.py
-
-import torch
-from lerobot.envs.factory import make_env
-
-# Load from the hub
-envs_dict = make_env("LightwheelAI/leisaac_env:envs/so101_pick_orange.py", n_envs=1, trust_remote_code=True)
-
-# Access the environment
-suite_name = next(iter(envs_dict))
-sync_vector_env = envs_dict[suite_name][0]
-# retrieve the isaac environment from the sync vector env
-env = sync_vector_env.envs[0].unwrapped
-
-# Use it like any gym environment
-obs, info = env.reset()
-
-while True:
-    action = torch.tensor(env.action_space.sample())
-    obs, reward, terminated, truncated, info = env.step(action)
-    if terminated or truncated:
-        obs, info = env.reset()
-
-env.close()
-```
-
-</details>
-
-```bash
-python envhub_random_action.py
-```
-
-You should see the SO101 arm swinging under purely random commands.
-
-### Teleoperation
-
-LeRobot’s teleoperation stack can drive the simulated arm.
-
-Connect the SO101 Leader controller, run the calibration command below.
-
-```bash
-lerobot-calibrate \
-    --teleop.type=so101_leader \
-    --teleop.port=/dev/ttyACM0 \
-    --teleop.id=leader
-```
-
-And then launch the teleop script.
-
-<details>
-<summary>Click to expand code example</summary>
-
-```python
-# envhub_teleop_example.py
-
-import logging
-import time
-import gymnasium as gym
-
-from dataclasses import asdict, dataclass
-from pprint import pformat
-
-from lerobot.teleoperators import (  # noqa: F401
-    Teleoperator,
-    TeleoperatorConfig,
-    make_teleoperator_from_config,
-    so101_leader,
-)
-from lerobot.utils.robot_utils import precise_sleep
-from lerobot.utils.utils import init_logging
-from lerobot.envs.factory import make_env
-
-
-@dataclass
-class TeleoperateConfig:
-    teleop: TeleoperatorConfig
-    env_name: str = "so101_pick_orange"
-    fps: int = 60
-
-
-@dataclass
-class EnvWrap:
-    env: gym.Env
-
-
-def make_env_from_leisaac(env_name: str = "so101_pick_orange"):
-    envs_dict = make_env(
-        f'LightwheelAI/leisaac_env:envs/{env_name}.py',
-        n_envs=1,
-        trust_remote_code=True
-    )
-    suite_name = next(iter(envs_dict))
-    sync_vector_env = envs_dict[suite_name][0]
-    env = sync_vector_env.envs[0].unwrapped
-
-    return env
-
-
-def teleop_loop(teleop: Teleoperator, env: gym.Env, fps: int):
-    from leisaac.devices.action_process import preprocess_device_action
-    from leisaac.assets.robots.lerobot import SO101_FOLLOWER_MOTOR_LIMITS
-    from leisaac.utils.env_utils import dynamic_reset_gripper_effort_limit_sim
-
-    env_wrap = EnvWrap(env=env)
-
-    obs, info = env.reset()
-    while True:
-        loop_start = time.perf_counter()
-        if env.cfg.dynamic_reset_gripper_effort_limit:
-            dynamic_reset_gripper_effort_limit_sim(env, 'so101leader')
-
-        raw_action = teleop.get_action()
-        processed_action = preprocess_device_action(
-            dict(
-                so101_leader=True,
-                joint_state={
-                    k.removesuffix(".pos"): v for k, v in raw_action.items()},
-                motor_limits=SO101_FOLLOWER_MOTOR_LIMITS),
-            env_wrap
-        )
-        obs, reward, terminated, truncated, info = env.step(processed_action)
-        if terminated or truncated:
-            obs, info = env.reset()
-
-        dt_s = time.perf_counter() - loop_start
-        precise_sleep(1 / fps - dt_s)
-        loop_s = time.perf_counter() - loop_start
-        print(f"\ntime: {loop_s * 1e3:.2f}ms ({1 / loop_s:.0f} Hz)")
-
-
-def teleoperate(cfg: TeleoperateConfig):
-    init_logging()
-    logging.info(pformat(asdict(cfg)))
-
-    teleop = make_teleoperator_from_config(cfg.teleop)
-    env = make_env_from_leisaac(cfg.env_name)
-
-    teleop.connect()
-    if hasattr(env, 'initialize'):
-        env.initialize()
-    try:
-        teleop_loop(teleop=teleop, env=env, fps=cfg.fps)
-    except KeyboardInterrupt:
-        pass
-    finally:
-        teleop.disconnect()
-        env.close()
-
-
-def main():
-    teleoperate(TeleoperateConfig(
-        teleop=so101_leader.SO101LeaderConfig(
-            port="/dev/ttyACM0",
-            id='leader',
-            use_degrees=False,
-        ),
-        env_name="so101_pick_orange",
-        fps=60,
-    ))
-
-
-if __name__ == "__main__":
-    main()
-
-```
-
-</details>
-
-```bash
-python envhub_teleop_example.py
-```
-
-Running the script lets you operate the simulated arm using the physical Leader device.
-
-## ☁️ Cloud Simulation (No GPU Required)
-
-Don’t have a local GPU or the right drivers? No problem! You can run LeIsaac entirely in the cloud with zero setup.
-LeIsaac works out-of-the-box on **NVIDIA Brev**, giving you a fully configured environment directly in your browser.
-
-👉 **Start here:** [https://lightwheelai.github.io/leisaac/docs/cloud_simulation/nvidia_brev](https://lightwheelai.github.io/leisaac/docs/cloud_simulation/nvidia_brev)
-
-Once your instance is deployed, simply open the link for **port 80 (HTTP)** to launch **Visual Studio Code Server** (default password: `password`). From there, you can run simulations, edit code, and visualize IsaacLab environments — all from your web browser.
-
-**No GPU, no drivers, no local installation. Just click and run.**
-
-## Additional Notes
-
-We keep EnvHub coverage aligned with the LeIsaac task. Currently supported:
-
-- `so101_pick_orange`
-- `so101_lift_cube`
-- `so101_clean_toytable`
-- `bi_so101_fold_cloth`
-
-Switch tasks by targeting a different script when calling `make_env`, for example:
-
-```python
-envs_dict_pick_orange = make_env("LightwheelAI/leisaac_env:envs/so101_pick_orange.py", n_envs=1, trust_remote_code=True)
-envs_dict_lift_cube = make_env("LightwheelAI/leisaac_env:envs/so101_lift_cube.py", n_envs=1, trust_remote_code=True)
-envs_dict_clean_toytable = make_env("LightwheelAI/leisaac_env:envs/so101_clean_toytable.py", n_envs=1, trust_remote_code=True)
-envs_dict_fold_cloth = make_env("LightwheelAI/leisaac_env:envs/bi_so101_fold_cloth.py", n_envs=1, trust_remote_code=True)
-```
-
-Note: when working with `bi_so101_fold_cloth`, call `initialize()` immediately after retrieving the env before performing any other operations:
-
-<details>
-<summary>Click to expand code example</summary>
-
-```python
-import torch
-from lerobot.envs.factory import make_env
-
-# Load from the hub
-envs_dict = make_env("LightwheelAI/leisaac_env:envs/bi_so101_fold_cloth.py", n_envs=1, trust_remote_code=True)
-
-# Access the environment
-suite_name = next(iter(envs_dict))
-sync_vector_env = envs_dict[suite_name][0]
-# retrieve the isaac environment from the sync vector env
-env = sync_vector_env.envs[0].unwrapped
-
-# NOTE: initialize() first
-env.initialize()
-
-# other operation with env...
-```
-
-</details>

docs/source/groot.mdx

@@ -40,7 +40,7 @@ python -c "import flash_attn; print(f'Flash Attention {flash_attn.__version__} i
 3. Install LeRobot by running:
 
 ```bash
-pip install lerobot[groot]
+pip install lerobot[groot] # consider also installing libero,dev and test tags
 ```
 
 ## Usage
@@ -83,9 +83,6 @@ accelerate launch \
 
 ### Libero Benchmark Results
 
-> [!NOTE]
-> Follow our instructions for Libero usage: [Libero](./libero)
-
 GR00T has demonstrated strong performance on the Libero benchmark suite. To compare and test its LeRobot implementation, we finetuned the GR00T N1.5 model for 30k steps on the Libero dataset and compared the results to the GR00T reference results.
 
 | Benchmark          | LeRobot Implementation | GR00T Reference |

docs/source/il_robots.mdx

@@ -393,7 +393,7 @@ import time
 from lerobot.datasets.lerobot_dataset import LeRobotDataset
 from lerobot.robots.so100_follower.config_so100_follower import SO100FollowerConfig
 from lerobot.robots.so100_follower.so100_follower import SO100Follower
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.utils import log_say
 
 episode_idx = 0
@@ -415,7 +415,7 @@ for idx in range(dataset.num_frames):
     }
     robot.send_action(action)
 
-    precise_sleep(1.0 / dataset.fps - (time.perf_counter() - t0))
+    busy_wait(1.0 / dataset.fps - (time.perf_counter() - t0))
 
 robot.disconnect()
 ```

docs/source/il_sim.mdx

@@ -0,0 +1,220 @@
+# Imitation Learning in Sim
+
+This tutorial will explain how to train a neural network to control a robot in simulation with imitation learning.
+
+**You'll learn:**
+
+1. How to record a dataset in simulation with [gym-hil](https://github.com/huggingface/gym-hil) and visualize the dataset.
+2. How to train a policy using your data.
+3. How to evaluate your policy in simulation and visualize the results.
+
+For the simulation environment we use the same [repo](https://github.com/huggingface/gym-hil) that is also being used by the Human-In-the-Loop (HIL) reinforcement learning algorithm.
+This environment is based on [MuJoCo](https://mujoco.org) and allows you to record datasets in LeRobotDataset format.
+Teleoperation is easiest with a controller like the Logitech F710, but you can also use your keyboard if you are up for the challenge.
+
+## Installation
+
+First, install the `gym_hil` package within the LeRobot environment, go to your LeRobot folder and run this command:
+
+```bash
+pip install -e ".[hilserl]"
+```
+
+## Teleoperate and Record a Dataset
+
+To use `gym_hil` with LeRobot, you need to use a configuration file. An example config file can be found [here](https://huggingface.co/datasets/lerobot/config_examples/resolve/main/sim_il/env_config.json).
+
+To teleoperate and collect a dataset, we need to modify this config file. Here's an example configuration for imitation learning data collection:
+
+```json
+{
+  "env": {
+    "type": "gym_manipulator",
+    "name": "gym_hil",
+    "task": "PandaPickCubeGamepad-v0",
+    "fps": 10
+  },
+  "dataset": {
+    "repo_id": "your_username/il_gym",
+    "root": null,
+    "task": "pick_cube",
+    "num_episodes_to_record": 30,
+    "replay_episode": null,
+    "push_to_hub": true
+  },
+  "mode": "record",
+  "device": "cuda"
+}
+```
+
+Key configuration points:
+
+- Set your `repo_id` in the `dataset` section: `"repo_id": "your_username/il_gym"`
+- Set `num_episodes_to_record: 30` to collect 30 demonstration episodes
+- Ensure `mode` is set to `"record"`
+- If you don't have an NVIDIA GPU, change `"device": "cuda"` to `"mps"` for macOS or `"cpu"`
+- To use keyboard instead of gamepad, change `"task"` to `"PandaPickCubeKeyboard-v0"`
+
+Then we can run this command to start:
+
+<hfoptions id="teleop_sim">
+<hfoption id="Linux">
+
+```bash
+python -m lerobot.rl.gym_manipulator --config_path path/to/env_config_gym_hil_il.json
+```
+
+</hfoption>
+<hfoption id="MacOS">
+
+```bash
+mjpython -m lerobot.rl.gym_manipulator --config_path path/to/env_config_gym_hil_il.json
+```
+
+</hfoption>
+</hfoptions>
+
+Once rendered you can teleoperate the robot with the gamepad or keyboard, below you can find the gamepad/keyboard controls.
+
+Note that to teleoperate the robot you have to hold the "Human Take Over Pause Policy" Button `RB` to enable control!
+
+**Gamepad Controls**
+
+<p align="center">
+  <img
+    src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/gamepad_guide.jpg?raw=true"
+    alt="Figure shows the control mappings on a Logitech gamepad."
+    title="Gamepad Control Mapping"
+    width="100%"
+  ></img>
+</p>
+<p align="center">
+  <i>Gamepad button mapping for robot control and episode management</i>
+</p>
+
+**Keyboard controls**
+
+For keyboard controls use the `spacebar` to enable control and the following keys to move the robot:
+
+```bash
+  Arrow keys: Move in X-Y plane
+  Shift and Shift_R: Move in Z axis
+  Right Ctrl and Left Ctrl: Open and close gripper
+  ESC: Exit
+```
+
+## Visualize a dataset
+
+If you uploaded your dataset to the hub you can [visualize your dataset online](https://huggingface.co/spaces/lerobot/visualize_dataset) by copy pasting your repo id.
+
+<p align="center">
+  <img
+    src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/dataset_visualizer_sim.png"
+    alt="Figure shows the dataset visualizer"
+    title="Dataset visualization"
+    width="100%"
+  ></img>
+</p>
+<p align="center">
+  <i>Dataset visualizer</i>
+</p>
+
+## Train a policy
+
+To train a policy to control your robot, use the [`lerobot-train`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/scripts/train.py) script. A few arguments are required. Here is an example command:
+
+```bash
+lerobot-train \
+  --dataset.repo_id=${HF_USER}/il_gym \
+  --policy.type=act \
+  --output_dir=outputs/train/il_sim_test \
+  --job_name=il_sim_test \
+  --policy.device=cuda \
+  --wandb.enable=true
+```
+
+Let's explain the command:
+
+1. We provided the dataset as argument with `--dataset.repo_id=${HF_USER}/il_gym`.
+2. We provided the policy with `policy.type=act`. This loads configurations from [`configuration_act.py`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/policies/act/configuration_act.py). Importantly, this policy will automatically adapt to the number of motor states, motor actions and cameras of your robot (e.g. `laptop` and `phone`) which have been saved in your dataset.
+3. We provided `policy.device=cuda` since we are training on a Nvidia GPU, but you could use `policy.device=mps` to train on Apple silicon.
+4. We provided `wandb.enable=true` to use [Weights and Biases](https://docs.wandb.ai/quickstart) for visualizing training plots. This is optional but if you use it, make sure you are logged in by running `wandb login`.
+
+Training should take several hours, 100k steps (which is the default) will take about 1h on Nvidia A100. You will find checkpoints in `outputs/train/il_sim_test/checkpoints`.
+
+#### Train using Collab
+
+If your local computer doesn't have a powerful GPU you could utilize Google Collab to train your model by following the [ACT training notebook](./notebooks#training-act).
+
+#### Upload policy checkpoints
+
+Once training is done, upload the latest checkpoint with:
+
+```bash
+huggingface-cli upload ${HF_USER}/il_sim_test \
+  outputs/train/il_sim_test/checkpoints/last/pretrained_model
+```
+
+You can also upload intermediate checkpoints with:
+
+```bash
+CKPT=010000
+huggingface-cli upload ${HF_USER}/il_sim_test${CKPT} \
+  outputs/train/il_sim_test/checkpoints/${CKPT}/pretrained_model
+```
+
+## Evaluate your policy in Sim
+
+To evaluate your policy we have to use a configuration file. An example can be found [here](https://huggingface.co/datasets/lerobot/config_examples/resolve/main/sim_il/eval_config.json).
+
+Here's an example evaluation configuration:
+
+```json
+{
+  "env": {
+    "type": "gym_manipulator",
+    "name": "gym_hil",
+    "task": "PandaPickCubeGamepad-v0",
+    "fps": 10
+  },
+  "dataset": {
+    "repo_id": "your_username/il_sim_dataset",
+    "dataset_root": null,
+    "task": "pick_cube"
+  },
+  "pretrained_policy_name_or_path": "your_username/il_sim_model",
+  "device": "cuda"
+}
+```
+
+Make sure to replace:
+
+- `repo_id` with the dataset you trained on (e.g., `your_username/il_sim_dataset`)
+- `pretrained_policy_name_or_path` with your model ID (e.g., `your_username/il_sim_model`)
+
+Then you can run this command to visualize your trained policy
+
+<hfoptions id="eval_policy">
+<hfoption id="Linux">
+
+```bash
+python -m lerobot.rl.eval_policy --config_path=path/to/eval_config_gym_hil.json
+```
+
+</hfoption>
+<hfoption id="MacOS">
+
+```bash
+mjpython -m lerobot.rl.eval_policy --config_path=path/to/eval_config_gym_hil.json
+```
+
+</hfoption>
+</hfoptions>
+
+> [!WARNING]
+> While the main workflow of training ACT in simulation is straightforward, there is significant room for exploring how to set up the task, define the initial state of the environment, and determine the type of data required during collection to learn the most effective policy. If your trained policy doesn't perform well, investigate the quality of the dataset it was trained on using our visualizers, as well as the action values and various hyperparameters related to ACT and the simulation.
+
+Congrats 🎉, you have finished this tutorial. If you want to continue with using LeRobot in simulation follow this [Tutorial on reinforcement learning in sim with HIL-SERL](https://huggingface.co/docs/lerobot/hilserl_sim)
+
+> [!TIP]
+> If you have any questions or need help, please reach out on [Discord](https://discord.com/invite/s3KuuzsPFb).

docs/source/installation.mdx

@@ -82,7 +82,7 @@ For a full list of optional dependencies, see:
 https://pypi.org/project/lerobot/
 
 > [!NOTE]
-> For lerobot 0.4.0, if you want to install pi, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`
+> For lerobot 0.4.0, if you want to install libero or pi, you will have to do: `pip install "lerobot[pi,libero]@git+https://github.com/huggingface/lerobot.git"`
 
 ### Troubleshooting
 

docs/source/libero.mdx

@@ -62,11 +62,6 @@ lerobot-eval \
 
 - Pass a comma-separated list to `--env.task` for multi-suite evaluation.
 
-### Control Mode
-
-LIBERO now supports two control modes: relative and absolute. This matters because different VLA checkpoints are trained with different mode of action to output hence control parameterizations.
-You can switch them with: `env.control_mode = "relative"` and `env.control_mode = "absolute"`
-
 ### Policy inputs and outputs
 
 When using LIBERO through LeRobot, policies interact with the environment via **observations** and **actions**:

docs/source/openarms.mdx

@@ -0,0 +1,328 @@
+# OpenArms Robot
+
+OpenArms is a 7 DOF robotic arm with a gripper, designed by [Enactic, Inc.](https://www.enactic.com/) It uses Damiao motors controlled via CAN bus communication and MIT control mode for smooth, precise motion.
+
+## Hardware Overview
+
+- **7 DOF per arm** (14 DOF total for dual arm setup)
+- **1 gripper per arm** (2 grippers total)
+- **Damiao motors** with 4 different types:
+  - **DM8009** (DM-J8009P-2EC) for shoulders (J1, J2) - high torque
+  - **DM4340** for shoulder rotation and elbow (J3, J4)
+  - **DM4310** (DM-J4310-2EC V1.1) for wrist (J5, J6, J7) and gripper (J8)
+- **24V power supply** required
+- **CAN interface device**:
+  - **Linux**: Any SocketCAN-compatible adapter
+  - **macOS**: CANable, PEAK PCAN-USB, or Kvaser USBcan
+- Proper CAN wiring (CANH, CANL, 120Ω termination)
+
+
+## Motor Configuration
+
+Each arm has the following motor configuration based on the [OpenArm setup guide](https://docs.openarm.dev/software/setup/):
+
+| Joint | Motor | Motor Type | Sender CAN ID | Receiver ID | Description |
+|-------|-------|------------|---------------|-------------|-------------|
+| J1 | joint_1 | DM8009 | 0x01 | 0x11 | Shoulder pan |
+| J2 | joint_2 | DM8009 | 0x02 | 0x12 | Shoulder lift |
+| J3 | joint_3 | DM4340 | 0x03 | 0x13 | Shoulder rotation |
+| J4 | joint_4 | DM4340 | 0x04 | 0x14 | Elbow flex |
+| J5 | joint_5 | DM4310 | 0x05 | 0x15 | Wrist roll |
+| J6 | joint_6 | DM4310 | 0x06 | 0x16 | Wrist pitch |
+| J7 | joint_7 | DM4310 | 0x07 | 0x17 | Wrist rotation |
+| J8 | gripper | DM4310 | 0x08 | 0x18 | Gripper |
+
+For dual arm setups, the left arm uses IDs 0x09-0x10 for joints 1-8 with the same motor types.
+
+## Quick Start 
+
+```bash
+# Install system dependencies
+sudo apt install can-utils iproute2
+
+# Install LeRobot with OpenArms support
+pip install -e ".[openarms]"
+```
+
+## Setup Guide
+
+### Step 1: Motor ID Configuration
+
+**IMPORTANT**: Before using the robot, motors must be configured with the correct CAN IDs.
+
+Refer to the [OpenArm Motor ID Configuration Guide](https://docs.openarm.dev/software/setup/motor-id) for detailed instructions using the Damiao Debugging Tools on Windows.
+
+Key points:
+- Each motor needs a unique **Sender CAN ID** (0x01-0x08)
+- Each motor needs a unique **Receiver/Master ID** (0x11-0x18)
+- Use the Damiao Debugging Tools to set these IDs
+
+### Step 2: Setup CAN Interface
+
+Configure your CAN interface as described in the [OpenArm CAN Setup Guide](https://docs.openarm.dev/software/setup/can-setup):
+
+#### Linux (SocketCAN)
+
+```bash
+# Find your CAN interface
+ip link show
+
+# Configure can0, 1, 2, 3
+sudo ip link set can0 down
+sudo ip link set can0 type can bitrate 1000000
+sudo ip link set can0 up
+
+sudo ip link set can1 down
+sudo ip link set can1 type can bitrate 1000000
+sudo ip link set can1 up
+
+sudo ip link set can2 down
+sudo ip link set can2 type can bitrate 1000000
+sudo ip link set can2 up
+
+sudo ip link set can3 down
+sudo ip link set can3 type can bitrate 1000000
+sudo ip link set can3 up
+
+# Verify configuration
+ip link show can0
+```
+
+or run:
+
+`examples/openarms/setup_can.sh`
+
+### Testing canbus and motor connection
+
+Please run this script to check if all motors can be found and to find your can-fd speed: `python examples/openarms/debug_can_communication.py`
+
+## Usage
+
+### Basic Setup
+
+
+```python
+from lerobot.robots.openarms import OpenArmsFollower
+from lerobot.robots.openarms.config_openarms_follower import OpenArmsFollowerConfig
+
+# Configure for dual arm setup
+config = OpenArmsFollowerConfig(
+    port="can0",
+    can_interface="socketcan",  # Or "auto" for auto-detection
+    id="openarms_dual",
+    is_dual_arm=True,
+)
+
+robot = OpenArmsFollower(config)
+robot.connect()
+```
+
+### Calibration
+
+On first use, you'll need to calibrate the robot:
+
+```python
+robot.calibrate()
+```
+
+The calibration process will:
+1. Disable torque on all motors
+2. Ask you to position arms in **hanging position with grippers closed**
+3. Set this as the zero position
+4. Ask you to move each joint through its full range
+5. Record min/max positions for each joint
+6. Save calibration to file
+
+### Reading Observations
+
+The robot provides comprehensive state information:
+
+```python
+observation = robot.get_observation()
+
+# Observation includes for each motor:
+# - {motor_name}.pos: Position in degrees
+# - {motor_name}.vel: Velocity in degrees/second  
+# - {motor_name}.torque: Motor torque
+# - {camera_name}: Camera images (if configured)
+
+print(f"Right arm joint 1 position: {observation['right_joint_1.pos']:.1f}°")
+print(f"Right arm joint 1 velocity: {observation['right_joint_1.vel']:.1f}°/s")
+print(f"Right arm joint 1 torque: {observation['right_joint_1.torque']:.3f} N·m")
+```
+
+### Sending Actions
+
+```python
+# Send target positions (in degrees)
+action = {
+    "right_joint_1.pos": 45.0,
+    "right_joint_2.pos": -30.0,
+    # ... all joints
+    "right_gripper.pos": 45.0,  # Half-closed
+}
+
+actual_action = robot.send_action(action)
+```
+
+### Gripper Control
+
+```python
+# Open gripper
+robot.open_gripper(arm="right")
+
+# Close gripper  
+robot.close_gripper(arm="right")
+```
+
+## Safety Features
+
+### 1. Maximum Relative Target
+
+Limits how far a joint can move in a single command to prevent sudden movements:
+
+```python
+config = OpenArmsFollowerConfig(
+    port="can0",
+    # Limit all joints to 10 degrees per command
+    max_relative_target=10.0,
+    
+    # Or set per-motor limits
+    max_relative_target={
+        "right_joint_1": 15.0,  # Slower moving joint
+        "right_joint_2": 10.0,
+        "right_gripper": 5.0,   # Very slow gripper
+    }
+)
+```
+
+**How it works**: If current position is 50° and you command 80°, with `max_relative_target=10.0`, the robot will only move to 60° in that step.
+
+### 2. Torque Limits
+
+Control maximum torque output, especially important for grippers and teleoperation:
+
+```python
+config = OpenArmsFollowerConfig(
+    port="can0",
+    # Gripper torque limit (fraction of motor's max torque)
+    gripper_torque_limit=0.5,  # 50% of max torque
+)
+```
+
+Lower torque limits prevent damage when gripping delicate objects.
+
+### 3. MIT Control Gains
+
+Control responsiveness and stability via PID-like gains:
+
+```python
+config = OpenArmsFollowerConfig(
+    port="can0",
+    position_kp=10.0,  # Position gain (higher = more responsive)
+    position_kd=0.5,   # Velocity damping (higher = more damped)
+)
+```
+
+**Guidelines**:
+- **For following (robot)**: Higher gains for responsiveness
+  - `position_kp=10.0`, `position_kd=0.5`
+- **For teleoperation (leader)**: Lower gains or disable torque for manual movement
+  - `manual_control=True` (torque disabled)
+
+### 4. Velocity Limits
+
+Velocity limits are enforced by the Damiao motors based on motor type. For DM4310:
+- Max velocity: 30 rad/s ≈ 1718°/s
+
+The motors will automatically limit velocity to safe values.
+
+## Teleoperation
+
+### Leader Arm Setup
+
+The leader arm is moved manually (torque disabled) to generate commands:
+
+```python
+from lerobot.teleoperators.openarms import OpenArmsLeader
+from lerobot.teleoperators.openarms.config_openarms_leader import OpenArmsLeaderConfig
+
+config = OpenArmsLeaderConfig(
+    port="can1",  # Separate CAN interface for leader
+    id="openarms_leader",
+    manual_control=True,  # Torque disabled for manual movement
+    is_dual_arm=True,
+)
+
+leader = OpenArmsLeader(config)
+leader.connect()
+
+# Read current position as action
+action = leader.get_action()
+# action contains positions for all joints in degrees
+```
+
+### Safety Considerations for Teleoperation
+
+1. **Use separate CAN interfaces** for leader and follower to avoid conflicts
+2. **Enable max_relative_target** on follower to smooth abrupt movements
+3. **Lower torque limits** on follower to prevent damage from tracking errors
+4. **Test with one arm** before enabling dual arm teleoperation
+5. **Have emergency stop** ready (power switch or CAN disable)
+
+```python
+# Recommended follower config for teleoperation
+follower_config = OpenArmsFollowerConfig(
+    port="can0",
+    max_relative_target=5.0,  # Small steps for smooth following
+    gripper_torque_limit=0.3, # Low torque for safety
+    position_kp=5.0,  # Lower gains for gentler following
+    position_kd=0.3,
+)
+```
+
+## Troubleshooting
+
+### Motor Shaking/Unstable
+
+- **Lower control gains**: Reduce `position_kp` and `position_kd`
+- **Check calibration**: Re-run calibration procedure
+- **Verify power**: Insufficient current can cause instability
+- **Check mechanical**: Loose connections, binding, or damaged components
+
+### CAN Bus Errors
+
+```bash
+# Check for errors
+ip -s link show can0
+
+# Reset CAN interface
+sudo ip link set can0 down
+sudo ip link set can0 up
+```
+
+### Control Mode
+
+OpenArms uses **MIT control mode** which allows simultaneous control of:
+- Position (degrees)
+- Velocity (degrees/second)
+- Torque (N·m)
+- Position gain (Kp)
+- Velocity damping (Kd)
+
+### Communication
+
+- **Protocol**: CAN 2.0 at 1 Mbps (or CAN-FD at 5 Mbps)
+- **Frame format**: Standard 11-bit IDs
+- **Update rate**: Typically 50-100 Hz depending on motor count
+- **Latency**: ~10-20ms per motor command
+
+## References
+
+- [OpenArm Official Documentation](https://docs.openarm.dev/)
+- [OpenArm Setup Guide](https://docs.openarm.dev/software/setup/)
+- [Motor ID Configuration](https://docs.openarm.dev/software/setup/motor-id)
+- [CAN Interface Setup](https://docs.openarm.dev/software/setup/can-setup)
+- [Motor Communication Test](https://docs.openarm.dev/software/setup/configure-test)
+- [Damiao Motor Documentation](https://wiki.seeedstudio.com/damiao_series/)
+- [Enactic GitHub](https://github.com/enactic/openarm_can)

docs/source/pi0.mdx

@@ -28,11 +28,6 @@ As described by Physical Intelligence, while AI has achieved remarkable success
    pip install -e ".[pi]"
    ```
 
-   > [!NOTE]
-   > For lerobot 0.4.0, if you want to install pi tag, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`.
-   >
-   > This will be solved in the next patch release
-
 ## Training Data and Capabilities
 
 π₀ is trained on the largest robot interaction dataset to date, combining three key data sources:

docs/source/pi05.mdx

@@ -36,11 +36,6 @@ This diverse training mixture creates a "curriculum" that enables generalization
    pip install -e ".[pi]"
    ```
 
-   > [!NOTE]
-   > For lerobot 0.4.0, if you want to install pi tag, you will have to do: `pip install "lerobot[pi]@git+https://github.com/huggingface/lerobot.git"`.
-   >
-   > This will be solved in the next patch release
-
 ## Usage
 
 To use π₀.₅ in your LeRobot configuration, specify the policy type as:

docs/source/rtc.mdx

@@ -1,188 +0,0 @@
-# Real-Time Chunking (RTC)
-
-Real-Time Chunking (RTC) is an inference-time method that allows large, flow-matching based robotic policies, such as [Pi0](./pi0), [Pi0.5](./pi05), and [SmolVLA](./smolvla), to produce smooth, continuous, and reactive motion despite having high inference latency.
-
-These policies generate chunks of future actions (e.g., 50 steps at a time) instead of single actions.
-Because the models are large, producing each chunk takes longer than the time it takes the robot to execute it.
-Naively executing chunks leads to problems such as pauses, jerky transitions, or sudden changes in strategy whenever the next chunk arrives late or disagrees with the previously executed actions.
-
-RTC solves this by asynchronously generating the next chunk while the robot continues executing the current one, and by guiding the new chunk so it aligns smoothly with the portion of the previous chunk that has already been executed.
-
-## How RTC Works (simplified)
-
-RTC lets the robot think ahead while it’s still moving. When the robot is carrying out one chunk of actions, RTC starts creating the next chunk early.
-But since the robot has already moved a bit by the time the new chunk is ready, RTC has to make sure the new chunk still lines up smoothly with what the robot is currently doing.
-
-To do this, RTC treats the beginning of the new chunk like an inpainting or “fill-in-the-gaps” problem:
-it gently adjusts the first part of the new chunk so it blends naturally with the robot’s ongoing motion. The result is no pauses, no sudden jumps.
-
-In technical terms, RTC adds a guidance term to the flow-matching denoising process that forces the overlapping timesteps of the new chunk to stay close to the executed portion of the previous chunk, typically using a soft transition mask.
-
-## Quick Start
-
-### Installation
-
-RTC is built into LeRobot. Just install the policy dependencies you need:
-
-```bash
-# For Pi0 or Pi0.5
-pip install -e ".[pi]"
-
-# For SmolVLA
-pip install -e ".[smolvla]"
-```
-
-### Using RTC with Pi0
-
-You can find a complete reference implementation in [eval_with_real_robot.py](examples/rtc/eval_with_real_robot.py).
-The snippet below provides a simplified pseudo-example of how RTC operates with Pi0 in your pipeline:
-
-```python
-from lerobot.policies.pi0 import PI0Policy, PI0Config
-from lerobot.configs.types import RTCAttentionSchedule
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
-from lerobot.policies.rtc.action_queue import ActionQueue
-
-# Load Pi0 with RTC enabled
-policy_cfg = PI0Config()
-
-# Enable RTC
-policy_cfg.rtc_config = RTCConfig(
-    enabled=True,
-    execution_horizon=10,  # How many steps to blend with previous chunk
-    max_guidance_weight=10.0,  # How strongly to enforce consistency
-    prefix_attention_schedule=RTCAttentionSchedule.EXP,  # Exponential blend
-)
-
-# Load the policy
-policy = PI0Policy.from_pretrained("lerobot/pi0_base", policy_cfg=policy_cfg, device="cuda")
-
-# Now use predict_action_chunk with RTC parameters
-inference_delay = 4  # How many steps of inference latency, this values should be calculated based on the inference latency of the policy
-
-# Initialize the action queue
-action_queue = ActionQueue(policy_cfg.rtc_config)
-
-# Start in a separate thread with the following function
-def get_actions():
-  while True:
-    if should_get_actions:
-
-      prev_actions = action_queue.get_left_over()
-      obs = get_robot_observations(robot)
-
-      # Generate actions WITH RTC
-      actions = policy.predict_action_chunk(
-          obs,
-          inference_delay=inference_delay,
-          prev_chunk_left_over=prev_actions,
-      )
-
-      action_queue.merge(
-          actions, actions, inference_delay
-      )
-
-for step in range(num_steps):
-    action = action_queue.get()
-
-    # Execute the first N actions
-    execute_actions(action)
-```
-
-## Key Parameters
-
-`RTCConfig` has the following parameters to tune:
-
-**`execution_horizon`**: How many timesteps from the previous chunk to maintain consistency with. Higher values mean smoother transitions but potentially less reactivity.
-
-Typical values: 8-12 steps
-
-```python
-RTCConfig(execution_horizon=10)
-```
-
-**`max_guidance_weight`**: How strongly to enforce consistency with the previous chunk. This is a hyperparameter that can be tuned to balance the smoothness of the transitions and the reactivity of the policy. For 10 steps flow matching (SmolVLA, Pi0, Pi0.5), a value of 10.0 is a optimal value.
-
-**`prefix_attention_schedule`**: How to weight consistency across the overlap region.
-
-- `LINEAR`: Linear decay from inference_delay to execution_horizon
-- `EXP`: Exponential decay (recommended for getting started)
-- `ONES`: Full weight across entire execution_horizon
-- `ZEROS`: Binary (full weight up to inference_delay, then zero)
-
-**`inference_delay`**: How many timesteps of inference latency your system has. This is passed to `predict_action_chunk()` rather than the config, since it may vary at runtime.
-
-## Testing RTC Offline
-
-Before running on a real robot, test RTC with dataset samples to visualize how it works:
-
-```bash
-python examples/rtc/eval_dataset.py \
-    --policy.path=lerobot/pi0_libero_finetuned \
-    --dataset.repo_id=HuggingFaceVLA/libero \
-    --rtc.execution_horizon=10 \
-    --rtc.max_guidance_weight=10.0 \
-    --device=cuda
-```
-
-The script generates a visualization of the denoising process, comparing standard generation (left) with RTC (right). In the RTC plots, you can see how the first few steps (blue/purple lines) are guided to match the red ground truth trajectory (previous chunk's tail), ensuring a smooth transition between chunks.
-
-<p align="center">
-  <img
-    src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/flow_matching.png"
-    alt="Denoising steps with and without RTC"
-    width="100%"
-  />
-</p>
-
-## Testing RTC with a Real Robot
-
-```bash
-python examples/rtc/eval_with_real_robot.py \
-    --policy.path=${HF_USERNAME}/policy_repo_id \
-    --robot.type=so100_follower \
-    --robot.port=/dev/tty.usbmodem58FA0834591 \
-    --robot.cameras="{ gripper: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30}, front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \
-    --task="Move green small object into the purple platform" \
-    --duration=120 \
-    --device=cuda
-```
-
-## How It Differs from the Async Inference in LeRobot
-
-Both RTC and [async inference](./async) improve real-time robot control, but they solve different problems.
-
-| Aspect        | Async Inference                                                            | RTC                                                 |
-| ------------- | -------------------------------------------------------------------------- | --------------------------------------------------- |
-| **Problem**   | Idle frames while waiting for inference                                    | Discontinuities between action chunks               |
-| **Solution**  | Decouple prediction from execution                                         | Guide new chunks to continue smoothly from previous |
-| **Benefit**   | No waiting, continuous action                                              | Smooth transitions, natural motion                  |
-| **Best Used** | Async inference is best used with large models with high inference latency | Flow-matching based policies                        |
-
-**Use both together** for maximum smoothness and reactivity!
-
-## Advanced: Debug Tracking
-
-RTC includes built-in debug tracking to help you understand what's happening during inference:
-
-```python
-# Enable debug tracking
-policy_cfg.rtc_config.debug = True
-policy_cfg.rtc_config.debug_maxlen = 100
-
-# After inference, access debug data
-debug_data = policy.rtc_processor.get_debug_data()
-
-# Visualize denoising steps, corrections, etc.
-from lerobot.policies.rtc.debug_visualizer import RTCDebugVisualizer
-visualizer = RTCDebugVisualizer()
-# ... create plots
-```
-
-See `examples/rtc/eval_dataset.py` for a complete example of visualization.
-
-## References
-
-- [Smooth-As-Butter Robot Policies](https://alexander-soare.github.io/robotics/2025/08/05/smooth-as-butter-robot-policies.html) - Excellent technical explanation with real robot results
-- [Physical Intelligence - Real-Time Chunking](https://www.physicalintelligence.company/research/real_time_chunking) - Original paper and research
-- [Kinetix RTC Implementation](https://github.com/Physical-Intelligence/real-time-chunking-kinetix) - Reference implementation from Physical Intelligence

docs/source/xvla.mdx

@@ -1,543 +0,0 @@
-# X-VLA: The First Soft-Prompted Robot Foundation Model for Any Robot, Any Task
-
-## Overview
-
-For years, robotics has aspired to build agents that can follow natural human instructions and operate dexterously across many environments and robot bodies. Recent breakthroughs in LLMs and VLMs suggest a path forward: extend these foundation-model architectures to embodied control by grounding them in actions. This has led to the rise of Vision-Language-Action (VLA) models, with the hope that a single generalist model could combine broad semantic understanding with robust manipulation skills.
-
-But training such models is difficult. Robot data is fragmented across platforms, sensors, embodiments, and collection protocols. Heterogeneity appears everywhere: different arm configurations, different action spaces, different camera setups, different visual domains, and different task distributions. These inconsistencies create major distribution shifts that make pretraining unstable and adaptation unreliable.
-
-Inspired by meta-learning and prompt learning, we ask: **"What if a VLA model could learn the structure of each robot and dataset the same way LLMs learn tasks, through prompts?"**
-
-**X-VLA** is a soft-prompted, flow-matching VLA framework that treats each hardware setup as a "task" and encodes it using a small set of learnable embeddings. These **Soft Prompts** capture embodiment and domain-specific variations, guiding the Transformer from the earliest stages of multimodal fusion. With this mechanism, X-VLA can reconcile diverse robot morphologies, data types, and sensor setups within a single unified architecture.
-
-<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/xvla-architecture.png" width="400">
-
-Built from pure Transformer encoders, X-VLA scales naturally with model size and dataset diversity. Across 6 simulation benchmarks and 3 real robots, Soft Prompts consistently outperform existing methods in handling hardware and domain differences. X-VLA-0.9B, trained on 290K episodes spanning seven robotic platforms, learns an embodiment-agnostic generalist policy in Phase I, and adapts efficiently to new robots in Phase II simply by learning a new set of prompts, while keeping the backbone frozen.
-
-<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/xvla-architecture2.png" width="400">
-
-With only 1% of parameters tuned (9M), X-VLA-0.9B achieves near-π₀ performance on LIBERO and Simpler-WidowX, despite using **300× fewer trainable parameters**. It also demonstrates strong real-world dexterity with minimal demonstrations, including folding cloths in under two minutes.
-
-<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/xvla-fold.png" width="400">
-
-X-VLA shows that generalist robot intelligence does not require increasingly complex architectures, only the right way to absorb heterogeneity. Soft Prompts offer a simple, scalable mechanism for unifying diverse robotic data, paving the way toward adaptable, cross-embodiment robot foundation models.
-
----
-
-## Installation
-
-After installing LeRobot, install the X-VLA dependencies:
-
-```bash
-pip install -e .[xvla]
-```
-
-After the new release, you'll be able to do:
-
-```bash
-pip install lerobot[xvla]
-```
-
----
-
-## Quick Start
-
-### Basic Usage
-
-To use X-VLA in your LeRobot configuration, specify the policy type as:
-
-```bash
-policy.type=xvla
-```
-
-### Evaluating Pre-trained Checkpoints
-
-Example evaluation with LIBERO:
-
-```bash
-lerobot-eval \
-  --policy.path="lerobot/xvla-libero" \
-  --env.type=libero \
-  --env.task=libero_spatial,libero_goal,libero_10 \
-  --env.control_mode=absolute \
-  --eval.batch_size=1 \
-  --eval.n_episodes=1 \
-  --env.episode_length=800 \
-  --seed=142
-```
-
----
-
-## Available Checkpoints
-
-### 🎯 Base Model
-
-**[lerobot/xvla-base](https://huggingface.co/lerobot/xvla-base)**
-
-A 0.9B parameter instantiation of X-VLA, trained with a carefully designed data processing and learning recipe. The training pipeline consists of two phases:
-
-- **Phase I: Pretraining** - Pretrained on 290K episodes from Droid, Robomind, and Agibot, spanning seven platforms across five types of robotic arms (single-arm to bi-manual setups). By leveraging soft prompts to absorb embodiment-specific variations, the model learns an embodiment-agnostic generalist policy.
-
-- **Phase II: Domain Adaptation** - Adapted to deployable policies for target domains. A new set of soft prompts is introduced and optimized to encode the hardware configuration of the novel domain, while the pretrained backbone remains frozen.
-
-### 🎮 Simulation Checkpoints
-
-**[lerobot/xvla-libero](https://huggingface.co/lerobot/xvla-libero)**
-
-Achieves 93% success rate on LIBERO benchmarks. Fine-tuned from the base model for simulation tasks.
-
-**[lerobot/xvla-widowx](https://huggingface.co/lerobot/xvla-widowx)**
-
-Fine-tuned on BridgeData for pick-and-place experiments on compact WidowX platforms. Demonstrates robust manipulation capabilities.
-
-### 🤖 Real-World Checkpoints
-
-**[lerobot/xvla-folding](https://huggingface.co/lerobot/xvla-folding)**
-
-A fine-tuned dexterous manipulation model trained on the high-quality Soft-FOLD cloth folding dataset. Achieves 100% success rate over 2 hours of continuous cloth folding.
-
-**[lerobot/xvla-agibot-world](https://huggingface.co/lerobot/xvla-agibot-world)**
-
-Optimized for AgileX robot dexterous manipulation tasks.
-
-**[lerobot/xvla-google-robot](https://huggingface.co/lerobot/xvla-google-robot)**
-
-Adapted for Google Robot platforms.
-
----
-
-## Training X-VLA
-
-### Recommended Training Configuration
-
-When fine-tuning X-VLA for a new embodiment or task, we recommend the following freezing strategy:
-
-```bash
-lerobot-train \
-  --dataset.repo_id=YOUR_DATASET \
-  --output_dir=./outputs/xvla_training \
-  --job_name=xvla_training \
-  --policy.path="lerobot/xvla-base" \
-  --policy.repo_id="HF_USER/xvla-your-robot" \
-  --steps=3000 \
-  --policy.device=cuda \
-  --policy.freeze_vision_encoder=True \
-  --policy.freeze_language_encoder=True \
-  --policy.train_policy_transformer=True \
-  --policy.train_soft_prompts=True \
-  --policy.action_mode=YOUR_ACTION_MODE
-```
-
-### Training Parameters Explained
-
-| Parameter                  | Default | Description                              |
-| -------------------------- | ------- | ---------------------------------------- |
-| `freeze_vision_encoder`    | `True`  | Freeze the VLM vision encoder weights    |
-| `freeze_language_encoder`  | `True`  | Freeze the VLM language encoder weights  |
-| `train_policy_transformer` | `True`  | Allow policy transformer layers to train |
-| `train_soft_prompts`       | `True`  | Allow soft prompts to train              |
-
-**💡 Best Practice**: For Phase II adaptation to new embodiments, freeze the VLM encoders and only train the policy transformer and soft prompts. This provides excellent sample efficiency with minimal compute.
-
-### Example: Training on Bimanual Robot
-
-```bash
-lerobot-train \
-  --dataset.repo_id=pepijn223/bimanual-so100-handover-cube \
-  --output_dir=./outputs/xvla_bimanual \
-  --job_name=xvla_so101_training \
-  --policy.path="lerobot/xvla-base" \
-  --policy.repo_id="YOUR_USERNAME/xvla-biso101" \
-  --steps=3000 \
-  --policy.device=cuda \
-  --policy.action_mode=so101_bimanual \
-  --policy.freeze_vision_encoder=True \
-  --policy.freeze_language_encoder=True \
-  --policy.train_policy_transformer=True \
-  --policy.train_soft_prompts=True
-```
-
-💡 **Best Performance:** If you have sufficient computational resources and want to achieve best X-VLA finetuning performance, you should follow the official finetuning strategy:
-
-**🔥 Full-finetune all components with a custom learning-rate scheme**
-
-To ensure stable optimization, the Vision-Language Model (VLM) must be trained with only 1/10 of the base learning rate, while all other components use the full LR.
-This LR ratio is crucial for achieving strong and stable finetuning performance.
-To enable this behavior, you must:
-
-1. Implement a custom optimizer and register it in your training config
-
-```
-from dataclasses import dataclass, asdict
-from lerobot.optim.optimizers import OptimizerConfig
-import torch
-
-@OptimizerConfig.register_subclass("xvla-adamw")
-@dataclass
-class XVLAAdamW(OptimizerConfig):
-    lr: float = 1e-4
-    betas: tuple[float, float] = (0.9, 0.99)
-    eps: float = 1e-8
-    weight_decay: float = 0.0
-    grad_clip_norm: float = 10.0
-
-    def build(self, params: dict) -> torch.optim.Optimizer:
-        """
-        Expect `named_parameters()` as input.
-        Apply lr = lr / 10 for all VLM-related parameters.
-        """
-        assert isinstance(params, dict), \
-            "Custom LR optimizer requires `named_parameters()` as inputs."
-        kwargs = asdict(self)
-        kwargs.pop("grad_clip_norm")
-        vlm_group, other_group = [], []
-        for name, p in params.items():
-            if not p.requires_grad:
-                continue
-            if "vlm" in name.lower():
-                vlm_group.append(p)
-            else:
-                other_group.append(p)
-
-        param_groups = [
-            {"params": vlm_group, "lr": self.lr * 0.1, "weight_decay": self.weight_decay * 0.1},
-            {"params": other_group, "lr": self.lr, "weight_decay": self.weight_decay},
-        ]
-
-        return torch.optim.AdamW(param_groups, **kwargs)
-```
-
-2. Modify X-VLA’s get_optim_params to return named parameters
-
-Replace:
-
-```
-def get_optim_params(self) -> dict:
-    """Return only trainable parameters for optimization."""
-    return filter(lambda p: p.requires_grad, self.parameters())
-```
-
-with:
-
-```
-def get_optim_params(self):
-    """Return trainable named parameters."""
-    return filter(lambda kv: kv[1].requires_grad, self.named_parameters())
-```
-
-This ensures the optimizer receives a dict of named parameters, allowing it to correctly detect VLM modules and apply the 1/10 LR rule.
-
-❕Note
-
-Completely matching the official reported performance may require an additional warm-up LR schedule for soft-prompts, which can bring minor improvements.
-We encourage implementing this in your customized training pipeline for optimal results.
-
----
-
-## Core Concepts
-
-### 1. Action Modes
-
-X-VLA uses an **Action Registry** system to handle different action spaces and embodiments. The `action_mode` parameter defines how actions are processed, what loss functions are used, and how predictions are post-processed.
-
-#### Available Action Modes
-
-| Action Mode      | Action Dim            | Description                                 | Use Case                             |
-| ---------------- | --------------------- | ------------------------------------------- | ------------------------------------ |
-| `ee6d`           | 20                    | End-effector with xyz, 6D rotation, gripper | Dual-arm setups with spatial control |
-| `joint`          | 14                    | Joint-space with gripper                    | Direct joint control robots          |
-| `agibot_ee6d`    | 20                    | AGI-bot variant with MSE loss               | AGI-bot platforms                    |
-| `franka_joint7`  | 7                     | Franka Panda 7-joint control                | Franka robots without gripper        |
-| `so101_bimanual` | 20 (model), 12 (real) | SO101 bimanual robot                        | Bimanual manipulation tasks          |
-
-#### Why Action Modes Matter
-
-When you have a pretrained checkpoint like `lerobot/xvla-base` trained with `action_dim=20`, and you want to train on a dataset with a different action dimension (e.g., 14 for bimanual arms), you can't simply trim the action dimension. The action mode orchestrates:
-
-1. **Loss Computation**: Different loss functions for different action components (MSE for joints, BCE for grippers, etc.)
-2. **Preprocessing**: Zeroing out gripper channels, padding dimensions
-3. **Postprocessing**: Applying sigmoid to gripper logits, trimming padding
-
-#### Example: BimanualSO101 Action Space
-
-The `so101_bimanual` action mode handles the mismatch between model output (20D) and real robot control (12D):
-
-```python
-# Model outputs 20 dimensions for compatibility
-dim_action = 20
-
-# Real robot only needs 12 dimensions
-# [left_arm (6), right_arm (6)] = [joints (5) + gripper (1)] × 2
-REAL_DIM = 12
-
-# Preprocessing: Pad 12D actions to 20D for training
-# Postprocessing: Trim 20D predictions to 12D for deployment
-```
-
-See the [action_hub.py](/home/jade_choghari/robot/lerobot/src/lerobot/policies/xvla/action_hub.py) implementation for details.
-
-### 2. Domain IDs
-
-Domain IDs are learnable identifiers for different robot configurations and camera setups. They allow X-VLA to distinguish between:
-
-- Different robots (Robot 1 vs Robot 2)
-- Different camera configurations (cam1 vs cam2)
-- Different combinations (Robot1-cam1-cam2 vs Robot1-cam1 vs Robot2-cam1)
-
-#### Setting Domain IDs
-
-**During Training**: By default, domain_id is set to 0 for general training.
-
-**During Evaluation**: Specify the domain_id that matches your checkpoint's training configuration.
-
-```python
-# Example: LIBERO checkpoint uses domain_id=3
-domain_id = 3
-```
-
-The domain_id is automatically added to observations by the `XVLAAddDomainIdProcessorStep` in the preprocessing pipeline.
-
-### 3. Processor Steps
-
-X-VLA requires specific preprocessing and postprocessing steps for proper operation.
-
-#### Required Preprocessing Steps
-
-1. **XVLAImageToFloatProcessorStep**: Converts images from [0, 255] to [0, 1] range
-2. **XVLAImageNetNormalizeProcessorStep**: Applies ImageNet normalization (required for VLM backbone)
-3. **XVLAAddDomainIdProcessorStep**: Adds domain_id to observations
-
-#### Example Custom Processor
-
-For LIBERO environments, a custom processor handles the specific observation format:
-
-```python
-from lerobot.policies.xvla.processor_xvla import LiberoProcessorStep
-
-processor = LiberoProcessorStep()
-# Handles robot_state dictionary, converts rotation matrices to 6D representation
-# Applies 180° image rotation for camera convention
-```
-
-### 4. Configuration Parameters
-
-Key configuration parameters for X-VLA:
-
-```python
-# Observation and action
-n_obs_steps: int = 1          # Number of observation timesteps
-chunk_size: int = 32           # Action sequence length
-n_action_steps: int = 32       # Number of action steps to execute
-
-# Model architecture
-hidden_size: int = 1024        # Transformer hidden dimension
-depth: int = 24                # Number of transformer layers
-num_heads: int = 16            # Number of attention heads
-num_domains: int = 30          # Maximum number of domain IDs
-len_soft_prompts: int = 32     # Length of soft prompt embeddings
-
-# Action space
-action_mode: str = "ee6d"      # Action space type
-use_proprio: bool = True       # Use proprioceptive state
-max_state_dim: int = 32        # Maximum state dimension
-
-# Vision
-num_image_views: int | None    # Number of camera views
-resize_imgs_with_padding: tuple[int, int] | None  # Target image size with padding
-
-# Training
-num_denoising_steps: int = 10  # Flow matching denoising steps
-```
-
----
-
-## Creating Custom Action Modes
-
-If your robot has a unique action space, you can create a custom action mode:
-
-### Step 1: Define Your Action Space
-
-```python
-from lerobot.policies.xvla.action_hub import BaseActionSpace, register_action
-import torch.nn as nn
-
-@register_action("my_custom_robot")
-class MyCustomActionSpace(BaseActionSpace):
-    """Custom action space for my robot."""
-
-    dim_action = 15  # Your robot's action dimension
-    gripper_idx = (7, 14)  # Gripper channel indices
-
-    def __init__(self):
-        super().__init__()
-        self.mse = nn.MSELoss()
-        self.bce = nn.BCEWithLogitsLoss()
-
-    def compute_loss(self, pred, target):
-        """Define your loss computation."""
-        # Example: MSE for joints, BCE for grippers
-        joints_loss = self.mse(pred[:, :, :7], target[:, :, :7])
-        gripper_loss = self.bce(pred[:, :, self.gripper_idx],
-                                target[:, :, self.gripper_idx])
-
-        return {
-            "joints_loss": joints_loss,
-            "gripper_loss": gripper_loss,
-        }
-
-    def preprocess(self, proprio, action, mode="train"):
-        """Preprocess actions before training."""
-        # Example: Zero out grippers in proprioception
-        proprio_m = proprio.clone()
-        action_m = action.clone() if action is not None else None
-        proprio_m[..., self.gripper_idx] = 0.0
-        if action_m is not None:
-            action_m[..., self.gripper_idx] = 0.0
-        return proprio_m, action_m
-
-    def postprocess(self, action):
-        """Post-process predictions for deployment."""
-        # Example: Apply sigmoid to gripper logits
-        action[..., self.gripper_idx] = torch.sigmoid(action[..., self.gripper_idx])
-        return action
-```
-
-### Step 2: Use Your Custom Action Mode
-
-```bash
-lerobot-train \
-  --policy.action_mode=my_custom_robot \
-  --dataset.repo_id=YOUR_DATASET \
-  --policy.path="lerobot/xvla-base" \
-  ...
-```
-
----
-
-## Advanced Topics
-
-### Multi-Camera Support
-
-X-VLA supports multiple camera views through the `num_image_views` parameter:
-
-```python
-# Configure for 3 camera views
-policy.num_image_views=3
-
-# Add empty cameras if you have fewer physical cameras
-policy.empty_cameras=1  # Adds 1 zero-padded camera view
-```
-
-### Custom Preprocessing Pipeline
-
-Create a custom preprocessing pipeline for your environment:
-
-```python
-from lerobot.processor import PolicyProcessorPipeline
-from lerobot.policies.xvla.processor_xvla import (
-    XVLAImageToFloatProcessorStep,
-    XVLAImageNetNormalizeProcessorStep,
-    XVLAAddDomainIdProcessorStep,
-)
-
-# Build custom pipeline
-preprocessor = PolicyProcessorPipeline(
-    steps=[
-        YourCustomProcessorStep(),  # Your custom processing
-        XVLAImageToFloatProcessorStep(),  # Required: convert to float
-        XVLAImageNetNormalizeProcessorStep(),  # Required: ImageNet norm
-        XVLAAddDomainIdProcessorStep(domain_id=5),  # Your domain ID
-    ]
-)
-```
-
-### Handling Different Action Dimensions
-
-When your dataset has fewer action dimensions than the pretrained model:
-
-**Option 1**: Use padding (automatic in most action modes)
-
-```python
-# Model expects 20D, dataset has 12D
-# Action mode handles padding internally
-action_mode = "so101_bimanual"  # Pads 12 → 20
-```
-
-**Option 2**: Create a custom action mode that maps dimensions explicitly
-
-```python
-@register_action("my_mapped_action")
-class MappedActionSpace(BaseActionSpace):
-    dim_action = 20
-    REAL_DIM = 12
-
-    def _pad_to_model_dim(self, x):
-        # Custom padding logic
-        ...
-```
-
----
-
-## Troubleshooting
-
-### Common Issues
-
-**Issue**: "Action dimension mismatch"
-
-- **Solution**: Check that your `action_mode` matches your robot's action space. Create a custom action mode if needed.
-
-**Issue**: "Image values outside [0, 1] range"
-
-- **Solution**: Ensure images are preprocessed with `XVLAImageToFloatProcessorStep` before normalization.
-
-**Issue**: "Domain ID not found"
-
-- **Solution**: Make sure `XVLAAddDomainIdProcessorStep` is in your preprocessing pipeline with the correct domain_id.
-
-**Issue**: "Low success rate on new embodiment"
-
-- **Solution**:
-  1. Verify your action_mode is correct
-  2. Check that soft prompts are being trained (`train_soft_prompts=True`)
-  3. Ensure proper preprocessing (ImageNet normalization, domain_id)
-  4. Consider increasing training steps
-
-**Issue**: "Out of memory during training"
-
-- **Solution**:
-  1. Reduce `chunk_size` (e.g., from 32 to 16)
-  2. Enable gradient checkpointing
-  3. Reduce batch size
-  4. Freeze more components
-
----
-
-## Citation
-
-If you use X-VLA in your research, please cite:
-
-```bibtex
-@article{zheng2025x,
-  title   = {X-VLA: Soft-Prompted Transformer as Scalable Cross-Embodiment Vision-Language-Action Model},
-  author  = {Zheng, Jinliang and Li, Jianxiong and Wang, Zhihao and Liu, Dongxiu and Kang, Xirui
-             and Feng, Yuchun and Zheng, Yinan and Zou, Jiayin and Chen, Yilun and Zeng, Jia and others},
-  journal = {arXiv preprint arXiv:2510.10274},
-  year    = {2025}
-}
-```
-
----
-
-## Additional Resources
-
-- [X-VLA Paper](https://arxiv.org) (coming soon)
-- [LeRobot Documentation](https://github.com/huggingface/lerobot)
-- [Action Registry Implementation](/home/jade_choghari/robot/lerobot/src/lerobot/policies/xvla/action_hub.py)
-- [Processor Implementation](/home/jade_choghari/robot/lerobot/src/lerobot/policies/xvla/processor_xvla.py)
-- [Model Configuration](/home/jade_choghari/robot/lerobot/src/lerobot/policies/xvla/configuration_xvla.py)
-
----
-
-## Contributing
-
-We welcome contributions! If you've implemented a new action mode or processor for your robot, please consider submitting a PR to help the community.

examples/backward_compatibility/replay.py

@@ -45,7 +45,7 @@ from lerobot.robots import (  # noqa: F401
     so101_follower,
 )
 from lerobot.utils.constants import ACTION
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.utils import (
     init_logging,
     log_say,
@@ -97,7 +97,7 @@ def replay(cfg: ReplayConfig):
         robot.send_action(action)
 
         dt_s = time.perf_counter() - start_episode_t
-        precise_sleep(1 / dataset.fps - dt_s)
+        busy_wait(1 / dataset.fps - dt_s)
 
     robot.disconnect()
 

examples/dataset/load_lerobot_dataset.py

@@ -34,106 +34,105 @@ from huggingface_hub import HfApi
 import lerobot
 from lerobot.datasets.lerobot_dataset import LeRobotDataset, LeRobotDatasetMetadata
 
+# We ported a number of existing datasets ourselves, use this to see the list:
+print("List of available datasets:")
+pprint(lerobot.available_datasets)
 
-def main():
-    # We ported a number of existing datasets ourselves, use this to see the list:
-    print("List of available datasets:")
-    pprint(lerobot.available_datasets)
+# You can also browse through the datasets created/ported by the community on the hub using the hub api:
+hub_api = HfApi()
+repo_ids = [info.id for info in hub_api.list_datasets(task_categories="robotics", tags=["LeRobot"])]
+pprint(repo_ids)
 
-    # You can also browse through the datasets created/ported by the community on the hub using the hub api:
-    hub_api = HfApi()
-    repo_ids = [info.id for info in hub_api.list_datasets(task_categories="robotics", tags=["LeRobot"])]
-    pprint(repo_ids)
+# Or simply explore them in your web browser directly at:
+# https://huggingface.co/datasets?other=LeRobot
 
-    # Or simply explore them in your web browser directly at:
-    # https://huggingface.co/datasets?other=LeRobot
+# Let's take this one for this example
+repo_id = "lerobot/aloha_mobile_cabinet"
+# We can have a look and fetch its metadata to know more about it:
+ds_meta = LeRobotDatasetMetadata(repo_id)
 
-    # Let's take this one for this example
-    repo_id = "lerobot/aloha_mobile_cabinet"
-    # We can have a look and fetch its metadata to know more about it:
-    ds_meta = LeRobotDatasetMetadata(repo_id)
+# By instantiating just this class, you can quickly access useful information about the content and the
+# structure of the dataset without downloading the actual data yet (only metadata files — which are
+# lightweight).
+print(f"Total number of episodes: {ds_meta.total_episodes}")
+print(f"Average number of frames per episode: {ds_meta.total_frames / ds_meta.total_episodes:.3f}")
+print(f"Frames per second used during data collection: {ds_meta.fps}")
+print(f"Robot type: {ds_meta.robot_type}")
+print(f"keys to access images from cameras: {ds_meta.camera_keys=}\n")
 
-    # By instantiating just this class, you can quickly access useful information about the content and the
-    # structure of the dataset without downloading the actual data yet (only metadata files — which are
-    # lightweight).
-    print(f"Total number of episodes: {ds_meta.total_episodes}")
-    print(f"Average number of frames per episode: {ds_meta.total_frames / ds_meta.total_episodes:.3f}")
-    print(f"Frames per second used during data collection: {ds_meta.fps}")
-    print(f"Robot type: {ds_meta.robot_type}")
-    print(f"keys to access images from cameras: {ds_meta.camera_keys=}\n")
+print("Tasks:")
+print(ds_meta.tasks)
+print("Features:")
+pprint(ds_meta.features)
 
-    print("Tasks:")
-    print(ds_meta.tasks)
-    print("Features:")
-    pprint(ds_meta.features)
+# You can also get a short summary by simply printing the object:
+print(ds_meta)
 
-    # You can also get a short summary by simply printing the object:
-    print(ds_meta)
+# You can then load the actual dataset from the hub.
+# Either load any subset of episodes:
+dataset = LeRobotDataset(repo_id, episodes=[0, 10, 11, 23])
 
-    # You can then load the actual dataset from the hub.
-    # Either load any subset of episodes:
-    dataset = LeRobotDataset(repo_id, episodes=[0, 10, 11, 23])
+# And see how many frames you have:
+print(f"Selected episodes: {dataset.episodes}")
+print(f"Number of episodes selected: {dataset.num_episodes}")
+print(f"Number of frames selected: {dataset.num_frames}")
 
-    # And see how many frames you have:
-    print(f"Selected episodes: {dataset.episodes}")
-    print(f"Number of episodes selected: {dataset.num_episodes}")
-    print(f"Number of frames selected: {dataset.num_frames}")
+# Or simply load the entire dataset:
+dataset = LeRobotDataset(repo_id)
+print(f"Number of episodes selected: {dataset.num_episodes}")
+print(f"Number of frames selected: {dataset.num_frames}")
 
-    # Or simply load the entire dataset:
-    dataset = LeRobotDataset(repo_id)
-    print(f"Number of episodes selected: {dataset.num_episodes}")
-    print(f"Number of frames selected: {dataset.num_frames}")
+# The previous metadata class is contained in the 'meta' attribute of the dataset:
+print(dataset.meta)
 
-    # The previous metadata class is contained in the 'meta' attribute of the dataset:
-    print(dataset.meta)
+# LeRobotDataset actually wraps an underlying Hugging Face dataset
+# (see https://huggingface.co/docs/datasets for more information).
+print(dataset.hf_dataset)
 
-    # LeRobotDataset actually wraps an underlying Hugging Face dataset
-    # (see https://huggingface.co/docs/datasets for more information).
-    print(dataset.hf_dataset)
+# LeRobot datasets also subclasses PyTorch datasets so you can do everything you know and love from working
+# with the latter, like iterating through the dataset.
+# The __getitem__ iterates over the frames of the dataset. Since our datasets are also structured by
+# episodes, you can access the frame indices of any episode using dataset.meta.episodes. Here, we access
+# frame indices associated to the first episode:
+episode_index = 0
+from_idx = dataset.meta.episodes["dataset_from_index"][episode_index]
+to_idx = dataset.meta.episodes["dataset_to_index"][episode_index]
 
-    # LeRobot datasets also subclasses PyTorch datasets so you can do everything you know and love from working
-    # with the latter, like iterating through the dataset.
-    # The __getitem__ iterates over the frames of the dataset. Since our datasets are also structured by
-    # episodes, you can access the frame indices of any episode using dataset.meta.episodes. Here, we access
-    # frame indices associated to the first episode:
-    episode_index = 0
-    from_idx = dataset.meta.episodes["dataset_from_index"][episode_index]
-    to_idx = dataset.meta.episodes["dataset_to_index"][episode_index]
+# Then we grab all the image frames from the first camera:
+camera_key = dataset.meta.camera_keys[0]
+frames = [dataset[idx][camera_key] for idx in range(from_idx, to_idx)]
 
-    # Then we grab all the image frames from the first camera:
-    camera_key = dataset.meta.camera_keys[0]
-    frames = [dataset[idx][camera_key] for idx in range(from_idx, to_idx)]
+# The objects returned by the dataset are all torch.Tensors
+print(type(frames[0]))
+print(frames[0].shape)
 
-    # The objects returned by the dataset are all torch.Tensors
-    print(type(frames[0]))
-    print(frames[0].shape)
+# Since we're using pytorch, the shape is in pytorch, channel-first convention (c, h, w).
+# We can compare this shape with the information available for that feature
+pprint(dataset.features[camera_key])
+# In particular:
+print(dataset.features[camera_key]["shape"])
+# The shape is in (h, w, c) which is a more universal format.
 
-    # Since we're using pytorch, the shape is in pytorch, channel-first convention (c, h, w).
-    # We can compare this shape with the information available for that feature
-    pprint(dataset.features[camera_key])
-    # In particular:
-    print(dataset.features[camera_key]["shape"])
-    # The shape is in (h, w, c) which is a more universal format.
+# For many machine learning applications we need to load the history of past observations or trajectories of
+# future actions. Our datasets can load previous and future frames for each key/modality, using timestamps
+# differences with the current loaded frame. For instance:
+delta_timestamps = {
+    # loads 4 images: 1 second before current frame, 500 ms before, 200 ms before, and current frame
+    camera_key: [-1, -0.5, -0.20, 0],
+    # loads 6 state vectors: 1.5 seconds before, 1 second before, ... 200 ms, 100 ms, and current frame
+    "observation.state": [-1.5, -1, -0.5, -0.20, -0.10, 0],
+    # loads 64 action vectors: current frame, 1 frame in the future, 2 frames, ... 63 frames in the future
+    "action": [t / dataset.fps for t in range(64)],
+}
+# Note that in any case, these delta_timestamps values need to be multiples of (1/fps) so that added to any
+# timestamp, you still get a valid timestamp.
 
-    # For many machine learning applications we need to load the history of past observations or trajectories of
-    # future actions. Our datasets can load previous and future frames for each key/modality, using timestamps
-    # differences with the current loaded frame. For instance:
-    delta_timestamps = {
-        # loads 4 images: 1 second before current frame, 500 ms before, 200 ms before, and current frame
-        camera_key: [-1, -0.5, -0.20, 0],
-        # loads 6 state vectors: 1.5 seconds before, 1 second before, ... 200 ms, 100 ms, and current frame
-        "observation.state": [-1.5, -1, -0.5, -0.20, -0.10, 0],
-        # loads 64 action vectors: current frame, 1 frame in the future, 2 frames, ... 63 frames in the future
-        "action": [t / dataset.fps for t in range(64)],
-    }
-    # Note that in any case, these delta_timestamps values need to be multiples of (1/fps) so that added to any
-    # timestamp, you still get a valid timestamp.
-
-    dataset = LeRobotDataset(repo_id, delta_timestamps=delta_timestamps)
-    print(f"\n{dataset[0][camera_key].shape=}")  # (4, c, h, w)
-    print(f"{dataset[0]['observation.state'].shape=}")  # (6, c)
-    print(f"{dataset[0]['action'].shape=}\n")  # (64, c)
+dataset = LeRobotDataset(repo_id, delta_timestamps=delta_timestamps)
+print(f"\n{dataset[0][camera_key].shape=}")  # (4, c, h, w)
+print(f"{dataset[0]['observation.state'].shape=}")  # (6, c)
+print(f"{dataset[0]['action'].shape=}\n")  # (64, c)
 
+if __name__ == "__main__":
     dataloader = torch.utils.data.DataLoader(
         dataset,
         num_workers=4,
@@ -145,7 +144,3 @@ def main():
         print(f"{batch['observation.state'].shape=}")  # (32, 6, c)
         print(f"{batch['action'].shape=}")  # (32, 64, c)
         break
-
-
-if __name__ == "__main__":
-    main()

examples/lekiwi/evaluate.py

@@ -33,68 +33,83 @@ TASK_DESCRIPTION = "My task description"
 HF_MODEL_ID = "<hf_username>/<model_repo_id>"
 HF_DATASET_ID = "<hf_username>/<eval_dataset_repo_id>"
 
+# Create the robot configuration & robot
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
 
-def main():
-    # Create the robot configuration & robot
-    robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
+robot = LeKiwiClient(robot_config)
 
-    robot = LeKiwiClient(robot_config)
+# Create policy
+policy = ACTPolicy.from_pretrained(HF_MODEL_ID)
 
-    # Create policy
-    policy = ACTPolicy.from_pretrained(HF_MODEL_ID)
+# Configure the dataset features
+action_features = hw_to_dataset_features(robot.action_features, ACTION)
+obs_features = hw_to_dataset_features(robot.observation_features, OBS_STR)
+dataset_features = {**action_features, **obs_features}
 
-    # Configure the dataset features
-    action_features = hw_to_dataset_features(robot.action_features, ACTION)
-    obs_features = hw_to_dataset_features(robot.observation_features, OBS_STR)
-    dataset_features = {**action_features, **obs_features}
+# Create the dataset
+dataset = LeRobotDataset.create(
+    repo_id=HF_DATASET_ID,
+    fps=FPS,
+    features=dataset_features,
+    robot_type=robot.name,
+    use_videos=True,
+    image_writer_threads=4,
+)
 
-    # Create the dataset
-    dataset = LeRobotDataset.create(
-        repo_id=HF_DATASET_ID,
+# Build Policy Processors
+preprocessor, postprocessor = make_pre_post_processors(
+    policy_cfg=policy,
+    pretrained_path=HF_MODEL_ID,
+    dataset_stats=dataset.meta.stats,
+    # The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
+    preprocessor_overrides={"device_processor": {"device": str(policy.config.device)}},
+)
+
+# Connect the robot
+# To connect you already should have this script running on LeKiwi: `python -m lerobot.robots.lekiwi.lekiwi_host --robot.id=my_awesome_kiwi`
+robot.connect()
+
+# TODO(Steven): Update this example to use pipelines
+teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
+
+# Initialize the keyboard listener and rerun visualization
+listener, events = init_keyboard_listener()
+init_rerun(session_name="lekiwi_evaluate")
+
+if not robot.is_connected:
+    raise ValueError("Robot is not connected!")
+
+print("Starting evaluate loop...")
+recorded_episodes = 0
+while recorded_episodes < NUM_EPISODES and not events["stop_recording"]:
+    log_say(f"Running inference, recording eval episode {recorded_episodes} of {NUM_EPISODES}")
+
+    # Main record loop
+    record_loop(
+        robot=robot,
+        events=events,
         fps=FPS,
-        features=dataset_features,
-        robot_type=robot.name,
-        use_videos=True,
-        image_writer_threads=4,
+        policy=policy,
+        preprocessor=preprocessor,  # Pass the pre and post policy processors
+        postprocessor=postprocessor,
+        dataset=dataset,
+        control_time_s=EPISODE_TIME_SEC,
+        single_task=TASK_DESCRIPTION,
+        display_data=True,
+        teleop_action_processor=teleop_action_processor,
+        robot_action_processor=robot_action_processor,
+        robot_observation_processor=robot_observation_processor,
     )
 
-    # Build Policy Processors
-    preprocessor, postprocessor = make_pre_post_processors(
-        policy_cfg=policy,
-        pretrained_path=HF_MODEL_ID,
-        dataset_stats=dataset.meta.stats,
-        # The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
-        preprocessor_overrides={"device_processor": {"device": str(policy.config.device)}},
-    )
-
-    # Connect the robot
-    # To connect you already should have this script running on LeKiwi: `python -m lerobot.robots.lekiwi.lekiwi_host --robot.id=my_awesome_kiwi`
-    robot.connect()
-
-    # TODO(Steven): Update this example to use pipelines
-    teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
-
-    # Initialize the keyboard listener and rerun visualization
-    listener, events = init_keyboard_listener()
-    init_rerun(session_name="lekiwi_evaluate")
-
-    if not robot.is_connected:
-        raise ValueError("Robot is not connected!")
-
-    print("Starting evaluate loop...")
-    recorded_episodes = 0
-    while recorded_episodes < NUM_EPISODES and not events["stop_recording"]:
-        log_say(f"Running inference, recording eval episode {recorded_episodes} of {NUM_EPISODES}")
-
-        # Main record loop
+    # Reset the environment if not stopping or re-recording
+    if not events["stop_recording"] and (
+        (recorded_episodes < NUM_EPISODES - 1) or events["rerecord_episode"]
+    ):
+        log_say("Reset the environment")
         record_loop(
             robot=robot,
             events=events,
             fps=FPS,
-            policy=policy,
-            preprocessor=preprocessor,  # Pass the pre and post policy processors
-            postprocessor=postprocessor,
-            dataset=dataset,
             control_time_s=EPISODE_TIME_SEC,
             single_task=TASK_DESCRIPTION,
             display_data=True,
@@ -103,42 +118,21 @@ def main():
             robot_observation_processor=robot_observation_processor,
         )
 
-        # Reset the environment if not stopping or re-recording
-        if not events["stop_recording"] and (
-            (recorded_episodes < NUM_EPISODES - 1) or events["rerecord_episode"]
-        ):
-            log_say("Reset the environment")
-            record_loop(
-                robot=robot,
-                events=events,
-                fps=FPS,
-                control_time_s=EPISODE_TIME_SEC,
-                single_task=TASK_DESCRIPTION,
-                display_data=True,
-                teleop_action_processor=teleop_action_processor,
-                robot_action_processor=robot_action_processor,
-                robot_observation_processor=robot_observation_processor,
-            )
+    if events["rerecord_episode"]:
+        log_say("Re-record episode")
+        events["rerecord_episode"] = False
+        events["exit_early"] = False
+        dataset.clear_episode_buffer()
+        continue
 
-        if events["rerecord_episode"]:
-            log_say("Re-record episode")
-            events["rerecord_episode"] = False
-            events["exit_early"] = False
-            dataset.clear_episode_buffer()
-            continue
+    # Save episode
+    dataset.save_episode()
+    recorded_episodes += 1
 
-        # Save episode
-        dataset.save_episode()
-        recorded_episodes += 1
+# Clean up
+log_say("Stop recording")
+robot.disconnect()
+listener.stop()
 
-    # Clean up
-    log_say("Stop recording")
-    robot.disconnect()
-    listener.stop()
-
-    dataset.finalize()
-    dataset.push_to_hub()
-
-
-if __name__ == "__main__":
-    main()
+dataset.finalize()
+dataset.push_to_hub()

examples/lekiwi/record.py

@@ -34,62 +34,78 @@ RESET_TIME_SEC = 10
 TASK_DESCRIPTION = "My task description"
 HF_REPO_ID = "<hf_username>/<dataset_repo_id>"
 
+# Create the robot and teleoperator configurations
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
+leader_arm_config = SO100LeaderConfig(port="/dev/tty.usbmodem585A0077581", id="my_awesome_leader_arm")
+keyboard_config = KeyboardTeleopConfig()
 
-def main():
-    # Create the robot and teleoperator configurations
-    robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
-    leader_arm_config = SO100LeaderConfig(port="/dev/tty.usbmodem585A0077581", id="my_awesome_leader_arm")
-    keyboard_config = KeyboardTeleopConfig()
+# Initialize the robot and teleoperator
+robot = LeKiwiClient(robot_config)
+leader_arm = SO100Leader(leader_arm_config)
+keyboard = KeyboardTeleop(keyboard_config)
 
-    # Initialize the robot and teleoperator
-    robot = LeKiwiClient(robot_config)
-    leader_arm = SO100Leader(leader_arm_config)
-    keyboard = KeyboardTeleop(keyboard_config)
+# TODO(Steven): Update this example to use pipelines
+teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
 
-    # TODO(Steven): Update this example to use pipelines
-    teleop_action_processor, robot_action_processor, robot_observation_processor = make_default_processors()
+# Configure the dataset features
+action_features = hw_to_dataset_features(robot.action_features, ACTION)
+obs_features = hw_to_dataset_features(robot.observation_features, OBS_STR)
+dataset_features = {**action_features, **obs_features}
 
-    # Configure the dataset features
-    action_features = hw_to_dataset_features(robot.action_features, ACTION)
-    obs_features = hw_to_dataset_features(robot.observation_features, OBS_STR)
-    dataset_features = {**action_features, **obs_features}
+# Create the dataset
+dataset = LeRobotDataset.create(
+    repo_id=HF_REPO_ID,
+    fps=FPS,
+    features=dataset_features,
+    robot_type=robot.name,
+    use_videos=True,
+    image_writer_threads=4,
+)
 
-    # Create the dataset
-    dataset = LeRobotDataset.create(
-        repo_id=HF_REPO_ID,
+# Connect the robot and teleoperator
+# To connect you already should have this script running on LeKiwi: `python -m lerobot.robots.lekiwi.lekiwi_host --robot.id=my_awesome_kiwi`
+robot.connect()
+leader_arm.connect()
+keyboard.connect()
+
+# Initialize the keyboard listener and rerun visualization
+listener, events = init_keyboard_listener()
+init_rerun(session_name="lekiwi_record")
+
+if not robot.is_connected or not leader_arm.is_connected or not keyboard.is_connected:
+    raise ValueError("Robot or teleop is not connected!")
+
+print("Starting record loop...")
+recorded_episodes = 0
+while recorded_episodes < NUM_EPISODES and not events["stop_recording"]:
+    log_say(f"Recording episode {recorded_episodes}")
+
+    # Main record loop
+    record_loop(
+        robot=robot,
+        events=events,
         fps=FPS,
-        features=dataset_features,
-        robot_type=robot.name,
-        use_videos=True,
-        image_writer_threads=4,
+        dataset=dataset,
+        teleop=[leader_arm, keyboard],
+        control_time_s=EPISODE_TIME_SEC,
+        single_task=TASK_DESCRIPTION,
+        display_data=True,
+        teleop_action_processor=teleop_action_processor,
+        robot_action_processor=robot_action_processor,
+        robot_observation_processor=robot_observation_processor,
     )
 
-    # Connect the robot and teleoperator
-    # To connect you already should have this script running on LeKiwi: `python -m lerobot.robots.lekiwi.lekiwi_host --robot.id=my_awesome_kiwi`
-    robot.connect()
-    leader_arm.connect()
-    keyboard.connect()
-
-    # Initialize the keyboard listener and rerun visualization
-    listener, events = init_keyboard_listener()
-    init_rerun(session_name="lekiwi_record")
-
-    if not robot.is_connected or not leader_arm.is_connected or not keyboard.is_connected:
-        raise ValueError("Robot or teleop is not connected!")
-
-    print("Starting record loop...")
-    recorded_episodes = 0
-    while recorded_episodes < NUM_EPISODES and not events["stop_recording"]:
-        log_say(f"Recording episode {recorded_episodes}")
-
-        # Main record loop
+    # Reset the environment if not stopping or re-recording
+    if not events["stop_recording"] and (
+        (recorded_episodes < NUM_EPISODES - 1) or events["rerecord_episode"]
+    ):
+        log_say("Reset the environment")
         record_loop(
             robot=robot,
             events=events,
             fps=FPS,
-            dataset=dataset,
             teleop=[leader_arm, keyboard],
-            control_time_s=EPISODE_TIME_SEC,
+            control_time_s=RESET_TIME_SEC,
             single_task=TASK_DESCRIPTION,
             display_data=True,
             teleop_action_processor=teleop_action_processor,
@@ -97,45 +113,23 @@ def main():
             robot_observation_processor=robot_observation_processor,
         )
 
-        # Reset the environment if not stopping or re-recording
-        if not events["stop_recording"] and (
-            (recorded_episodes < NUM_EPISODES - 1) or events["rerecord_episode"]
-        ):
-            log_say("Reset the environment")
-            record_loop(
-                robot=robot,
-                events=events,
-                fps=FPS,
-                teleop=[leader_arm, keyboard],
-                control_time_s=RESET_TIME_SEC,
-                single_task=TASK_DESCRIPTION,
-                display_data=True,
-                teleop_action_processor=teleop_action_processor,
-                robot_action_processor=robot_action_processor,
-                robot_observation_processor=robot_observation_processor,
-            )
+    if events["rerecord_episode"]:
+        log_say("Re-record episode")
+        events["rerecord_episode"] = False
+        events["exit_early"] = False
+        dataset.clear_episode_buffer()
+        continue
 
-        if events["rerecord_episode"]:
-            log_say("Re-record episode")
-            events["rerecord_episode"] = False
-            events["exit_early"] = False
-            dataset.clear_episode_buffer()
-            continue
+    # Save episode
+    dataset.save_episode()
+    recorded_episodes += 1
 
-        # Save episode
-        dataset.save_episode()
-        recorded_episodes += 1
+# Clean up
+log_say("Stop recording")
+robot.disconnect()
+leader_arm.disconnect()
+keyboard.disconnect()
+listener.stop()
 
-    # Clean up
-    log_say("Stop recording")
-    robot.disconnect()
-    leader_arm.disconnect()
-    keyboard.disconnect()
-    listener.stop()
-
-    dataset.finalize()
-    dataset.push_to_hub()
-
-
-if __name__ == "__main__":
-    main()
+dataset.finalize()
+dataset.push_to_hub()

examples/lekiwi/replay.py

@@ -20,48 +20,42 @@ from lerobot.datasets.lerobot_dataset import LeRobotDataset
 from lerobot.robots.lekiwi.config_lekiwi import LeKiwiClientConfig
 from lerobot.robots.lekiwi.lekiwi_client import LeKiwiClient
 from lerobot.utils.constants import ACTION
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.utils import log_say
 
 EPISODE_IDX = 0
 
+# Initialize the robot config
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
 
-def main():
-    # Initialize the robot config
-    robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="lekiwi")
+# Initialize the robot
+robot = LeKiwiClient(robot_config)
 
-    # Initialize the robot
-    robot = LeKiwiClient(robot_config)
+# Fetch the dataset to replay
+dataset = LeRobotDataset("<hf_username>/<dataset_repo_id>", episodes=[EPISODE_IDX])
+# Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
+episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == EPISODE_IDX)
+actions = episode_frames.select_columns(ACTION)
 
-    # Fetch the dataset to replay
-    dataset = LeRobotDataset("<hf_username>/<dataset_repo_id>", episodes=[EPISODE_IDX])
-    # Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
-    episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == EPISODE_IDX)
-    actions = episode_frames.select_columns(ACTION)
+# Connect to the robot
+robot.connect()
 
-    # Connect to the robot
-    robot.connect()
+if not robot.is_connected:
+    raise ValueError("Robot is not connected!")
 
-    if not robot.is_connected:
-        raise ValueError("Robot is not connected!")
+print("Starting replay loop...")
+log_say(f"Replaying episode {EPISODE_IDX}")
+for idx in range(len(episode_frames)):
+    t0 = time.perf_counter()
 
-    print("Starting replay loop...")
-    log_say(f"Replaying episode {EPISODE_IDX}")
-    for idx in range(len(episode_frames)):
-        t0 = time.perf_counter()
+    # Get recorded action from dataset
+    action = {
+        name: float(actions[idx][ACTION][i]) for i, name in enumerate(dataset.features[ACTION]["names"])
+    }
 
-        # Get recorded action from dataset
-        action = {
-            name: float(actions[idx][ACTION][i]) for i, name in enumerate(dataset.features[ACTION]["names"])
-        }
+    # Send action to robot
+    _ = robot.send_action(action)
 
-        # Send action to robot
-        _ = robot.send_action(action)
+    busy_wait(max(1.0 / dataset.fps - (time.perf_counter() - t0), 0.0))
 
-        precise_sleep(max(1.0 / dataset.fps - (time.perf_counter() - t0), 0.0))
-
-    robot.disconnect()
-
-
-if __name__ == "__main__":
-    main()
+robot.disconnect()

examples/lekiwi/teleoperate.py

@@ -19,60 +19,54 @@ import time
 from lerobot.robots.lekiwi import LeKiwiClient, LeKiwiClientConfig
 from lerobot.teleoperators.keyboard.teleop_keyboard import KeyboardTeleop, KeyboardTeleopConfig
 from lerobot.teleoperators.so100_leader import SO100Leader, SO100LeaderConfig
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.visualization_utils import init_rerun, log_rerun_data
 
 FPS = 30
 
+# Create the robot and teleoperator configurations
+robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="my_lekiwi")
+teleop_arm_config = SO100LeaderConfig(port="/dev/tty.usbmodem585A0077581", id="my_awesome_leader_arm")
+keyboard_config = KeyboardTeleopConfig(id="my_laptop_keyboard")
 
-def main():
-    # Create the robot and teleoperator configurations
-    robot_config = LeKiwiClientConfig(remote_ip="172.18.134.136", id="my_lekiwi")
-    teleop_arm_config = SO100LeaderConfig(port="/dev/tty.usbmodem585A0077581", id="my_awesome_leader_arm")
-    keyboard_config = KeyboardTeleopConfig(id="my_laptop_keyboard")
+# Initialize the robot and teleoperator
+robot = LeKiwiClient(robot_config)
+leader_arm = SO100Leader(teleop_arm_config)
+keyboard = KeyboardTeleop(keyboard_config)
 
-    # Initialize the robot and teleoperator
-    robot = LeKiwiClient(robot_config)
-    leader_arm = SO100Leader(teleop_arm_config)
-    keyboard = KeyboardTeleop(keyboard_config)
+# Connect to the robot and teleoperator
+# To connect you already should have this script running on LeKiwi: `python -m lerobot.robots.lekiwi.lekiwi_host --robot.id=my_awesome_kiwi`
+robot.connect()
+leader_arm.connect()
+keyboard.connect()
 
-    # Connect to the robot and teleoperator
-    # To connect you already should have this script running on LeKiwi: `python -m lerobot.robots.lekiwi.lekiwi_host --robot.id=my_awesome_kiwi`
-    robot.connect()
-    leader_arm.connect()
-    keyboard.connect()
+# Init rerun viewer
+init_rerun(session_name="lekiwi_teleop")
 
-    # Init rerun viewer
-    init_rerun(session_name="lekiwi_teleop")
+if not robot.is_connected or not leader_arm.is_connected or not keyboard.is_connected:
+    raise ValueError("Robot or teleop is not connected!")
 
-    if not robot.is_connected or not leader_arm.is_connected or not keyboard.is_connected:
-        raise ValueError("Robot or teleop is not connected!")
+print("Starting teleop loop...")
+while True:
+    t0 = time.perf_counter()
 
-    print("Starting teleop loop...")
-    while True:
-        t0 = time.perf_counter()
+    # Get robot observation
+    observation = robot.get_observation()
 
-        # Get robot observation
-        observation = robot.get_observation()
+    # Get teleop action
+    # Arm
+    arm_action = leader_arm.get_action()
+    arm_action = {f"arm_{k}": v for k, v in arm_action.items()}
+    # Keyboard
+    keyboard_keys = keyboard.get_action()
+    base_action = robot._from_keyboard_to_base_action(keyboard_keys)
 
-        # Get teleop action
-        # Arm
-        arm_action = leader_arm.get_action()
-        arm_action = {f"arm_{k}": v for k, v in arm_action.items()}
-        # Keyboard
-        keyboard_keys = keyboard.get_action()
-        base_action = robot._from_keyboard_to_base_action(keyboard_keys)
+    action = {**arm_action, **base_action} if len(base_action) > 0 else arm_action
 
-        action = {**arm_action, **base_action} if len(base_action) > 0 else arm_action
+    # Send action to robot
+    _ = robot.send_action(action)
 
-        # Send action to robot
-        _ = robot.send_action(action)
+    # Visualize
+    log_rerun_data(observation=observation, action=action)
 
-        # Visualize
-        log_rerun_data(observation=observation, action=action)
-
-        precise_sleep(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))
-
-
-if __name__ == "__main__":
-    main()
+    busy_wait(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))

examples/openarms/debug_can_communication.py

@@ -0,0 +1,416 @@
+#!/usr/bin/env python3
+"""
+Comprehensive debug script for OpenArms CAN FD communication.
+Tests all 4 CAN interfaces with CAN FD support.
+"""
+
+import can
+import time
+import sys
+import subprocess
+
+def check_can_interface(port):
+    """Check if CAN interface is UP and configured."""
+    try:
+        result = subprocess.run(['ip', 'link', 'show', port], 
+                              capture_output=True, text=True)
+        if result.returncode != 0:
+            return False, "Interface not found", None
+        
+        output = result.stdout
+        if 'UP' not in output:
+            return False, "Interface is DOWN", None
+        
+        # Check if CAN FD is enabled
+        is_fd = 'fd on' in output.lower() or 'canfd' in output.lower()
+        
+        return True, "Interface is UP", is_fd
+    except FileNotFoundError:
+        return None, "Cannot check (ip command not found)", None
+
+
+def test_motor_on_interface(bus, motor_id, timeout=2.0, use_fd=False):
+    """
+    Test a single motor and return all responses.
+    
+    Returns:
+        list of (arbitration_id, data) tuples for all responses received
+    """
+    # Send enable command
+    enable_msg = can.Message(
+        arbitration_id=motor_id,
+        data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFC],
+        is_extended_id=False,
+        is_fd=use_fd
+    )
+    
+    try:
+        bus.send(enable_msg)
+    except Exception as e:
+        return None, f"Send error: {e}"
+    
+    # Listen for responses
+    responses = []
+    start_time = time.time()
+    
+    while time.time() - start_time < timeout:
+        msg = bus.recv(timeout=0.1)
+        if msg:
+            responses.append((msg.arbitration_id, msg.data, msg.is_fd if hasattr(msg, 'is_fd') else False))
+    
+    # Send disable command
+    disable_msg = can.Message(
+        arbitration_id=motor_id,
+        data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD],
+        is_extended_id=False,
+        is_fd=use_fd
+    )
+    try:
+        bus.send(disable_msg)
+    except:
+        pass
+    
+    return responses, None
+
+
+def test_interface(port, interface_type="socketcan", use_can_fd=True):
+    """Test all 8 motors on a single CAN interface."""
+    
+    results = {
+        'interface': port,
+        'status': None,
+        'is_fd': use_can_fd,
+        'motors': {}
+    }
+    
+    # Check interface status
+    status_ok, status_msg, interface_has_fd = check_can_interface(port)
+    
+    if interface_has_fd is not None:
+        results['interface_fd_enabled'] = interface_has_fd
+        if use_can_fd and not interface_has_fd:
+            status_msg += " (CAN FD NOT enabled on interface!)"
+        elif interface_has_fd:
+            status_msg += " (CAN FD enabled)"
+    
+    results['status'] = status_msg
+    
+    if status_ok is False:
+        return results
+    
+    # Try to connect
+    try:
+        if use_can_fd:
+            print(f"  Connecting to {port} with CAN FD (1 Mbps / 5 Mbps)...")
+            bus = can.interface.Bus(
+                channel=port,
+                interface=interface_type,
+                bitrate=1000000,
+                data_bitrate=5000000,
+                fd=True
+            )
+        else:
+            print(f"  Connecting to {port} with CAN 2.0 (1 Mbps)...")
+            bus = can.interface.Bus(
+                channel=port,
+                interface=interface_type,
+                bitrate=1000000
+            )
+    except Exception as e:
+        results['status'] = f"Connection failed: {e}"
+        return results
+    
+    try:
+        # Clear any pending messages
+        while bus.recv(timeout=0.01):
+            pass
+        
+        # Test each motor (0x01 to 0x08)
+        for motor_id in range(0x01, 0x09):
+            responses, error = test_motor_on_interface(bus, motor_id, timeout=1.0, use_fd=use_can_fd)
+            
+            if error:
+                results['motors'][motor_id] = {'error': error}
+            elif responses:
+                results['motors'][motor_id] = {
+                    'found': True,
+                    'responses': responses
+                }
+            else:
+                results['motors'][motor_id] = {
+                    'found': False,
+                    'responses': []
+                }
+            
+            time.sleep(0.05)  # Small delay between motors
+        
+    finally:
+        bus.shutdown()
+    
+    return results
+
+
+def print_results(all_results):
+    """Print formatted results for all interfaces."""
+    
+    print("SUMMARY - Motors Found on Each Interface")
+    
+    motor_names = {
+        0x01: "joint_1 (Shoulder pan)",
+        0x02: "joint_2 (Shoulder lift)",
+        0x03: "joint_3 (Shoulder rotation)",
+        0x04: "joint_4 (Elbow flex)",
+        0x05: "joint_5 (Wrist roll)",
+        0x06: "joint_6 (Wrist pitch)",
+        0x07: "joint_7 (Wrist rotation)",
+        0x08: "gripper",
+    }
+    
+    total_found = 0
+    
+    for result in all_results:
+        interface = result['interface']
+        status = result['status']
+        
+        print(f"{interface}: {status}")
+        if result.get('is_fd'):
+            print(f"  Mode: CAN FD")
+        else:
+            print(f"  Mode: CAN 2.0")
+        
+        if 'Connection failed' in status or 'DOWN' in status:
+            print(f"  ⚠ Cannot test {interface}")
+            continue
+        
+        motors_found = 0
+        
+        for motor_id in range(0x01, 0x09):
+            motor_data = result['motors'].get(motor_id, {})
+            motor_name = motor_names.get(motor_id, "Unknown")
+            
+            if motor_data.get('error'):
+                print(f"  Motor 0x{motor_id:02X} ({motor_name}): ✗ {motor_data['error']}")
+            elif motor_data.get('found'):
+                motors_found += 1
+                total_found += 1
+                responses = motor_data['responses']
+                print(f"  Motor 0x{motor_id:02X} ({motor_name}): ✓ FOUND")
+                
+                for resp_id, data, is_fd in responses:
+                    data_hex = data.hex()
+                    fd_flag = " [FD]" if is_fd else " [2.0]"
+                    print(f"    → Response from 0x{resp_id:02X}{fd_flag}: {data_hex}")
+            else:
+                print(f"  Motor 0x{motor_id:02X} ({motor_name}): ✗ No response")
+        
+        print(f"\n  Summary: {motors_found}/8 motors found on {interface}")
+    
+    # Overall summary
+    print("OVERALL SUMMARY")
+    print(f"Total motors found across all interfaces: {total_found}")
+    
+    # Analyze configuration
+    print("DIAGNOSIS")
+    
+    for result in all_results:
+        interface = result['interface']
+        motors_found = sum(1 for m in result['motors'].values() if m.get('found'))
+        
+        if motors_found == 0:
+            print(f"\n⚠ {interface}: NO MOTORS FOUND")
+            print("  Possible issues:")
+            print("    1. CAN FD mode mismatch (interface vs motor configuration)")
+            print("    2. Missing 120Ω termination resistors at BOTH cable ends")
+            print("    3. Motor timeout parameter set incorrectly (should NOT be 0)")
+            print("    4. CANH/CANL wiring issue")
+            print("    5. Cable too long (>40m for CAN FD at 5Mbps)")
+            
+            # Check FD mismatch
+            if result.get('is_fd') and not result.get('interface_fd_enabled'):
+                print("    ⚠️ CRITICAL: Trying CAN FD but interface NOT configured for FD!")
+                print(f"       Fix: sudo ip link set {interface} type can bitrate 1000000 dbitrate 5000000 fd on")
+                
+        elif motors_found < 8:
+            print(f"\n⚠ {interface}: Only {motors_found}/8 motors responding")
+            print("  Check power and connections for missing motors")
+        else:
+            print(f"\n✓ {interface}: All 8 motors responding correctly!")
+    
+    # Check for unexpected response IDs
+    print("RESPONSE ID ANALYSIS")
+    
+    for result in all_results:
+        interface = result['interface']
+        unexpected = []
+        
+        for motor_id, motor_data in result['motors'].items():
+            if motor_data.get('found'):
+                expected_id = motor_id + 0x10
+                actual_ids = [resp[0] for resp in motor_data['responses']]
+                
+                if expected_id not in actual_ids:
+                    unexpected.append((motor_id, actual_ids))
+        
+        if unexpected:
+            print(f"\n⚠ {interface}: Unexpected response IDs detected")
+            for motor_id, actual_ids in unexpected:
+                expected_id = motor_id + 0x10
+                print(f"  Motor 0x{motor_id:02X}: Expected 0x{expected_id:02X}, "
+                      f"got {[f'0x{id:02X}' for id in actual_ids]}")
+            print("  → Motor Master IDs need reconfiguration")
+        else:
+            motors_found = sum(1 for m in result['motors'].values() if m.get('found'))
+            if motors_found > 0:
+                print(f"\n✓ {interface}: All responding motors use correct IDs")
+
+
+def test_communication_speed(interface, motor_id, num_iterations=100):
+    """
+    Test communication speed with a motor.
+    
+    Returns:
+        tuple: (hz, avg_latency_ms) or (None, None) if test failed
+    """
+    try:
+        # Connect to interface
+        bus = can.interface.Bus(
+            channel=interface,
+            interface="socketcan",
+            bitrate=1000000,
+            data_bitrate=5000000,
+            fd=True
+        )
+        
+        # Send refresh commands and measure round-trip time
+        latencies = []
+        successful = 0
+        
+        for _ in range(num_iterations):
+            start = time.perf_counter()
+            
+            # Send enable command (lightweight operation)
+            enable_msg = can.Message(
+                arbitration_id=motor_id,
+                data=[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFC],
+                is_extended_id=False,
+                is_fd=True
+            )
+            bus.send(enable_msg)
+            
+            # Wait for response
+            msg = bus.recv(timeout=0.1)
+            
+            if msg:
+                latency = (time.perf_counter() - start) * 1000  # Convert to ms
+                latencies.append(latency)
+                successful += 1
+        
+        bus.shutdown()
+        
+        if successful > 0:
+            avg_latency = sum(latencies) / len(latencies)
+            hz = 1000.0 / avg_latency if avg_latency > 0 else 0
+            return hz, avg_latency
+        
+        return None, None
+        
+    except Exception as e:
+        print(f"    Speed test error: {e}")
+        return None, None
+
+
+def main():
+    """Main function to test all CAN interfaces with CAN FD."""
+    
+    print("\nThis will test all 4 CAN interfaces (can0-can3) with CAN FD")
+    print("Testing motors 0x01-0x08 on each interface")
+    print()
+    print("Make sure:")
+    print("  ✓ Motors are powered (24V)")
+    print("  ✓ CAN interfaces configured with FD mode:")
+    print("    ./examples/openarms/setup_can.sh")
+    print("  ✓ Motor 'timeout' parameter NOT set to 0 (use Damiao tools)")
+    print("  ✓ CAN wiring includes 120Ω termination at BOTH ends")
+    print()
+    
+    input("Press ENTER to start testing...")
+    
+    # Test all 4 interfaces with CAN FD
+    all_results = []
+    
+    for i in range(4):
+        interface = f"can{i}"
+        print(f"Testing {interface}...")
+        
+        result = test_interface(interface, use_can_fd=True)
+        all_results.append(result)
+        
+        # Quick status
+        if 'Connection failed' in result['status'] or 'DOWN' in result['status']:
+            print(f"  ⚠ {interface}: {result['status']}")
+        else:
+            motors_found = sum(1 for m in result['motors'].values() if m.get('found'))
+            print(f"  {interface}: {motors_found}/8 motors found")
+        
+        time.sleep(0.2)
+    
+    # Print detailed results
+    print_results(all_results)
+    
+    print("Testing Complete!")
+    
+    all_found = sum(sum(1 for m in r['motors'].values() if m.get('found')) for r in all_results)
+    
+    if all_found == 0:
+        print("\n⚠️ CRITICAL: No motors found on any interface!")
+        print("\nTop issues to check:")
+        print("  1. Motor 'timeout' parameter (use Damiao tools to set > 0)")
+        print("  2. CAN FD not enabled (run ./examples/openarms/setup_can.sh)")
+        print("  3. Missing termination resistors")
+        print("\nTry:")
+        print("  a) Check motor parameters with Damiao Debugging Tools")
+        print("  b) Verify CAN FD is enabled: ip -d link show can0 | grep fd")
+        print("  c) Run setup script: ./examples/openarms/setup_can.sh")
+    else:
+        # Run speed test on interfaces with motors
+        print("COMMUNICATION SPEED TEST")
+        print("\nTesting maximum communication frequency...")
+        
+        for result in all_results:
+            interface = result['interface']
+            
+            # Find first responding motor
+            responding_motor = None
+            for motor_id, motor_data in result['motors'].items():
+                if motor_data.get('found'):
+                    responding_motor = motor_id
+                    break
+            
+            if responding_motor:
+                print(f"\n{interface}: Testing with motor 0x{responding_motor:02X}...")
+                hz, latency = test_communication_speed(interface, responding_motor, num_iterations=100)
+                
+                if hz:
+                    print(f"  ✓ Max frequency: {hz:.1f} Hz")
+                    print(f"  ✓ Avg latency: {latency:.2f} ms")
+                    print(f"  ✓ Commands per second: ~{int(hz)}")
+                else:
+                    print(f"  ✗ Speed test failed")
+            else:
+                print(f"\n{interface}: No motors found, skipping speed test")
+
+        print()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\n\nTesting interrupted by user.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\nUnexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+

examples/openarms/setup_can.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Setup all OpenArms CAN interfaces with CAN FD
+
+set -e
+
+echo "=========================================="
+echo "OpenArms CAN FD Interface Setup"
+echo "=========================================="
+echo ""
+echo "Mode: CAN FD"
+echo "  - Nominal bitrate: 1 Mbps"
+echo "  - Data bitrate: 5 Mbps"
+echo ""
+echo "Configuring interfaces can0, can1, can2, can3..."
+echo ""
+
+# Configure each CAN interface with CAN FD
+for i in 0 1 2 3; do
+    interface="can$i"
+    
+    # Check if interface exists
+    if ! ip link show "$interface" &> /dev/null; then
+        echo "⚠ $interface: Not found, skipping"
+        continue
+    fi
+    
+    # Bring down interface
+    sudo ip link set "$interface" down 2>/dev/null
+    
+    # Configure CAN FD mode
+    sudo ip link set "$interface" type can \
+        bitrate 1000000 \
+        dbitrate 5000000 \
+        fd on
+    
+    # Bring up interface
+    sudo ip link set "$interface" up
+    
+    # Verify configuration
+    if ip link show "$interface" | grep -q "UP"; then
+        echo "✓ $interface: Configured and UP"
+    else
+        echo "✗ $interface: Failed to bring UP"
+    fi
+done
+
+echo ""
+echo "=========================================="
+echo "Verification"
+echo "=========================================="
+echo ""
+
+# Show detailed status for each interface
+for i in 0 1 2 3; do
+    interface="can$i"
+    if ip link show "$interface" &> /dev/null; then
+        echo "$interface:"
+        # Show key parameters
+        ip -d link show "$interface" | grep -E "can|state|bitrate|dbitrate" | head -3
+        echo ""
+    fi
+done
+
+echo "=========================================="
+echo "Setup Complete!"
+echo "=========================================="
+echo ""
+echo "All interfaces configured for CAN FD mode"
+echo ""
+echo "Next steps:"
+echo "  1. Test motors: python debug_can_communication.py"
+echo "  2. Run teleoperation: python examples/openarms/teleop.py"
+echo ""

examples/openarms/teleop.py

@@ -0,0 +1,148 @@
+"""
+OpenArms Teleoperation Example - Full Dual Arms
+
+This script demonstrates teleoperation of OpenArms follower robot using an OpenArms leader arm.
+It first calibrates both devices, then enters a teleoperation loop for both arms.
+"""
+
+import time
+
+from lerobot.robots.openarms.openarms_follower import OpenArmsFollower
+from lerobot.robots.openarms.config_openarms_follower import OpenArmsFollowerConfig
+from lerobot.teleoperators.openarms.openarms_leader import OpenArmsLeader
+from lerobot.teleoperators.openarms.config_openarms_leader import OpenArmsLeaderConfig
+
+
+follower_config = OpenArmsFollowerConfig(
+    port_left="can0",   # CAN interface for follower left arm
+    port_right="can1",  # CAN interface for follower right arm
+    can_interface="socketcan",  # Linux SocketCAN
+    id="openarms_follower",
+    disable_torque_on_disconnect=True,
+    max_relative_target=5.0,  # Safety limit
+)
+
+
+leader_config = OpenArmsLeaderConfig(
+    port_left="can2",   # CAN interface for leader left arm
+    port_right="can3",  # CAN interface for leader right arm
+    can_interface="socketcan",  # Linux SocketCAN
+    id="openarms_leader",
+    manual_control=True,  # Enable manual control (torque disabled)
+)
+
+print("=" * 60)
+print("OpenArms Teleoperation - Full Dual Arms")
+print("=" * 60)
+
+# Initialize devices
+print("\n[1/4] Initializing devices...")
+follower = OpenArmsFollower(follower_config)
+leader = OpenArmsLeader(leader_config)
+
+# Connect and calibrate follower
+print("\n[2/4] Connecting and calibrating follower robot...")
+print("Note: If you have existing calibration, just press ENTER to use it.")
+follower.connect(calibrate=True)
+
+# Connect and calibrate leader
+print("\n[3/4] Connecting and calibrating leader arm...")
+print("Note: The leader arm will have torque disabled for manual control.")
+leader.connect(calibrate=True)
+
+# Wait for user to be ready
+print("\n[4/4] Ready for teleoperation!")
+print("\nBoth arms will be controlled (16 motors total):")
+print("  RIGHT ARM: joints 1-7 + gripper")
+print("  LEFT ARM: joints 1-7 + gripper")
+
+print("\nPress ENTER to start teleoperation...")
+input()
+
+print("\nTeleoperation started! Move both leader arms.")
+print("Press Ctrl+C to stop.\n")
+
+# All joints for both arms (16 motors total)
+all_joints = [
+    # Right arm
+    "right_joint_1",
+    "right_joint_2",
+    "right_joint_3",
+    "right_joint_4",
+    "right_joint_5",
+    "right_joint_6",
+    "right_joint_7",
+    "right_gripper",
+    # Left arm
+    "left_joint_1",
+    "left_joint_2",
+    "left_joint_3",
+    "left_joint_4",
+    "left_joint_5",
+    "left_joint_6",
+    "left_joint_7",
+    "left_gripper",
+]
+
+# Performance monitoring
+loop_times = []
+start_time = time.perf_counter()
+last_print_time = start_time
+
+try:
+    while True:
+        loop_start = time.perf_counter()
+        
+        # Get action from leader
+        leader_action = leader.get_action()
+        
+        # Filter to only position data for all joints (both arms)
+        joint_action = {}
+        for joint in all_joints:
+            pos_key = f"{joint}.pos"
+            if pos_key in leader_action:
+                joint_action[pos_key] = leader_action[pos_key]
+        
+        # Send action to follower (both arms)
+        if joint_action:
+            follower.send_action(joint_action)
+        
+        # Measure loop time
+        loop_end = time.perf_counter()
+        loop_time = loop_end - loop_start
+        loop_times.append(loop_time)
+        
+        # Print stats every 2 seconds
+        if loop_end - last_print_time >= 2.0:
+            if loop_times:
+                avg_time = sum(loop_times) / len(loop_times)
+                current_hz = 1.0 / avg_time if avg_time > 0 else 0
+                min_time = min(loop_times)
+                max_time = max(loop_times)
+                max_hz = 1.0 / min_time if min_time > 0 else 0
+                min_hz = 1.0 / max_time if max_time > 0 else 0
+                
+                print(f"[Hz Stats] Avg: {current_hz:.1f} Hz | "
+                      f"Range: {min_hz:.1f}-{max_hz:.1f} Hz | "
+                      f"Avg loop time: {avg_time*1000:.1f} ms")
+                
+                # Reset for next measurement window
+                loop_times = []
+                last_print_time = loop_end
+            
+except KeyboardInterrupt:
+    print("\n\nStopping teleoperation...")
+finally:
+    # Disconnect devices
+    print("Disconnecting devices...")
+    try:
+        follower.disconnect()
+    except Exception as e:
+        print(f"Error disconnecting follower: {e}")
+    
+    try:
+        leader.disconnect()
+    except Exception as e:
+        print(f"Error disconnecting leader: {e}")
+    
+    print("Done!")

examples/phone_to_so100/evaluate.py

@@ -52,114 +52,125 @@ TASK_DESCRIPTION = "My task description"
 HF_MODEL_ID = "<hf_username>/<model_repo_id>"
 HF_DATASET_ID = "<hf_username>/<dataset_repo_id>"
 
+# Create the robot configuration & robot
+camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
+robot_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem58760434471",
+    id="my_awesome_follower_arm",
+    cameras=camera_config,
+    use_degrees=True,
+)
 
-def main():
-    # Create the robot configuration & robot
-    camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
-    robot_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem58760434471",
-        id="my_awesome_follower_arm",
-        cameras=camera_config,
-        use_degrees=True,
-    )
+robot = SO100Follower(robot_config)
 
-    robot = SO100Follower(robot_config)
+# Create policy
+policy = ACTPolicy.from_pretrained(HF_MODEL_ID)
 
-    # Create policy
-    policy = ACTPolicy.from_pretrained(HF_MODEL_ID)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(robot.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(robot.bus.motors.keys()),
-    )
-
-    # Build pipeline to convert EE action to joints action
-    robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        steps=[
-            InverseKinematicsEEToJoints(
-                kinematics=kinematics_solver,
-                motor_names=list(robot.bus.motors.keys()),
-                initial_guess_current_joints=True,
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
-
-    # Build pipeline to convert joints observation to EE observation
-    robot_joints_to_ee_pose_processor = RobotProcessorPipeline[RobotObservation, RobotObservation](
-        steps=[
-            ForwardKinematicsJointsToEE(
-                kinematics=kinematics_solver, motor_names=list(robot.bus.motors.keys())
-            )
-        ],
-        to_transition=observation_to_transition,
-        to_output=transition_to_observation,
-    )
-
-    # Create the dataset
-    dataset = LeRobotDataset.create(
-        repo_id=HF_DATASET_ID,
-        fps=FPS,
-        features=combine_feature_dicts(
-            aggregate_pipeline_dataset_features(
-                pipeline=robot_joints_to_ee_pose_processor,
-                initial_features=create_initial_features(observation=robot.observation_features),
-                use_videos=True,
-            ),
-            # User for now should be explicit on the feature keys that were used for record
-            # Alternatively, the user can pass the processor step that has the right features
-            aggregate_pipeline_dataset_features(
-                pipeline=make_default_teleop_action_processor(),
-                initial_features=create_initial_features(
-                    action={
-                        f"ee.{k}": PolicyFeature(type=FeatureType.ACTION, shape=(1,))
-                        for k in ["x", "y", "z", "wx", "wy", "wz", "gripper_pos"]
-                    }
-                ),
-                use_videos=True,
-            ),
+# Build pipeline to convert EE action to joints action
+robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        InverseKinematicsEEToJoints(
+            kinematics=kinematics_solver,
+            motor_names=list(robot.bus.motors.keys()),
+            initial_guess_current_joints=True,
         ),
-        robot_type=robot.name,
-        use_videos=True,
-        image_writer_threads=4,
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
+
+# Build pipeline to convert joints observation to EE observation
+robot_joints_to_ee_pose_processor = RobotProcessorPipeline[RobotObservation, RobotObservation](
+    steps=[
+        ForwardKinematicsJointsToEE(kinematics=kinematics_solver, motor_names=list(robot.bus.motors.keys()))
+    ],
+    to_transition=observation_to_transition,
+    to_output=transition_to_observation,
+)
+
+# Create the dataset
+dataset = LeRobotDataset.create(
+    repo_id=HF_DATASET_ID,
+    fps=FPS,
+    features=combine_feature_dicts(
+        aggregate_pipeline_dataset_features(
+            pipeline=robot_joints_to_ee_pose_processor,
+            initial_features=create_initial_features(observation=robot.observation_features),
+            use_videos=True,
+        ),
+        # User for now should be explicit on the feature keys that were used for record
+        # Alternatively, the user can pass the processor step that has the right features
+        aggregate_pipeline_dataset_features(
+            pipeline=make_default_teleop_action_processor(),
+            initial_features=create_initial_features(
+                action={
+                    f"ee.{k}": PolicyFeature(type=FeatureType.ACTION, shape=(1,))
+                    for k in ["x", "y", "z", "wx", "wy", "wz", "gripper_pos"]
+                }
+            ),
+            use_videos=True,
+        ),
+    ),
+    robot_type=robot.name,
+    use_videos=True,
+    image_writer_threads=4,
+)
+
+# Build Policy Processors
+preprocessor, postprocessor = make_pre_post_processors(
+    policy_cfg=policy,
+    pretrained_path=HF_MODEL_ID,
+    dataset_stats=dataset.meta.stats,
+    # The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
+    preprocessor_overrides={"device_processor": {"device": str(policy.config.device)}},
+)
+
+# Connect the robot
+robot.connect()
+
+# Initialize the keyboard listener and rerun visualization
+listener, events = init_keyboard_listener()
+init_rerun(session_name="phone_so100_evaluate")
+
+if not robot.is_connected:
+    raise ValueError("Robot is not connected!")
+
+print("Starting evaluate loop...")
+episode_idx = 0
+for episode_idx in range(NUM_EPISODES):
+    log_say(f"Running inference, recording eval episode {episode_idx + 1} of {NUM_EPISODES}")
+
+    # Main record loop
+    record_loop(
+        robot=robot,
+        events=events,
+        fps=FPS,
+        policy=policy,
+        preprocessor=preprocessor,  # Pass the pre and post policy processors
+        postprocessor=postprocessor,
+        dataset=dataset,
+        control_time_s=EPISODE_TIME_SEC,
+        single_task=TASK_DESCRIPTION,
+        display_data=True,
+        teleop_action_processor=make_default_teleop_action_processor(),
+        robot_action_processor=robot_ee_to_joints_processor,
+        robot_observation_processor=robot_joints_to_ee_pose_processor,
     )
 
-    # Build Policy Processors
-    preprocessor, postprocessor = make_pre_post_processors(
-        policy_cfg=policy,
-        pretrained_path=HF_MODEL_ID,
-        dataset_stats=dataset.meta.stats,
-        # The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
-        preprocessor_overrides={"device_processor": {"device": str(policy.config.device)}},
-    )
-
-    # Connect the robot
-    robot.connect()
-
-    # Initialize the keyboard listener and rerun visualization
-    listener, events = init_keyboard_listener()
-    init_rerun(session_name="phone_so100_evaluate")
-
-    if not robot.is_connected:
-        raise ValueError("Robot is not connected!")
-
-    print("Starting evaluate loop...")
-    episode_idx = 0
-    for episode_idx in range(NUM_EPISODES):
-        log_say(f"Running inference, recording eval episode {episode_idx + 1} of {NUM_EPISODES}")
-
-        # Main record loop
+    # Reset the environment if not stopping or re-recording
+    if not events["stop_recording"] and ((episode_idx < NUM_EPISODES - 1) or events["rerecord_episode"]):
+        log_say("Reset the environment")
         record_loop(
             robot=robot,
             events=events,
             fps=FPS,
-            policy=policy,
-            preprocessor=preprocessor,  # Pass the pre and post policy processors
-            postprocessor=postprocessor,
-            dataset=dataset,
             control_time_s=EPISODE_TIME_SEC,
             single_task=TASK_DESCRIPTION,
             display_data=True,
@@ -168,40 +179,21 @@ def main():
             robot_observation_processor=robot_joints_to_ee_pose_processor,
         )
 
-        # Reset the environment if not stopping or re-recording
-        if not events["stop_recording"] and ((episode_idx < NUM_EPISODES - 1) or events["rerecord_episode"]):
-            log_say("Reset the environment")
-            record_loop(
-                robot=robot,
-                events=events,
-                fps=FPS,
-                control_time_s=EPISODE_TIME_SEC,
-                single_task=TASK_DESCRIPTION,
-                display_data=True,
-                teleop_action_processor=make_default_teleop_action_processor(),
-                robot_action_processor=robot_ee_to_joints_processor,
-                robot_observation_processor=robot_joints_to_ee_pose_processor,
-            )
+    if events["rerecord_episode"]:
+        log_say("Re-record episode")
+        events["rerecord_episode"] = False
+        events["exit_early"] = False
+        dataset.clear_episode_buffer()
+        continue
 
-        if events["rerecord_episode"]:
-            log_say("Re-record episode")
-            events["rerecord_episode"] = False
-            events["exit_early"] = False
-            dataset.clear_episode_buffer()
-            continue
+    # Save episode
+    dataset.save_episode()
+    episode_idx += 1
 
-        # Save episode
-        dataset.save_episode()
-        episode_idx += 1
+# Clean up
+log_say("Stop recording")
+robot.disconnect()
+listener.stop()
 
-    # Clean up
-    log_say("Stop recording")
-    robot.disconnect()
-    listener.stop()
-
-    dataset.finalize()
-    dataset.push_to_hub()
-
-
-if __name__ == "__main__":
-    main()
+dataset.finalize()
+dataset.push_to_hub()

examples/phone_to_so100/record.py

@@ -50,122 +50,133 @@ RESET_TIME_SEC = 30
 TASK_DESCRIPTION = "My task description"
 HF_REPO_ID = "<hf_username>/<dataset_repo_id>"
 
+# Create the robot and teleoperator configurations
+camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
+robot_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411",
+    id="my_awesome_follower_arm",
+    cameras=camera_config,
+    use_degrees=True,
+)
+teleop_config = PhoneConfig(phone_os=PhoneOS.IOS)  # or PhoneOS.ANDROID
 
-def main():
-    # Create the robot and teleoperator configurations
-    camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
-    robot_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411",
-        id="my_awesome_follower_arm",
-        cameras=camera_config,
-        use_degrees=True,
-    )
-    teleop_config = PhoneConfig(phone_os=PhoneOS.IOS)  # or PhoneOS.ANDROID
+# Initialize the robot and teleoperator
+robot = SO100Follower(robot_config)
+phone = Phone(teleop_config)
 
-    # Initialize the robot and teleoperator
-    robot = SO100Follower(robot_config)
-    phone = Phone(teleop_config)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(robot.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(robot.bus.motors.keys()),
-    )
-
-    # Build pipeline to convert phone action to EE action
-    phone_to_robot_ee_pose_processor = RobotProcessorPipeline[
-        tuple[RobotAction, RobotObservation], RobotAction
-    ](
-        steps=[
-            MapPhoneActionToRobotAction(platform=teleop_config.phone_os),
-            EEReferenceAndDelta(
-                kinematics=kinematics_solver,
-                end_effector_step_sizes={"x": 0.5, "y": 0.5, "z": 0.5},
-                motor_names=list(robot.bus.motors.keys()),
-                use_latched_reference=True,
-            ),
-            EEBoundsAndSafety(
-                end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
-                max_ee_step_m=0.20,
-            ),
-            GripperVelocityToJoint(speed_factor=20.0),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
-
-    # Build pipeline to convert EE action to joints action
-    robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        steps=[
-            InverseKinematicsEEToJoints(
-                kinematics=kinematics_solver,
-                motor_names=list(robot.bus.motors.keys()),
-                initial_guess_current_joints=True,
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
-
-    # Build pipeline to convert joint observation to EE observation
-    robot_joints_to_ee_pose = RobotProcessorPipeline[RobotObservation, RobotObservation](
-        steps=[
-            ForwardKinematicsJointsToEE(
-                kinematics=kinematics_solver, motor_names=list(robot.bus.motors.keys())
-            )
-        ],
-        to_transition=observation_to_transition,
-        to_output=transition_to_observation,
-    )
-
-    # Create the dataset
-    dataset = LeRobotDataset.create(
-        repo_id=HF_REPO_ID,
-        fps=FPS,
-        features=combine_feature_dicts(
-            # Run the feature contract of the pipelines
-            # This tells you how the features would look like after the pipeline steps
-            aggregate_pipeline_dataset_features(
-                pipeline=phone_to_robot_ee_pose_processor,
-                initial_features=create_initial_features(action=phone.action_features),
-                use_videos=True,
-            ),
-            aggregate_pipeline_dataset_features(
-                pipeline=robot_joints_to_ee_pose,
-                initial_features=create_initial_features(observation=robot.observation_features),
-                use_videos=True,
-            ),
+# Build pipeline to convert phone action to EE action
+phone_to_robot_ee_pose_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        MapPhoneActionToRobotAction(platform=teleop_config.phone_os),
+        EEReferenceAndDelta(
+            kinematics=kinematics_solver,
+            end_effector_step_sizes={"x": 0.5, "y": 0.5, "z": 0.5},
+            motor_names=list(robot.bus.motors.keys()),
+            use_latched_reference=True,
         ),
-        robot_type=robot.name,
-        use_videos=True,
-        image_writer_threads=4,
+        EEBoundsAndSafety(
+            end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
+            max_ee_step_m=0.20,
+        ),
+        GripperVelocityToJoint(speed_factor=20.0),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
+
+# Build pipeline to convert EE action to joints action
+robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        InverseKinematicsEEToJoints(
+            kinematics=kinematics_solver,
+            motor_names=list(robot.bus.motors.keys()),
+            initial_guess_current_joints=True,
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
+
+# Build pipeline to convert joint observation to EE observation
+robot_joints_to_ee_pose = RobotProcessorPipeline[RobotObservation, RobotObservation](
+    steps=[
+        ForwardKinematicsJointsToEE(kinematics=kinematics_solver, motor_names=list(robot.bus.motors.keys()))
+    ],
+    to_transition=observation_to_transition,
+    to_output=transition_to_observation,
+)
+
+# Create the dataset
+dataset = LeRobotDataset.create(
+    repo_id=HF_REPO_ID,
+    fps=FPS,
+    features=combine_feature_dicts(
+        # Run the feature contract of the pipelines
+        # This tells you how the features would look like after the pipeline steps
+        aggregate_pipeline_dataset_features(
+            pipeline=phone_to_robot_ee_pose_processor,
+            initial_features=create_initial_features(action=phone.action_features),
+            use_videos=True,
+        ),
+        aggregate_pipeline_dataset_features(
+            pipeline=robot_joints_to_ee_pose,
+            initial_features=create_initial_features(observation=robot.observation_features),
+            use_videos=True,
+        ),
+    ),
+    robot_type=robot.name,
+    use_videos=True,
+    image_writer_threads=4,
+)
+
+# Connect the robot and teleoperator
+robot.connect()
+phone.connect()
+
+# Initialize the keyboard listener and rerun visualization
+listener, events = init_keyboard_listener()
+init_rerun(session_name="phone_so100_record")
+
+if not robot.is_connected or not phone.is_connected:
+    raise ValueError("Robot or teleop is not connected!")
+
+
+print("Starting record loop. Move your phone to teleoperate the robot...")
+episode_idx = 0
+while episode_idx < NUM_EPISODES and not events["stop_recording"]:
+    log_say(f"Recording episode {episode_idx + 1} of {NUM_EPISODES}")
+
+    # Main record loop
+    record_loop(
+        robot=robot,
+        events=events,
+        fps=FPS,
+        teleop=phone,
+        dataset=dataset,
+        control_time_s=EPISODE_TIME_SEC,
+        single_task=TASK_DESCRIPTION,
+        display_data=True,
+        teleop_action_processor=phone_to_robot_ee_pose_processor,
+        robot_action_processor=robot_ee_to_joints_processor,
+        robot_observation_processor=robot_joints_to_ee_pose,
     )
 
-    # Connect the robot and teleoperator
-    robot.connect()
-    phone.connect()
-
-    # Initialize the keyboard listener and rerun visualization
-    listener, events = init_keyboard_listener()
-    init_rerun(session_name="phone_so100_record")
-
-    if not robot.is_connected or not phone.is_connected:
-        raise ValueError("Robot or teleop is not connected!")
-
-    print("Starting record loop. Move your phone to teleoperate the robot...")
-    episode_idx = 0
-    while episode_idx < NUM_EPISODES and not events["stop_recording"]:
-        log_say(f"Recording episode {episode_idx + 1} of {NUM_EPISODES}")
-
-        # Main record loop
+    # Reset the environment if not stopping or re-recording
+    if not events["stop_recording"] and (episode_idx < NUM_EPISODES - 1 or events["rerecord_episode"]):
+        log_say("Reset the environment")
         record_loop(
             robot=robot,
             events=events,
             fps=FPS,
             teleop=phone,
-            dataset=dataset,
-            control_time_s=EPISODE_TIME_SEC,
+            control_time_s=RESET_TIME_SEC,
             single_task=TASK_DESCRIPTION,
             display_data=True,
             teleop_action_processor=phone_to_robot_ee_pose_processor,
@@ -173,42 +184,22 @@ def main():
             robot_observation_processor=robot_joints_to_ee_pose,
         )
 
-        # Reset the environment if not stopping or re-recording
-        if not events["stop_recording"] and (episode_idx < NUM_EPISODES - 1 or events["rerecord_episode"]):
-            log_say("Reset the environment")
-            record_loop(
-                robot=robot,
-                events=events,
-                fps=FPS,
-                teleop=phone,
-                control_time_s=RESET_TIME_SEC,
-                single_task=TASK_DESCRIPTION,
-                display_data=True,
-                teleop_action_processor=phone_to_robot_ee_pose_processor,
-                robot_action_processor=robot_ee_to_joints_processor,
-                robot_observation_processor=robot_joints_to_ee_pose,
-            )
+    if events["rerecord_episode"]:
+        log_say("Re-recording episode")
+        events["rerecord_episode"] = False
+        events["exit_early"] = False
+        dataset.clear_episode_buffer()
+        continue
 
-        if events["rerecord_episode"]:
-            log_say("Re-recording episode")
-            events["rerecord_episode"] = False
-            events["exit_early"] = False
-            dataset.clear_episode_buffer()
-            continue
+    # Save episode
+    dataset.save_episode()
+    episode_idx += 1
 
-        # Save episode
-        dataset.save_episode()
-        episode_idx += 1
+# Clean up
+log_say("Stop recording")
+robot.disconnect()
+phone.disconnect()
+listener.stop()
 
-    # Clean up
-    log_say("Stop recording")
-    robot.disconnect()
-    phone.disconnect()
-    listener.stop()
-
-    dataset.finalize()
-    dataset.push_to_hub()
-
-
-if __name__ == "__main__":
-    main()
+dataset.finalize()
+dataset.push_to_hub()

examples/phone_to_so100/replay.py

@@ -29,78 +29,72 @@ from lerobot.robots.so100_follower.robot_kinematic_processor import (
 )
 from lerobot.robots.so100_follower.so100_follower import SO100Follower
 from lerobot.utils.constants import ACTION
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.utils import log_say
 
 EPISODE_IDX = 0
 HF_REPO_ID = "<hf_username>/<dataset_repo_id>"
 
+# Initialize the robot config
+robot_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
+)
 
-def main():
-    # Initialize the robot config
-    robot_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
-    )
+# Initialize the robot
+robot = SO100Follower(robot_config)
 
-    # Initialize the robot
-    robot = SO100Follower(robot_config)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(robot.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(robot.bus.motors.keys()),
-    )
+# Build pipeline to convert EE action to joints action
+robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        InverseKinematicsEEToJoints(
+            kinematics=kinematics_solver,
+            motor_names=list(robot.bus.motors.keys()),
+            initial_guess_current_joints=False,  # Because replay is open loop
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
 
-    # Build pipeline to convert EE action to joints action
-    robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        steps=[
-            InverseKinematicsEEToJoints(
-                kinematics=kinematics_solver,
-                motor_names=list(robot.bus.motors.keys()),
-                initial_guess_current_joints=False,  # Because replay is open loop
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
+# Fetch the dataset to replay
+dataset = LeRobotDataset(HF_REPO_ID, episodes=[EPISODE_IDX])
+# Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
+episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == EPISODE_IDX)
+actions = episode_frames.select_columns(ACTION)
 
-    # Fetch the dataset to replay
-    dataset = LeRobotDataset(HF_REPO_ID, episodes=[EPISODE_IDX])
-    # Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
-    episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == EPISODE_IDX)
-    actions = episode_frames.select_columns(ACTION)
+# Connect to the robot
+robot.connect()
 
-    # Connect to the robot
-    robot.connect()
+if not robot.is_connected:
+    raise ValueError("Robot is not connected!")
 
-    if not robot.is_connected:
-        raise ValueError("Robot is not connected!")
+print("Starting replay loop...")
+log_say(f"Replaying episode {EPISODE_IDX}")
+for idx in range(len(episode_frames)):
+    t0 = time.perf_counter()
 
-    print("Starting replay loop...")
-    log_say(f"Replaying episode {EPISODE_IDX}")
-    for idx in range(len(episode_frames)):
-        t0 = time.perf_counter()
+    # Get recorded action from dataset
+    ee_action = {
+        name: float(actions[idx][ACTION][i]) for i, name in enumerate(dataset.features[ACTION]["names"])
+    }
 
-        # Get recorded action from dataset
-        ee_action = {
-            name: float(actions[idx][ACTION][i]) for i, name in enumerate(dataset.features[ACTION]["names"])
-        }
+    # Get robot observation
+    robot_obs = robot.get_observation()
 
-        # Get robot observation
-        robot_obs = robot.get_observation()
+    # Dataset EE -> robot joints
+    joint_action = robot_ee_to_joints_processor((ee_action, robot_obs))
 
-        # Dataset EE -> robot joints
-        joint_action = robot_ee_to_joints_processor((ee_action, robot_obs))
+    # Send action to robot
+    _ = robot.send_action(joint_action)
 
-        # Send action to robot
-        _ = robot.send_action(joint_action)
+    busy_wait(1.0 / dataset.fps - (time.perf_counter() - t0))
 
-        precise_sleep(1.0 / dataset.fps - (time.perf_counter() - t0))
-
-    # Clean up
-    robot.disconnect()
-
-
-if __name__ == "__main__":
-    main()
+# Clean up
+robot.disconnect()

examples/phone_to_so100/teleoperate.py

@@ -32,90 +32,82 @@ from lerobot.robots.so100_follower.so100_follower import SO100Follower
 from lerobot.teleoperators.phone.config_phone import PhoneConfig, PhoneOS
 from lerobot.teleoperators.phone.phone_processor import MapPhoneActionToRobotAction
 from lerobot.teleoperators.phone.teleop_phone import Phone
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.visualization_utils import init_rerun, log_rerun_data
 
 FPS = 30
 
+# Initialize the robot and teleoperator
+robot_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
+)
+teleop_config = PhoneConfig(phone_os=PhoneOS.IOS)  # or PhoneOS.ANDROID
 
-def main():
-    # Initialize the robot and teleoperator
-    robot_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
-    )
-    teleop_config = PhoneConfig(phone_os=PhoneOS.IOS)  # or PhoneOS.ANDROID
+# Initialize the robot and teleoperator
+robot = SO100Follower(robot_config)
+teleop_device = Phone(teleop_config)
 
-    # Initialize the robot and teleoperator
-    robot = SO100Follower(robot_config)
-    teleop_device = Phone(teleop_config)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(robot.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(robot.bus.motors.keys()),
-    )
+# Build pipeline to convert phone action to ee pose action to joint action
+phone_to_robot_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        MapPhoneActionToRobotAction(platform=teleop_config.phone_os),
+        EEReferenceAndDelta(
+            kinematics=kinematics_solver,
+            end_effector_step_sizes={"x": 0.5, "y": 0.5, "z": 0.5},
+            motor_names=list(robot.bus.motors.keys()),
+            use_latched_reference=True,
+        ),
+        EEBoundsAndSafety(
+            end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
+            max_ee_step_m=0.10,
+        ),
+        GripperVelocityToJoint(
+            speed_factor=20.0,
+        ),
+        InverseKinematicsEEToJoints(
+            kinematics=kinematics_solver,
+            motor_names=list(robot.bus.motors.keys()),
+            initial_guess_current_joints=True,
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
 
-    # Build pipeline to convert phone action to ee pose action to joint action
-    phone_to_robot_joints_processor = RobotProcessorPipeline[
-        tuple[RobotAction, RobotObservation], RobotAction
-    ](
-        steps=[
-            MapPhoneActionToRobotAction(platform=teleop_config.phone_os),
-            EEReferenceAndDelta(
-                kinematics=kinematics_solver,
-                end_effector_step_sizes={"x": 0.5, "y": 0.5, "z": 0.5},
-                motor_names=list(robot.bus.motors.keys()),
-                use_latched_reference=True,
-            ),
-            EEBoundsAndSafety(
-                end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
-                max_ee_step_m=0.10,
-            ),
-            GripperVelocityToJoint(
-                speed_factor=20.0,
-            ),
-            InverseKinematicsEEToJoints(
-                kinematics=kinematics_solver,
-                motor_names=list(robot.bus.motors.keys()),
-                initial_guess_current_joints=True,
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
+# Connect to the robot and teleoperator
+robot.connect()
+teleop_device.connect()
 
-    # Connect to the robot and teleoperator
-    robot.connect()
-    teleop_device.connect()
+# Init rerun viewer
+init_rerun(session_name="phone_so100_teleop")
 
-    # Init rerun viewer
-    init_rerun(session_name="phone_so100_teleop")
+if not robot.is_connected or not teleop_device.is_connected:
+    raise ValueError("Robot or teleop is not connected!")
 
-    if not robot.is_connected or not teleop_device.is_connected:
-        raise ValueError("Robot or teleop is not connected!")
+print("Starting teleop loop. Move your phone to teleoperate the robot...")
+while True:
+    t0 = time.perf_counter()
 
-    print("Starting teleop loop. Move your phone to teleoperate the robot...")
-    while True:
-        t0 = time.perf_counter()
+    # Get robot observation
+    robot_obs = robot.get_observation()
 
-        # Get robot observation
-        robot_obs = robot.get_observation()
+    # Get teleop action
+    phone_obs = teleop_device.get_action()
 
-        # Get teleop action
-        phone_obs = teleop_device.get_action()
+    # Phone -> EE pose -> Joints transition
+    joint_action = phone_to_robot_joints_processor((phone_obs, robot_obs))
 
-        # Phone -> EE pose -> Joints transition
-        joint_action = phone_to_robot_joints_processor((phone_obs, robot_obs))
+    # Send action to robot
+    _ = robot.send_action(joint_action)
 
-        # Send action to robot
-        _ = robot.send_action(joint_action)
+    # Visualize
+    log_rerun_data(observation=phone_obs, action=joint_action)
 
-        # Visualize
-        log_rerun_data(observation=phone_obs, action=joint_action)
-
-        precise_sleep(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))
-
-
-if __name__ == "__main__":
-    main()
+    busy_wait(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))

examples/port_datasets/slurm_aggregate_shards.py

@@ -15,12 +15,16 @@
 # limitations under the License.
 
 import argparse
+import logging
 from pathlib import Path
 
 from datatrove.executor import LocalPipelineExecutor
 from datatrove.executor.slurm import SlurmPipelineExecutor
 from datatrove.pipeline.base import PipelineStep
-from port_droid import DROID_SHARDS
+from port_datasets.droid_rlds.port_droid import DROID_SHARDS
+
+from lerobot.datasets.aggregate import aggregate_datasets
+from lerobot.utils.utils import init_logging
 
 
 class AggregateDatasets(PipelineStep):
@@ -34,11 +38,6 @@ class AggregateDatasets(PipelineStep):
         self.aggr_repo_id = aggregated_repo_id
 
     def run(self, data=None, rank: int = 0, world_size: int = 1):
-        import logging
-
-        from lerobot.datasets.aggregate import aggregate_datasets
-        from lerobot.utils.utils import init_logging
-
         init_logging()
 
         # Since aggregate_datasets already handles parallel processing internally,

examples/port_datasets/slurm_port_shards.py

@@ -20,7 +20,7 @@ from pathlib import Path
 from datatrove.executor import LocalPipelineExecutor
 from datatrove.executor.slurm import SlurmPipelineExecutor
 from datatrove.pipeline.base import PipelineStep
-from port_droid import DROID_SHARDS
+from port_datasets.droid_rlds.port_droid import DROID_SHARDS
 
 
 class PortDroidShards(PipelineStep):
@@ -35,7 +35,7 @@ class PortDroidShards(PipelineStep):
 
     def run(self, data=None, rank: int = 0, world_size: int = 1):
         from datasets.utils.tqdm import disable_progress_bars
-        from port_droid import port_droid, validate_dataset
+        from port_datasets.droid_rlds.port_droid import port_droid, validate_dataset
 
         from lerobot.utils.utils import init_logging
 

examples/port_datasets/slurm_upload.py

@@ -24,7 +24,7 @@ from datatrove.executor.slurm import SlurmPipelineExecutor
 from datatrove.pipeline.base import PipelineStep
 from huggingface_hub import HfApi
 from huggingface_hub.constants import REPOCARD_NAME
-from port_droid import DROID_SHARDS
+from port_datasets.droid_rlds.port_droid import DROID_SHARDS
 
 from lerobot.datasets.lerobot_dataset import CODEBASE_VERSION, LeRobotDatasetMetadata
 from lerobot.datasets.utils import create_lerobot_dataset_card
@@ -185,11 +185,11 @@ class UploadDataset(PipelineStep):
 
 
 def make_upload_executor(
-    repo_id, job_name, logs_dir, workers, partition, cpus_per_task, mem_per_cpu, private=False, slurm=True
+    repo_id, job_name, logs_dir, workers, partition, cpus_per_task, mem_per_cpu, slurm=True
 ):
     kwargs = {
         "pipeline": [
-            UploadDataset(repo_id, private=private),
+            UploadDataset(repo_id),
         ],
         "logging_dir": str(logs_dir / job_name),
     }
@@ -267,12 +267,6 @@ def main():
         default="1950M",
         help="Memory per cpu that each worker will use.",
     )
-    parser.add_argument(
-        "--private",
-        action="store_true",
-        default=False,
-        help="Whether to create a private repository.",
-    )
 
     init_logging()
 

examples/rtc/eval_dataset.py

@@ -1,951 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Evaluate Real-Time Chunking (RTC) performance on dataset samples.
-
-This script takes two random samples from a dataset:
-- Uses actions from the first sample as previous chunk
-- Generates new actions for the second sample with and without RTC
-
-It compares action predictions with and without RTC on dataset samples,
-measuring consistency and ground truth alignment.
-
-Usage:
-    # Basic usage with smolvla policy
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=helper2424/smolvla_check_rtc_last3 \
-        --dataset.repo_id=helper2424/check_rtc \
-        --rtc.execution_horizon=8 \
-        --device=mps \
-        --rtc.max_guidance_weight=10.0 \
-        --rtc.prefix_attention_schedule=EXP \
-        --seed=10
-
-    # Basic usage with pi0.5 policy
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=lerobot/pi05_libero_finetuned \
-        --dataset.repo_id=HuggingFaceVLA/libero \
-        --rtc.execution_horizon=10 \
-        --device=mps
-        --seed=10
-
-    # Basic usage with pi0.5 policy with cuda device
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=lerobot/pi05_libero_finetuned \
-        --dataset.repo_id=HuggingFaceVLA/libero \
-        --rtc.execution_horizon=8 \
-        --device=cuda
-
-    # Basic usage with pi0 policy with cuda device
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=lerobot/pi0_libero_finetuned \
-        --dataset.repo_id=HuggingFaceVLA/libero \
-        --rtc.execution_horizon=8 \
-        --device=cuda
-
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=lipsop/reuben_pi0 \
-        --dataset.repo_id=ReubenLim/so101_cube_in_cup \
-        --rtc.execution_horizon=8 \
-        --device=cuda
-
-    # With torch.compile for faster inference (PyTorch 2.0+)
-    # Note: CUDA graphs disabled by default due to in-place ops in denoising loop
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=helper2424/smolvla_check_rtc_last3 \
-        --dataset.repo_id=helper2424/check_rtc \
-        --rtc.execution_horizon=8 \
-        --device=mps \
-        --use_torch_compile=true \
-        --torch_compile_mode=max-autotune
-
-    # With torch.compile on CUDA (CUDA graphs disabled by default)
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=helper2424/smolvla_check_rtc_last3 \
-        --dataset.repo_id=helper2424/check_rtc \
-        --rtc.execution_horizon=8 \
-        --device=cuda \
-        --use_torch_compile=true \
-        --torch_compile_mode=reduce-overhead
-
-    # Enable CUDA graphs (advanced - may cause tensor aliasing errors)
-    uv run python examples/rtc/eval_dataset.py \
-        --policy.path=helper2424/smolvla_check_rtc_last3 \
-        --dataset.repo_id=helper2424/check_rtc \
-        --use_torch_compile=true \
-        --torch_compile_backend=inductor \
-        --torch_compile_mode=max-autotune \
-        --torch_compile_disable_cudagraphs=false
-"""
-
-import gc
-import logging
-import os
-import random
-from dataclasses import dataclass, field
-
-import numpy as np
-import torch
-
-try:
-    import matplotlib.pyplot as plt
-
-    MATPLOTLIB_AVAILABLE = True
-except ImportError:
-    MATPLOTLIB_AVAILABLE = False
-    plt = None
-
-from lerobot.configs import parser
-from lerobot.configs.default import DatasetConfig
-from lerobot.configs.policies import PreTrainedConfig
-from lerobot.configs.types import RTCAttentionSchedule
-from lerobot.datasets.factory import resolve_delta_timestamps
-from lerobot.datasets.lerobot_dataset import LeRobotDataset, LeRobotDatasetMetadata
-from lerobot.policies.factory import get_policy_class, make_pre_post_processors
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
-from lerobot.policies.rtc.debug_visualizer import RTCDebugVisualizer
-from lerobot.utils.hub import HubMixin
-from lerobot.utils.utils import init_logging
-
-
-def set_seed(seed: int):
-    """Set random seed for reproducibility."""
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed(seed)
-        torch.cuda.manual_seed_all(seed)
-    if torch.backends.mps.is_available():
-        torch.mps.manual_seed(seed)
-    torch.backends.cudnn.deterministic = True
-    torch.backends.cudnn.benchmark = False
-
-
-def _check_matplotlib_available():
-    """Check if matplotlib is available, raise helpful error if not."""
-    if not MATPLOTLIB_AVAILABLE:
-        raise ImportError(
-            "matplotlib is required for RTC debug visualizations. "
-            "Please install it by running:\n"
-            "  uv pip install matplotlib"
-        )
-
-
-@dataclass
-class RTCEvalConfig(HubMixin):
-    """Configuration for RTC evaluation."""
-
-    # Policy configuration
-    policy: PreTrainedConfig | None = None
-
-    # Dataset configuration
-    dataset: DatasetConfig = field(default_factory=DatasetConfig)
-
-    # RTC configuration
-    rtc: RTCConfig = field(
-        default_factory=lambda: RTCConfig(
-            enabled=True,
-            execution_horizon=20,
-            max_guidance_weight=10.0,
-            prefix_attention_schedule=RTCAttentionSchedule.EXP,
-            debug=True,
-            debug_maxlen=1000,
-        )
-    )
-
-    # Device configuration
-    device: str | None = field(
-        default=None,
-        metadata={"help": "Device to run on (cuda, cpu, mps, auto)"},
-    )
-
-    # Output configuration
-    output_dir: str = field(
-        default="rtc_debug_output",
-        metadata={"help": "Directory to save debug visualizations"},
-    )
-
-    # Seed configuration
-    seed: int = field(
-        default=42,
-        metadata={"help": "Random seed for reproducibility"},
-    )
-
-    inference_delay: int = field(
-        default=4,
-        metadata={"help": "Inference delay for RTC"},
-    )
-
-    # Torch compile configuration
-    use_torch_compile: bool = field(
-        default=False,
-        metadata={"help": "Use torch.compile for faster inference (PyTorch 2.0+)"},
-    )
-
-    torch_compile_backend: str = field(
-        default="inductor",
-        metadata={"help": "Backend for torch.compile (inductor, aot_eager, cudagraphs)"},
-    )
-
-    torch_compile_mode: str = field(
-        default="default",
-        metadata={"help": "Compilation mode (default, reduce-overhead, max-autotune)"},
-    )
-
-    torch_compile_disable_cudagraphs: bool = field(
-        default=True,
-        metadata={
-            "help": "Disable CUDA graphs in torch.compile. Required due to in-place tensor "
-            "operations in denoising loop (x_t += dt * v_t) which cause tensor aliasing issues."
-        },
-    )
-
-    def __post_init__(self):
-        # Parse policy path
-        policy_path = parser.get_path_arg("policy")
-        if policy_path:
-            cli_overrides = parser.get_cli_overrides("policy")
-            self.policy = PreTrainedConfig.from_pretrained(policy_path, cli_overrides=cli_overrides)
-            self.policy.pretrained_path = policy_path
-        else:
-            raise ValueError("Policy path is required (--policy.path)")
-
-        # Auto-detect device if not specified
-        if self.device is None or self.device == "auto":
-            if torch.cuda.is_available():
-                self.device = "cuda"
-            elif torch.backends.mps.is_available():
-                self.device = "mps"
-            else:
-                self.device = "cpu"
-            logging.info(f"Auto-detected device: {self.device}")
-
-    @classmethod
-    def __get_path_fields__(cls) -> list[str]:
-        """This enables the parser to load config from the policy using `--policy.path=local/dir`"""
-        return ["policy"]
-
-
-class RTCEvaluator:
-    """Evaluator for RTC on dataset samples."""
-
-    def __init__(self, cfg: RTCEvalConfig):
-        self.cfg = cfg
-        self.device = cfg.device
-
-        # Load dataset with proper delta_timestamps based on policy configuration
-        # Calculate delta_timestamps using the same logic as make_dataset factory
-        logging.info(f"Loading dataset: {cfg.dataset.repo_id}")
-
-        # Get dataset metadata to extract FPS
-        ds_meta = LeRobotDatasetMetadata(cfg.dataset.repo_id)
-
-        # Calculate delta_timestamps from policy's delta_indices
-        delta_timestamps = resolve_delta_timestamps(cfg.policy, ds_meta)
-
-        # Create dataset with calculated delta_timestamps
-        self.dataset = LeRobotDataset(
-            cfg.dataset.repo_id,
-            delta_timestamps=delta_timestamps,
-        )
-        logging.info(f"Dataset loaded: {len(self.dataset)} samples, {self.dataset.num_episodes} episodes")
-
-        # Create preprocessor/postprocessor
-        self.preprocessor, self.postprocessor = make_pre_post_processors(
-            policy_cfg=cfg.policy,
-            pretrained_path=cfg.policy.pretrained_path,
-            preprocessor_overrides={
-                "device_processor": {"device": self.device},
-            },
-        )
-
-        logging.info("=" * 80)
-        logging.info("Ready to run evaluation with sequential policy loading:")
-        logging.info("  1. policy_prev_chunk - Generate reference chunk, then destroy")
-        logging.info("  2. policy_no_rtc - Generate without RTC, then destroy")
-        logging.info("  3. policy_rtc - Generate with RTC, then destroy")
-        logging.info("  Note: Only one policy in memory at a time for efficient memory usage")
-        logging.info("=" * 80)
-
-    def _init_policy(self, name: str, rtc_enabled: bool, rtc_debug: bool):
-        """Initialize a single policy instance with specified RTC configuration.
-
-        Args:
-            name: Name identifier for logging purposes
-            rtc_enabled: Whether to enable RTC for this policy
-            rtc_debug: Whether to enable debug tracking for this policy
-
-        Returns:
-            Configured policy instance with optional torch.compile applied
-        """
-        logging.info(f"Initializing {name}...")
-
-        # Load policy from pretrained
-        policy_class = get_policy_class(self.cfg.policy.type)
-
-        config = PreTrainedConfig.from_pretrained(self.cfg.policy.pretrained_path)
-
-        if self.cfg.policy.type == "pi05" or self.cfg.policy.type == "pi0":
-            config.compile_model = self.cfg.use_torch_compile
-
-        policy = policy_class.from_pretrained(self.cfg.policy.pretrained_path, config=config)
-        policy = policy.to(self.device)
-        policy.eval()
-
-        # Configure RTC
-        rtc_config = RTCConfig(
-            enabled=rtc_enabled,
-            execution_horizon=self.cfg.rtc.execution_horizon,
-            max_guidance_weight=self.cfg.rtc.max_guidance_weight,
-            prefix_attention_schedule=self.cfg.rtc.prefix_attention_schedule,
-            debug=rtc_debug,
-            debug_maxlen=self.cfg.rtc.debug_maxlen,
-        )
-        policy.config.rtc_config = rtc_config
-        policy.init_rtc_processor()
-
-        logging.info(f"  RTC enabled: {rtc_enabled}")
-        logging.info(f"  RTC debug: {rtc_debug}")
-        logging.info(f"  Policy config: {config}")
-
-        # Apply torch.compile to predict_action_chunk method if enabled
-        if self.cfg.use_torch_compile:
-            policy = self._apply_torch_compile(policy, name)
-
-        logging.info(f"✓ {name} initialized successfully")
-        return policy
-
-    def _apply_torch_compile(self, policy, policy_name: str):
-        """Apply torch.compile to the policy's predict_action_chunk method.
-
-        Args:
-            policy: Policy instance to compile
-            policy_name: Name for logging purposes
-
-        Returns:
-            Policy with compiled predict_action_chunk method
-        """
-
-        # PI models handle their own compilation
-        if policy.type == "pi05" or policy.type == "pi0":
-            return policy
-
-        try:
-            # Check if torch.compile is available (PyTorch 2.0+)
-            if not hasattr(torch, "compile"):
-                logging.warning(
-                    f"  [{policy_name}] torch.compile is not available. Requires PyTorch 2.0+. "
-                    f"Current version: {torch.__version__}. Skipping compilation."
-                )
-                return policy
-
-            logging.info(f"  [{policy_name}] Applying torch.compile to predict_action_chunk...")
-            logging.info(f"    Backend: {self.cfg.torch_compile_backend}")
-            logging.info(f"    Mode: {self.cfg.torch_compile_mode}")
-            logging.info(f"    Disable CUDA graphs: {self.cfg.torch_compile_disable_cudagraphs}")
-            logging.info("    Note: Debug tracker excluded from compilation via @torch._dynamo.disable")
-
-            # Compile the predict_action_chunk method
-            # - Debug tracker is excluded from compilation via @torch._dynamo.disable
-            # - CUDA graphs disabled to prevent tensor aliasing from in-place ops (x_t += dt * v_t)
-            compile_kwargs = {
-                "backend": self.cfg.torch_compile_backend,
-                "mode": self.cfg.torch_compile_mode,
-            }
-
-            # Disable CUDA graphs if requested (prevents tensor aliasing issues)
-            if self.cfg.torch_compile_disable_cudagraphs:
-                compile_kwargs["options"] = {"triton.cudagraphs": False}
-
-            original_method = policy.predict_action_chunk
-            compiled_method = torch.compile(original_method, **compile_kwargs)
-            policy.predict_action_chunk = compiled_method
-            logging.info(f"  ✓ [{policy_name}] Successfully compiled predict_action_chunk")
-
-        except Exception as e:
-            logging.error(f"  [{policy_name}] Failed to apply torch.compile: {e}")
-            logging.warning(f"  [{policy_name}] Continuing without torch.compile")
-
-        return policy
-
-    def _destroy_policy(self, policy, policy_name: str):
-        """Explicitly destroy a policy and free all associated memory.
-
-        This method performs aggressive cleanup to ensure maximum memory is freed,
-        which is critical for large models (e.g., VLAs with billions of parameters).
-
-        Args:
-            policy: Policy instance to destroy
-            policy_name: Name for logging purposes
-        """
-        logging.info(f"  Destroying {policy_name} and freeing memory...")
-
-        try:
-            # Step 1: Move policy to CPU to free GPU/MPS memory
-            policy.cpu()
-
-            # Step 2: Delete the policy object
-            del policy
-
-            # Step 3: Force garbage collection to reclaim memory immediately
-            gc.collect()
-
-            # Step 4: Clear device-specific caches
-            if torch.cuda.is_available():
-                torch.cuda.empty_cache()
-                torch.cuda.synchronize()  # Ensure all operations complete
-
-            if torch.backends.mps.is_available():
-                torch.mps.empty_cache()
-
-            logging.info(f"  ✓ {policy_name} destroyed and memory freed")
-
-        except Exception as e:
-            logging.warning(f"  Warning: Error during {policy_name} cleanup: {e}")
-
-    def run_evaluation(self):
-        """Run evaluation on two random dataset samples using three separate policies.
-
-        Note: Policies are deinitalized after each step to free memory. Large models
-        (e.g., VLA models with billions of parameters) cannot fit three instances in
-        memory simultaneously. By deleting and garbage collecting after each step,
-        we ensure only one policy is loaded at a time.
-        """
-        # Create output directory
-        os.makedirs(self.cfg.output_dir, exist_ok=True)
-        logging.info(f"Output directory: {self.cfg.output_dir}")
-
-        logging.info("=" * 80)
-        logging.info("Starting RTC evaluation")
-        logging.info(f"Inference delay: {self.cfg.inference_delay}")
-        logging.info("=" * 80)
-
-        # Load two random samples from dataset
-        data_loader = torch.utils.data.DataLoader(self.dataset, batch_size=1, shuffle=True)
-        loader_iter = iter(data_loader)
-        first_sample = next(loader_iter)
-        second_sample = next(loader_iter)
-
-        preprocessed_first_sample = self.preprocessor(first_sample)
-        preprocessed_second_sample = self.preprocessor(second_sample)
-
-        # ============================================================================
-        # Step 1: Generate previous chunk using policy_prev_chunk
-        # ============================================================================
-        # This policy is only used to generate the reference chunk and then freed
-        logging.info("=" * 80)
-        logging.info("Step 1: Generating previous chunk with policy_prev_chunk")
-        logging.info("=" * 80)
-
-        # Initialize policy 1
-        policy_prev_chunk_policy = self._init_policy(
-            name="policy_prev_chunk",
-            rtc_enabled=False,
-            rtc_debug=False,
-        )
-        with torch.no_grad():
-            prev_chunk_left_over = policy_prev_chunk_policy.predict_action_chunk(
-                preprocessed_first_sample,
-            )[:, :25, :].squeeze(0)
-        logging.info(f"  Generated prev_chunk shape: {prev_chunk_left_over.shape}")
-
-        # Destroy policy_prev_chunk to free memory for large models
-        self._destroy_policy(policy_prev_chunk_policy, "policy_prev_chunk")
-
-        # ============================================================================
-        # Step 2: Generate actions WITHOUT RTC using policy_no_rtc
-        # ============================================================================
-        logging.info("=" * 80)
-        logging.info("Step 2: Generating actions WITHOUT RTC with policy_no_rtc")
-        logging.info("=" * 80)
-
-        set_seed(self.cfg.seed)
-
-        # Initialize policy 2
-        policy_no_rtc_policy = self._init_policy(
-            name="policy_no_rtc",
-            rtc_enabled=False,
-            rtc_debug=True,
-        )
-
-        # Sample noise (use same noise for both RTC and non-RTC for fair comparison)
-        noise_size = (1, policy_no_rtc_policy.config.chunk_size, policy_no_rtc_policy.config.max_action_dim)
-        noise = policy_no_rtc_policy.model.sample_noise(noise_size, self.device)
-        noise_clone = noise.clone()
-        policy_no_rtc_policy.rtc_processor.reset_tracker()
-        with torch.no_grad():
-            no_rtc_actions = policy_no_rtc_policy.predict_action_chunk(
-                preprocessed_second_sample,
-                noise=noise,
-            )
-        no_rtc_tracked_steps = policy_no_rtc_policy.rtc_processor.tracker.get_all_steps()
-        logging.info(f"  Tracked {len(no_rtc_tracked_steps)} steps without RTC")
-        logging.info(f"  Generated no_rtc_actions shape: {no_rtc_actions.shape}")
-
-        # Destroy policy_no_rtc to free memory before loading policy_rtc
-        self._destroy_policy(policy_no_rtc_policy, "policy_no_rtc")
-
-        # ============================================================================
-        # Step 3: Generate actions WITH RTC using policy_rtc
-        # ============================================================================
-        logging.info("=" * 80)
-        logging.info("Step 3: Generating actions WITH RTC with policy_rtc")
-        logging.info("=" * 80)
-
-        set_seed(self.cfg.seed)
-
-        # Initialize policy 3
-        policy_rtc_policy = self._init_policy(
-            name="policy_rtc",
-            rtc_enabled=True,
-            rtc_debug=True,
-        )
-        policy_rtc_policy.rtc_processor.reset_tracker()
-        with torch.no_grad():
-            rtc_actions = policy_rtc_policy.predict_action_chunk(
-                preprocessed_second_sample,
-                noise=noise_clone,
-                inference_delay=self.cfg.inference_delay,
-                prev_chunk_left_over=prev_chunk_left_over,
-                execution_horizon=self.cfg.rtc.execution_horizon,
-            )
-        rtc_tracked_steps = policy_rtc_policy.rtc_processor.get_all_debug_steps()
-        logging.info(f"  Tracked {len(rtc_tracked_steps)} steps with RTC")
-        logging.info(f"  Generated rtc_actions shape: {rtc_actions.shape}")
-
-        # Save num_steps before destroying policy (needed for plotting)
-        try:
-            num_steps = policy_rtc_policy.config.num_steps
-        except Exception as e:
-            logging.error(f"  Error getting num_steps: {e}")
-            num_steps = policy_rtc_policy.config.num_inference_steps
-            logging.warning(f"  Using num_inference_steps: {num_steps} instead of num_steps")
-
-        # Destroy policy_rtc after final use
-        self._destroy_policy(policy_rtc_policy, "policy_rtc")
-
-        # Plot and save results
-        logging.info("=" * 80)
-        logging.info("Plotting results...")
-        self.plot_tracked_data(rtc_tracked_steps, no_rtc_tracked_steps, prev_chunk_left_over, num_steps)
-
-        # Plot final actions comparison
-        logging.info("=" * 80)
-        logging.info("Plotting final actions comparison...")
-        self.plot_final_actions_comparison(rtc_actions, no_rtc_actions, prev_chunk_left_over)
-
-        logging.info("=" * 80)
-        logging.info("Evaluation completed successfully")
-
-    def plot_final_actions_comparison(self, rtc_actions, no_rtc_actions, prev_chunk_left_over):
-        """Plot final action predictions comparison on a single chart.
-
-        Args:
-            rtc_actions: Final actions from RTC policy
-            no_rtc_actions: Final actions from non-RTC policy
-            prev_chunk_left_over: Previous chunk used as ground truth
-        """
-        _check_matplotlib_available()
-
-        # Remove batch dimension if present
-        rtc_actions_plot = rtc_actions.squeeze(0).cpu() if len(rtc_actions.shape) == 3 else rtc_actions.cpu()
-        no_rtc_actions_plot = (
-            no_rtc_actions.squeeze(0).cpu() if len(no_rtc_actions.shape) == 3 else no_rtc_actions.cpu()
-        )
-        prev_chunk_plot = prev_chunk_left_over.cpu()
-
-        # Create figure with 6 subplots (one per action dimension)
-        fig, axes = plt.subplots(6, 1, figsize=(16, 12))
-        fig.suptitle("Final Action Predictions Comparison (Raw)", fontsize=16)
-
-        # Plot each action dimension
-        for dim_idx, ax in enumerate(axes):
-            # Plot previous chunk (ground truth) in red
-            RTCDebugVisualizer.plot_waypoints(
-                [ax],
-                prev_chunk_plot[:, dim_idx : dim_idx + 1],
-                start_from=0,
-                color="red",
-                label="Previous Chunk (Ground Truth)",
-                linewidth=2.5,
-                alpha=0.8,
-            )
-
-            # Plot no-RTC actions in blue
-            RTCDebugVisualizer.plot_waypoints(
-                [ax],
-                no_rtc_actions_plot[:, dim_idx : dim_idx + 1],
-                start_from=0,
-                color="blue",
-                label="No RTC",
-                linewidth=2,
-                alpha=0.7,
-            )
-
-            # Plot RTC actions in green
-            RTCDebugVisualizer.plot_waypoints(
-                [ax],
-                rtc_actions_plot[:, dim_idx : dim_idx + 1],
-                start_from=0,
-                color="green",
-                label="RTC",
-                linewidth=2,
-                alpha=0.7,
-            )
-
-            # Add vertical lines for inference delay and execution horizon
-            inference_delay = self.cfg.inference_delay
-            execution_horizon = self.cfg.rtc.execution_horizon
-
-            if inference_delay > 0:
-                ax.axvline(
-                    x=inference_delay - 1,
-                    color="orange",
-                    linestyle="--",
-                    alpha=0.5,
-                    label=f"Inference Delay ({inference_delay})",
-                )
-
-            if execution_horizon > 0:
-                ax.axvline(
-                    x=execution_horizon,
-                    color="purple",
-                    linestyle="--",
-                    alpha=0.5,
-                    label=f"Execution Horizon ({execution_horizon})",
-                )
-
-            ax.set_ylabel(f"Dim {dim_idx}", fontsize=10)
-            ax.grid(True, alpha=0.3)
-
-            # Set x-axis ticks to show all integer values
-            max_len = max(rtc_actions_plot.shape[0], no_rtc_actions_plot.shape[0], prev_chunk_plot.shape[0])
-            ax.set_xticks(range(0, max_len, max(1, max_len // 20)))  # Show ~20 ticks
-            ax.set_xlim(-0.5, max_len - 0.5)
-
-        axes[-1].set_xlabel("Step", fontsize=10)
-
-        # Collect legend handles and labels from first subplot
-        handles, labels = axes[0].get_legend_handles_labels()
-        # Remove duplicates while preserving order
-        seen = set()
-        unique_handles = []
-        unique_labels = []
-        for handle, label in zip(handles, labels, strict=True):
-            if label not in seen:
-                seen.add(label)
-                unique_handles.append(handle)
-                unique_labels.append(label)
-
-        # Add legend outside the plot area (to the right)
-        fig.legend(
-            unique_handles,
-            unique_labels,
-            loc="center right",
-            fontsize=9,
-            bbox_to_anchor=(1.0, 0.5),
-            framealpha=0.9,
-        )
-
-        # Save figure
-        output_path = os.path.join(self.cfg.output_dir, "final_actions_comparison.png")
-        fig.tight_layout(rect=[0, 0, 0.85, 1])  # Leave space for legend on right
-        fig.savefig(output_path, dpi=150, bbox_inches="tight")
-        logging.info(f"Saved final actions comparison to {output_path}")
-        plt.close(fig)
-
-    def plot_tracked_data(self, rtc_tracked_steps, no_rtc_tracked_steps, prev_chunk_left_over, num_steps):
-        _check_matplotlib_available()
-
-        # Create side-by-side figures for denoising visualization
-        fig_xt, axs_xt = self._create_figure("x_t Denoising: No RTC (left) vs RTC (right)")
-        fig_vt, axs_vt = self._create_figure("v_t Denoising: No RTC (left) vs RTC (right)")
-        fig_corr, axs_corr = self._create_figure("Correction: No RTC (left) vs RTC (right)")
-        fig_x1t, axs_x1t = self._create_figure(
-            "x1_t Predicted State & Error: No RTC (left - empty) vs RTC (right)"
-        )
-        self._plot_denoising_steps_from_tracker(
-            rtc_tracked_steps,
-            axs_xt[:, 1],  # Right column for x_t
-            axs_vt[:, 1],  # Right column for v_t
-            axs_corr[:, 1],  # Right column for correction
-            axs_x1t[:, 1],  # Right column for x1_t
-            num_steps,
-            add_labels=True,  # Add labels for RTC (right column)
-        )
-
-        self._plot_denoising_steps_from_tracker(
-            no_rtc_tracked_steps,
-            axs_xt[:, 0],  # Left column for x_t
-            axs_vt[:, 0],  # Left column for v_t
-            axs_corr[:, 0],  # Left column for correction
-            axs_x1t[:, 0],  # Left column for x1_t
-            num_steps,
-            add_labels=False,  # No labels for No RTC (left column)
-        )
-
-        # Plot no-RTC x_t data on right chart as orange dashed line for comparison
-        self._plot_no_rtc_xt_reference(no_rtc_tracked_steps, axs_xt[:, 1], num_steps)
-
-        # Plot ground truth on x_t axes
-        RTCDebugVisualizer.plot_waypoints(
-            axs_xt[:, 1], prev_chunk_left_over, start_from=0, color="red", label="Ground truth"
-        )
-
-        # Plot ground truth on x1_t axes
-        RTCDebugVisualizer.plot_waypoints(
-            axs_x1t[:, 1], prev_chunk_left_over, start_from=0, color="red", label="Ground truth"
-        )
-
-        # Plot ground truth on x_t axes (no labels for left column)
-        RTCDebugVisualizer.plot_waypoints(
-            axs_xt[:, 0], prev_chunk_left_over, start_from=0, color="red", label=None
-        )
-
-        RTCDebugVisualizer.plot_waypoints(
-            axs_x1t[:, 0], prev_chunk_left_over, start_from=0, color="red", label=None
-        )
-
-        # Add legends outside the plot area for each figure
-        self._add_figure_legend(fig_xt, axs_xt)
-        self._add_figure_legend(fig_vt, axs_vt)
-        self._add_figure_legend(fig_corr, axs_corr)
-        self._add_figure_legend(fig_x1t, axs_x1t)
-
-        # Save denoising plots
-        self._save_figure(fig_xt, os.path.join(self.cfg.output_dir, "denoising_xt_comparison.png"))
-        self._save_figure(fig_vt, os.path.join(self.cfg.output_dir, "denoising_vt_comparison.png"))
-        self._save_figure(fig_corr, os.path.join(self.cfg.output_dir, "denoising_correction_comparison.png"))
-        self._save_figure(fig_x1t, os.path.join(self.cfg.output_dir, "denoising_x1t_comparison.png"))
-
-    def _create_figure(self, title):
-        fig, axs = plt.subplots(6, 2, figsize=(24, 12))
-        fig.suptitle(title, fontsize=16)
-
-        for ax in axs[:, 0]:
-            ax.set_title("No RTC (N/A)" if ax == axs[0, 0] else "", fontsize=12)
-        for ax in axs[:, 1]:
-            ax.set_title("RTC" if ax == axs[0, 1] else "", fontsize=12)
-
-        return fig, axs
-
-    def _add_figure_legend(self, fig, axs):
-        """Add a legend outside the plot area on the right side.
-
-        Args:
-            fig: Matplotlib figure to add legend to
-            axs: Array of axes to collect legend handles from
-        """
-        # Collect all handles and labels from the first row of axes (right column)
-        handles, labels = axs[0, 1].get_legend_handles_labels()
-
-        # Remove duplicates while preserving order
-        seen = set()
-        unique_handles = []
-        unique_labels = []
-        for handle, label in zip(handles, labels, strict=True):
-            if label not in seen:
-                seen.add(label)
-                unique_handles.append(handle)
-                unique_labels.append(label)
-
-        # Add legend outside the plot area (to the right, close to charts)
-        if unique_handles:
-            fig.legend(
-                unique_handles,
-                unique_labels,
-                loc="center left",
-                fontsize=8,
-                bbox_to_anchor=(0.87, 0.5),
-                framealpha=0.9,
-                ncol=1,
-            )
-
-    def _save_figure(self, fig, path):
-        fig.tight_layout(rect=[0, 0, 0.85, 1])  # Leave space for legend/colorbar on right
-        fig.savefig(path, dpi=150, bbox_inches="tight")
-        logging.info(f"Saved figure to {path}")
-        plt.close(fig)
-
-    def _plot_denoising_steps_from_tracker(
-        self, tracked_steps, xt_axs, vt_axs, corr_axs, x1t_axs, num_steps, add_labels=True
-    ):
-        """Plot denoising steps from tracker data.
-
-        Args:
-            tracked_steps: List of DebugStep objects containing debug steps
-            xt_axs: Matplotlib axes for x_t plots (array of 6 axes)
-            vt_axs: Matplotlib axes for v_t plots (array of 6 axes)
-            corr_axs: Matplotlib axes for correction plots (array of 6 axes)
-            x1t_axs: Matplotlib axes for x1_t plots (array of 6 axes)
-            num_steps: Total number of denoising steps for colormap
-            add_labels: Whether to add legend labels for the plots
-        """
-
-        logging.info("=" * 80)
-        logging.info(f"Plotting {len(tracked_steps)} steps")
-
-        debug_steps = tracked_steps
-        if not debug_steps:
-            return
-
-        # Define colors for different denoise steps (using a colormap)
-        colors = plt.cm.viridis(np.linspace(0, 1, num_steps))
-
-        for step_idx, debug_step in enumerate(debug_steps):
-            color = colors[step_idx % len(colors)]
-            label = f"Step {step_idx}" if add_labels else None
-
-            # Plot x_t
-            if debug_step.x_t is not None:
-                RTCDebugVisualizer.plot_waypoints(
-                    xt_axs, debug_step.x_t, start_from=0, color=color, label=label
-                )
-
-            # Plot v_t
-            if debug_step.v_t is not None:
-                RTCDebugVisualizer.plot_waypoints(
-                    vt_axs, debug_step.v_t, start_from=0, color=color, label=label
-                )
-
-            # Plot correction on separate axes
-            if debug_step.correction is not None:
-                RTCDebugVisualizer.plot_waypoints(
-                    corr_axs,
-                    debug_step.correction,
-                    start_from=0,
-                    color=color,
-                    label=label,
-                )
-
-            # Plot x1_t (predicted state)
-            if x1t_axs is not None and debug_step.x1_t is not None:
-                x1t_label = f"x1_t Step {step_idx}" if add_labels else None
-                RTCDebugVisualizer.plot_waypoints(
-                    x1t_axs,
-                    debug_step.x1_t,
-                    start_from=0,
-                    color=color,
-                    label=x1t_label,
-                )
-
-            # Plot error in orange dashed
-            if x1t_axs is not None and debug_step.err is not None:
-                error_chunk = (
-                    debug_step.err[0].cpu().numpy()
-                    if len(debug_step.err.shape) == 3
-                    else debug_step.err.cpu().numpy()
-                )
-
-                num_dims = min(error_chunk.shape[-1], 6)
-                error_label = f"error Step {step_idx}" if add_labels else None
-                for j in range(num_dims):
-                    x1t_axs[j].plot(
-                        np.arange(0, error_chunk.shape[0]),
-                        error_chunk[:, j],
-                        color="orange",
-                        linestyle="--",
-                        alpha=0.7,
-                        label=error_label,
-                    )
-
-        # Recalculate axis limits after plotting to ensure proper scaling
-        self._rescale_axes(xt_axs)
-        self._rescale_axes(vt_axs)
-        self._rescale_axes(corr_axs)
-        self._rescale_axes(x1t_axs)
-
-    def _plot_no_rtc_xt_reference(self, no_rtc_tracked_steps, xt_axs, num_steps):
-        """Plot final no-RTC x_t data as orange dashed line on the RTC chart for comparison.
-
-        Args:
-            no_rtc_tracked_steps: List of DebugStep objects containing no-RTC debug steps
-            xt_axs: Matplotlib axes for x_t plots (array of 6 axes, right column)
-            num_steps: Total number of denoising steps for colormap
-        """
-        debug_steps = no_rtc_tracked_steps
-        if not debug_steps:
-            return
-
-        # Plot only the final x_t step as orange dashed line
-        final_step = debug_steps[-1]
-        logging.info("Plotting final no-RTC x_t step as orange dashed reference")
-
-        if final_step.x_t is not None:
-            x_t_chunk = (
-                final_step.x_t[0].cpu().numpy()
-                if len(final_step.x_t.shape) == 3
-                else final_step.x_t.cpu().numpy()
-            )
-
-            num_dims = min(x_t_chunk.shape[-1], 6)
-            for j in range(num_dims):
-                xt_axs[j].plot(
-                    np.arange(0, x_t_chunk.shape[0]),
-                    x_t_chunk[:, j],
-                    color="orange",
-                    linestyle="--",
-                    alpha=0.7,
-                    linewidth=2,
-                    label="No RTC (final)" if j == 0 else "",
-                )
-
-    def _rescale_axes(self, axes):
-        """Rescale axes to show all data with proper margins.
-
-        Args:
-            axes: Array of matplotlib axes to rescale
-        """
-        for ax in axes:
-            ax.relim()
-            ax.autoscale_view()
-
-            # Add 10% margin to y-axis for better visualization
-            ylim = ax.get_ylim()
-            y_range = ylim[1] - ylim[0]
-            if y_range > 0:  # Avoid division by zero
-                margin = y_range * 0.1
-                ax.set_ylim(ylim[0] - margin, ylim[1] + margin)
-
-            # Set x-axis ticks to show all integer values
-            xlim = ax.get_xlim()
-            max_len = int(xlim[1]) + 1
-            if max_len > 0:
-                ax.set_xticks(range(0, max_len, max(1, max_len // 20)))  # Show ~20 ticks
-                ax.set_xlim(-0.5, max_len - 0.5)
-
-
-@parser.wrap()
-def main(cfg: RTCEvalConfig):
-    """Main entry point for RTC evaluation."""
-    # Set random seed for reproducibility
-    set_seed(cfg.seed)
-
-    init_logging()
-
-    logging.info("=" * 80)
-    logging.info("RTC Dataset Evaluation")
-    logging.info(f"Config: {cfg}")
-    logging.info("=" * 80)
-
-    evaluator = RTCEvaluator(cfg)
-    evaluator.run_evaluation()
-
-
-if __name__ == "__main__":
-    main()

examples/rtc/eval_with_real_robot.py

@@ -1,549 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Demo script showing how to use Real-Time Chunking (RTC) with action chunking policies on real robots.
-
-This script demonstrates:
-1. Creating a robot and policy (SmolVLA, Pi0, etc.) with RTC
-2. Consuming actions from the policy while the robot executes
-3. Periodically requesting new action chunks in the background using threads
-4. Managing action buffers and timing for real-time operation
-
-For simulation environments, see eval_with_simulation.py
-
-Usage:
-    # Run RTC with Real robot with RTC
-    uv run examples/rtc/eval_with_real_robot.py \
-        --policy.path=helper2424/smolvla_check_rtc_last3 \
-        --policy.device=mps \
-        --rtc.enabled=true \
-        --rtc.execution_horizon=20 \
-        --robot.type=so100_follower \
-        --robot.port=/dev/tty.usbmodem58FA0834591 \
-        --robot.id=so100_follower \
-        --robot.cameras="{ gripper: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30}, front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \
-        --task="Move green small object into the purple platform" \
-        --duration=120
-
-    # Run RTC with Real robot without RTC
-    uv run examples/rtc/eval_with_real_robot.py \
-        --policy.path=helper2424/smolvla_check_rtc_last3 \
-        --policy.device=mps \
-        --rtc.enabled=false \
-        --robot.type=so100_follower \
-        --robot.port=/dev/tty.usbmodem58FA0834591 \
-        --robot.id=so100_follower \
-        --robot.cameras="{ gripper: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30}, front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \
-        --task="Move green small object into the purple platform" \
-        --duration=120
-
-    # Run RTC with Real robot with pi0.5 policy
-    uv run examples/rtc/eval_with_real_robot.py \
-        --policy.path=helper2424/pi05_check_rtc \
-        --policy.device=mps \
-        --rtc.enabled=true \
-        --rtc.execution_horizon=20 \
-        --robot.type=so100_follower \
-        --robot.port=/dev/tty.usbmodem58FA0834591 \
-        --robot.id=so100_follower \
-        --robot.cameras="{ gripper: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}, front: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30}}" \
-        --task="Move green small object into the purple platform" \
-        --duration=120
-"""
-
-import logging
-import math
-import sys
-import time
-import traceback
-from dataclasses import dataclass, field
-from threading import Event, Lock, Thread
-
-import torch
-from torch import Tensor
-
-from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig  # noqa: F401
-from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig  # noqa: F401
-from lerobot.configs import parser
-from lerobot.configs.policies import PreTrainedConfig
-from lerobot.configs.types import RTCAttentionSchedule
-from lerobot.datasets.utils import build_dataset_frame, hw_to_dataset_features
-from lerobot.policies.factory import get_policy_class, make_pre_post_processors
-from lerobot.policies.rtc.action_queue import ActionQueue
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
-from lerobot.policies.rtc.latency_tracker import LatencyTracker
-from lerobot.processor.factory import (
-    make_default_robot_action_processor,
-    make_default_robot_observation_processor,
-)
-from lerobot.rl.process import ProcessSignalHandler
-from lerobot.robots import (  # noqa: F401
-    Robot,
-    RobotConfig,
-    koch_follower,
-    so100_follower,
-    so101_follower,
-)
-from lerobot.robots.utils import make_robot_from_config
-from lerobot.utils.constants import OBS_IMAGES
-from lerobot.utils.hub import HubMixin
-from lerobot.utils.utils import init_logging
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class RobotWrapper:
-    def __init__(self, robot: Robot):
-        self.robot = robot
-        self.lock = Lock()
-
-    def get_observation(self) -> dict[str, Tensor]:
-        with self.lock:
-            return self.robot.get_observation()
-
-    def send_action(self, action: Tensor):
-        with self.lock:
-            self.robot.send_action(action)
-
-    def observation_features(self) -> list[str]:
-        with self.lock:
-            return self.robot.observation_features
-
-    def action_features(self) -> list[str]:
-        with self.lock:
-            return self.robot.action_features
-
-
-@dataclass
-class RTCDemoConfig(HubMixin):
-    """Configuration for RTC demo with action chunking policies and real robots."""
-
-    # Policy configuration
-    policy: PreTrainedConfig | None = None
-
-    # Robot configuration
-    robot: RobotConfig | None = None
-
-    # RTC configuration
-    rtc: RTCConfig = field(
-        default_factory=lambda: RTCConfig(
-            execution_horizon=10,
-            max_guidance_weight=1.0,
-            prefix_attention_schedule=RTCAttentionSchedule.EXP,
-        )
-    )
-
-    # Demo parameters
-    duration: float = 30.0  # Duration to run the demo (seconds)
-    fps: float = 10.0  # Action execution frequency (Hz)
-
-    # Compute device
-    device: str | None = None  # Device to run on (cuda, cpu, auto)
-
-    # Get new actions horizon. The amount of executed steps after which will be requested new actions.
-    # It should be higher than inference delay + execution horizon.
-    action_queue_size_to_get_new_actions: int = 30
-
-    # Task to execute
-    task: str = field(default="", metadata={"help": "Task to execute"})
-
-    # Torch compile configuration
-    use_torch_compile: bool = field(
-        default=False,
-        metadata={"help": "Use torch.compile for faster inference (PyTorch 2.0+)"},
-    )
-
-    torch_compile_backend: str = field(
-        default="inductor",
-        metadata={"help": "Backend for torch.compile (inductor, aot_eager, cudagraphs)"},
-    )
-
-    torch_compile_mode: str = field(
-        default="default",
-        metadata={"help": "Compilation mode (default, reduce-overhead, max-autotune)"},
-    )
-
-    torch_compile_disable_cudagraphs: bool = field(
-        default=True,
-        metadata={
-            "help": "Disable CUDA graphs in torch.compile. Required due to in-place tensor "
-            "operations in denoising loop (x_t += dt * v_t) which cause tensor aliasing issues."
-        },
-    )
-
-    def __post_init__(self):
-        # HACK: We parse again the cli args here to get the pretrained path if there was one.
-        policy_path = parser.get_path_arg("policy")
-        if policy_path:
-            cli_overrides = parser.get_cli_overrides("policy")
-            self.policy = PreTrainedConfig.from_pretrained(policy_path, cli_overrides=cli_overrides)
-            self.policy.pretrained_path = policy_path
-        else:
-            raise ValueError("Policy path is required")
-
-        # Validate that robot configuration is provided
-        if self.robot is None:
-            raise ValueError("Robot configuration must be provided")
-
-    @classmethod
-    def __get_path_fields__(cls) -> list[str]:
-        """This enables the parser to load config from the policy using `--policy.path=local/dir`"""
-        return ["policy"]
-
-
-def is_image_key(k: str) -> bool:
-    return k.startswith(OBS_IMAGES)
-
-
-def get_actions(
-    policy,
-    robot: RobotWrapper,
-    robot_observation_processor,
-    action_queue: ActionQueue,
-    shutdown_event: Event,
-    cfg: RTCDemoConfig,
-):
-    """Thread function to request action chunks from the policy.
-
-    Args:
-        policy: The policy instance (SmolVLA, Pi0, etc.)
-        robot: The robot instance for getting observations
-        robot_observation_processor: Processor for raw robot observations
-        action_queue: Queue to put new action chunks
-        shutdown_event: Event to signal shutdown
-        cfg: Demo configuration
-    """
-    try:
-        logger.info("[GET_ACTIONS] Starting get actions thread")
-
-        latency_tracker = LatencyTracker()  # Track latency of action chunks
-        fps = cfg.fps
-        time_per_chunk = 1.0 / fps
-
-        dataset_features = hw_to_dataset_features(robot.observation_features(), "observation")
-        policy_device = policy.config.device
-
-        # Load preprocessor and postprocessor from pretrained files
-        # The stats are embedded in the processor .safetensors files
-        logger.info(f"[GET_ACTIONS] Loading preprocessor/postprocessor from {cfg.policy.pretrained_path}")
-
-        preprocessor, postprocessor = make_pre_post_processors(
-            policy_cfg=cfg.policy,
-            pretrained_path=cfg.policy.pretrained_path,
-            dataset_stats=None,  # Will load from pretrained processor files
-            preprocessor_overrides={
-                "device_processor": {"device": cfg.policy.device},
-            },
-        )
-
-        logger.info("[GET_ACTIONS] Preprocessor/postprocessor loaded successfully with embedded stats")
-
-        get_actions_threshold = cfg.action_queue_size_to_get_new_actions
-
-        if not cfg.rtc.enabled:
-            get_actions_threshold = 0
-
-        while not shutdown_event.is_set():
-            if action_queue.qsize() <= get_actions_threshold:
-                current_time = time.perf_counter()
-                action_index_before_inference = action_queue.get_action_index()
-                prev_actions = action_queue.get_left_over()
-
-                inference_latency = latency_tracker.max()
-                inference_delay = math.ceil(inference_latency / time_per_chunk)
-
-                obs = robot.get_observation()
-
-                # Apply robot observation processor
-                obs_processed = robot_observation_processor(obs)
-
-                obs_with_policy_features = build_dataset_frame(
-                    dataset_features, obs_processed, prefix="observation"
-                )
-
-                for name in obs_with_policy_features:
-                    obs_with_policy_features[name] = torch.from_numpy(obs_with_policy_features[name])
-                    if "image" in name:
-                        obs_with_policy_features[name] = (
-                            obs_with_policy_features[name].type(torch.float32) / 255
-                        )
-                        obs_with_policy_features[name] = (
-                            obs_with_policy_features[name].permute(2, 0, 1).contiguous()
-                        )
-                    obs_with_policy_features[name] = obs_with_policy_features[name].unsqueeze(0)
-                    obs_with_policy_features[name] = obs_with_policy_features[name].to(policy_device)
-
-                obs_with_policy_features["task"] = [cfg.task]  # Task should be a list, not a string!
-                obs_with_policy_features["robot_type"] = (
-                    robot.robot.name if hasattr(robot.robot, "name") else ""
-                )
-
-                preproceseded_obs = preprocessor(obs_with_policy_features)
-
-                # Generate actions WITH RTC
-                actions = policy.predict_action_chunk(
-                    preproceseded_obs,
-                    inference_delay=inference_delay,
-                    prev_chunk_left_over=prev_actions,
-                )
-
-                # Store original actions (before postprocessing) for RTC
-                original_actions = actions.squeeze(0).clone()
-
-                postprocessed_actions = postprocessor(actions)
-
-                postprocessed_actions = postprocessed_actions.squeeze(0)
-
-                new_latency = time.perf_counter() - current_time
-                new_delay = math.ceil(new_latency / time_per_chunk)
-                latency_tracker.add(new_latency)
-
-                if cfg.action_queue_size_to_get_new_actions < cfg.rtc.execution_horizon + new_delay:
-                    logger.warning(
-                        "[GET_ACTIONS] cfg.action_queue_size_to_get_new_actions Too small, It should be higher than inference delay + execution horizon."
-                    )
-
-                action_queue.merge(
-                    original_actions, postprocessed_actions, new_delay, action_index_before_inference
-                )
-            else:
-                # Small sleep to prevent busy waiting
-                time.sleep(0.1)
-
-        logger.info("[GET_ACTIONS] get actions thread shutting down")
-    except Exception as e:
-        logger.error(f"[GET_ACTIONS] Fatal exception in get_actions thread: {e}")
-        logger.error(traceback.format_exc())
-        sys.exit(1)
-
-
-def actor_control(
-    robot: RobotWrapper,
-    robot_action_processor,
-    action_queue: ActionQueue,
-    shutdown_event: Event,
-    cfg: RTCDemoConfig,
-):
-    """Thread function to execute actions on the robot.
-
-    Args:
-        robot: The robot instance
-        action_queue: Queue to get actions from
-        shutdown_event: Event to signal shutdown
-        cfg: Demo configuration
-    """
-    try:
-        logger.info("[ACTOR] Starting actor thread")
-
-        action_count = 0
-        action_interval = 1.0 / cfg.fps
-
-        while not shutdown_event.is_set():
-            start_time = time.perf_counter()
-
-            # Try to get an action from the queue with timeout
-            action = action_queue.get()
-
-            if action is not None:
-                action = action.cpu()
-                action_dict = {key: action[i].item() for i, key in enumerate(robot.action_features())}
-                action_processed = robot_action_processor((action_dict, None))
-                robot.send_action(action_processed)
-
-                action_count += 1
-
-            dt_s = time.perf_counter() - start_time
-            time.sleep(max(0, (action_interval - dt_s) - 0.001))
-
-        logger.info(f"[ACTOR] Actor thread shutting down. Total actions executed: {action_count}")
-    except Exception as e:
-        logger.error(f"[ACTOR] Fatal exception in actor_control thread: {e}")
-        logger.error(traceback.format_exc())
-        sys.exit(1)
-
-
-def _apply_torch_compile(policy, cfg: RTCDemoConfig):
-    """Apply torch.compile to the policy's predict_action_chunk method.
-
-    Args:
-        policy: Policy instance to compile
-        cfg: Configuration containing torch compile settings
-
-    Returns:
-        Policy with compiled predict_action_chunk method
-    """
-
-    # PI models handle their own compilation
-    if policy.type == "pi05" or policy.type == "pi0":
-        return policy
-
-    try:
-        # Check if torch.compile is available (PyTorch 2.0+)
-        if not hasattr(torch, "compile"):
-            logger.warning(
-                f"torch.compile is not available. Requires PyTorch 2.0+. "
-                f"Current version: {torch.__version__}. Skipping compilation."
-            )
-            return policy
-
-        logger.info("Applying torch.compile to predict_action_chunk...")
-        logger.info(f"  Backend: {cfg.torch_compile_backend}")
-        logger.info(f"  Mode: {cfg.torch_compile_mode}")
-        logger.info(f"  Disable CUDA graphs: {cfg.torch_compile_disable_cudagraphs}")
-
-        # Compile the predict_action_chunk method
-        # - CUDA graphs disabled to prevent tensor aliasing from in-place ops (x_t += dt * v_t)
-        compile_kwargs = {
-            "backend": cfg.torch_compile_backend,
-            "mode": cfg.torch_compile_mode,
-        }
-
-        # Disable CUDA graphs if requested (prevents tensor aliasing issues)
-        if cfg.torch_compile_disable_cudagraphs:
-            compile_kwargs["options"] = {"triton.cudagraphs": False}
-
-        original_method = policy.predict_action_chunk
-        compiled_method = torch.compile(original_method, **compile_kwargs)
-        policy.predict_action_chunk = compiled_method
-        logger.info("✓ Successfully compiled predict_action_chunk")
-
-    except Exception as e:
-        logger.error(f"Failed to apply torch.compile: {e}")
-        logger.warning("Continuing without torch.compile")
-
-    return policy
-
-
-@parser.wrap()
-def demo_cli(cfg: RTCDemoConfig):
-    """Main entry point for RTC demo with draccus configuration."""
-
-    # Initialize logging
-    init_logging()
-
-    logger.info(f"Using device: {cfg.device}")
-
-    # Setup signal handler for graceful shutdown
-    signal_handler = ProcessSignalHandler(use_threads=True, display_pid=False)
-    shutdown_event = signal_handler.shutdown_event
-
-    policy = None
-    robot = None
-    get_actions_thread = None
-    actor_thread = None
-
-    policy_class = get_policy_class(cfg.policy.type)
-
-    # Load config and set compile_model for pi0/pi05 models
-    config = PreTrainedConfig.from_pretrained(cfg.policy.pretrained_path)
-
-    if cfg.policy.type == "pi05" or cfg.policy.type == "pi0":
-        config.compile_model = cfg.use_torch_compile
-
-    policy = policy_class.from_pretrained(cfg.policy.pretrained_path, config=config)
-
-    # Turn on RTC
-    policy.config.rtc_config = cfg.rtc
-
-    # Init RTC processort, as by default if RTC disabled in the config
-    # The processor won't be created
-    policy.init_rtc_processor()
-
-    assert policy.name in ["smolvla", "pi05", "pi0"], "Only smolvla, pi05, and pi0 are supported for RTC"
-
-    policy = policy.to(cfg.device)
-    policy.eval()
-
-    # Apply torch.compile to predict_action_chunk method if enabled
-    if cfg.use_torch_compile:
-        policy = _apply_torch_compile(policy, cfg)
-
-    # Create robot
-    logger.info(f"Initializing robot: {cfg.robot.type}")
-    robot = make_robot_from_config(cfg.robot)
-    robot.connect()
-    robot_wrapper = RobotWrapper(robot)
-
-    # Create robot observation processor
-    robot_observation_processor = make_default_robot_observation_processor()
-    robot_action_processor = make_default_robot_action_processor()
-
-    # Create action queue for communication between threads
-    action_queue = ActionQueue(cfg.rtc)
-
-    # Start chunk requester thread
-    get_actions_thread = Thread(
-        target=get_actions,
-        args=(policy, robot_wrapper, robot_observation_processor, action_queue, shutdown_event, cfg),
-        daemon=True,
-        name="GetActions",
-    )
-    get_actions_thread.start()
-    logger.info("Started get actions thread")
-
-    # Start action executor thread
-    actor_thread = Thread(
-        target=actor_control,
-        args=(robot_wrapper, robot_action_processor, action_queue, shutdown_event, cfg),
-        daemon=True,
-        name="Actor",
-    )
-    actor_thread.start()
-    logger.info("Started actor thread")
-
-    logger.info("Started stop by duration thread")
-
-    # Main thread monitors for duration or shutdown
-    logger.info(f"Running demo for {cfg.duration} seconds...")
-    start_time = time.time()
-
-    while not shutdown_event.is_set() and (time.time() - start_time) < cfg.duration:
-        time.sleep(10)
-
-        # Log queue status periodically
-        if int(time.time() - start_time) % 5 == 0:
-            logger.info(f"[MAIN] Action queue size: {action_queue.qsize()}")
-
-        if time.time() - start_time > cfg.duration:
-            break
-
-    logger.info("Demo duration reached or shutdown requested")
-
-    # Signal shutdown
-    shutdown_event.set()
-
-    # Wait for threads to finish
-    if get_actions_thread and get_actions_thread.is_alive():
-        logger.info("Waiting for chunk requester thread to finish...")
-        get_actions_thread.join()
-
-    if actor_thread and actor_thread.is_alive():
-        logger.info("Waiting for action executor thread to finish...")
-        actor_thread.join()
-
-    # Cleanup robot
-    if robot:
-        robot.disconnect()
-        logger.info("Robot disconnected")
-
-    logger.info("Cleanup completed")
-
-
-if __name__ == "__main__":
-    demo_cli()
-    logging.info("RTC demo finished")

examples/so100_to_so100_EE/evaluate.py

@@ -52,114 +52,126 @@ TASK_DESCRIPTION = "My task description"
 HF_MODEL_ID = "<hf_username>/<model_repo_id>"
 HF_DATASET_ID = "<hf_username>/<dataset_repo_id>"
 
+# Create the robot configuration & robot
+camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
+robot_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411",
+    id="my_awesome_follower_arm",
+    cameras=camera_config,
+    use_degrees=True,
+)
 
-def main():
-    # Create the robot configuration & robot
-    camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
-    robot_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411",
-        id="my_awesome_follower_arm",
-        cameras=camera_config,
-        use_degrees=True,
-    )
+robot = SO100Follower(robot_config)
 
-    robot = SO100Follower(robot_config)
+# Create policy
+policy = ACTPolicy.from_pretrained(HF_MODEL_ID)
 
-    # Create policy
-    policy = ACTPolicy.from_pretrained(HF_MODEL_ID)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(robot.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(robot.bus.motors.keys()),
-    )
-
-    # Build pipeline to convert EE action to joints action
-    robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        steps=[
-            InverseKinematicsEEToJoints(
-                kinematics=kinematics_solver,
-                motor_names=list(robot.bus.motors.keys()),
-                initial_guess_current_joints=True,
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
-
-    # Build pipeline to convert joints observation to EE observation
-    robot_joints_to_ee_pose_processor = RobotProcessorPipeline[RobotObservation, RobotObservation](
-        steps=[
-            ForwardKinematicsJointsToEE(
-                kinematics=kinematics_solver, motor_names=list(robot.bus.motors.keys())
-            )
-        ],
-        to_transition=observation_to_transition,
-        to_output=transition_to_observation,
-    )
-
-    # Create the dataset
-    dataset = LeRobotDataset.create(
-        repo_id=HF_DATASET_ID,
-        fps=FPS,
-        features=combine_feature_dicts(
-            aggregate_pipeline_dataset_features(
-                pipeline=robot_joints_to_ee_pose_processor,
-                initial_features=create_initial_features(observation=robot.observation_features),
-                use_videos=True,
-            ),
-            # User for now should be explicit on the feature keys that were used for record
-            # Alternatively, the user can pass the processor step that has the right features
-            aggregate_pipeline_dataset_features(
-                pipeline=make_default_teleop_action_processor(),
-                initial_features=create_initial_features(
-                    action={
-                        f"ee.{k}": PolicyFeature(type=FeatureType.ACTION, shape=(1,))
-                        for k in ["x", "y", "z", "wx", "wy", "wz", "gripper_pos"]
-                    }
-                ),
-                use_videos=True,
-            ),
+# Build pipeline to convert EE action to joints action
+robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        InverseKinematicsEEToJoints(
+            kinematics=kinematics_solver,
+            motor_names=list(robot.bus.motors.keys()),
+            initial_guess_current_joints=True,
         ),
-        robot_type=robot.name,
-        use_videos=True,
-        image_writer_threads=4,
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
+
+# Build pipeline to convert joints observation to EE observation
+robot_joints_to_ee_pose_processor = RobotProcessorPipeline[RobotObservation, RobotObservation](
+    steps=[
+        ForwardKinematicsJointsToEE(kinematics=kinematics_solver, motor_names=list(robot.bus.motors.keys()))
+    ],
+    to_transition=observation_to_transition,
+    to_output=transition_to_observation,
+)
+
+
+# Create the dataset
+dataset = LeRobotDataset.create(
+    repo_id=HF_DATASET_ID,
+    fps=FPS,
+    features=combine_feature_dicts(
+        aggregate_pipeline_dataset_features(
+            pipeline=robot_joints_to_ee_pose_processor,
+            initial_features=create_initial_features(observation=robot.observation_features),
+            use_videos=True,
+        ),
+        # User for now should be explicit on the feature keys that were used for record
+        # Alternatively, the user can pass the processor step that has the right features
+        aggregate_pipeline_dataset_features(
+            pipeline=make_default_teleop_action_processor(),
+            initial_features=create_initial_features(
+                action={
+                    f"ee.{k}": PolicyFeature(type=FeatureType.ACTION, shape=(1,))
+                    for k in ["x", "y", "z", "wx", "wy", "wz", "gripper_pos"]
+                }
+            ),
+            use_videos=True,
+        ),
+    ),
+    robot_type=robot.name,
+    use_videos=True,
+    image_writer_threads=4,
+)
+
+# Build Policy Processors
+preprocessor, postprocessor = make_pre_post_processors(
+    policy_cfg=policy,
+    pretrained_path=HF_MODEL_ID,
+    dataset_stats=dataset.meta.stats,
+    # The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
+    preprocessor_overrides={"device_processor": {"device": str(policy.config.device)}},
+)
+
+# Connect the robot and teleoperator
+robot.connect()
+
+# Initialize the keyboard listener and rerun visualization
+listener, events = init_keyboard_listener()
+init_rerun(session_name="so100_so100_evaluate")
+
+if not robot.is_connected:
+    raise ValueError("Robot is not connected!")
+
+print("Starting evaluate loop...")
+episode_idx = 0
+for episode_idx in range(NUM_EPISODES):
+    log_say(f"Running inference, recording eval episode {episode_idx + 1} of {NUM_EPISODES}")
+
+    # Main record loop
+    record_loop(
+        robot=robot,
+        events=events,
+        fps=FPS,
+        policy=policy,
+        preprocessor=preprocessor,  # Pass the pre and post policy processors
+        postprocessor=postprocessor,
+        dataset=dataset,
+        control_time_s=EPISODE_TIME_SEC,
+        single_task=TASK_DESCRIPTION,
+        display_data=True,
+        teleop_action_processor=make_default_teleop_action_processor(),
+        robot_action_processor=robot_ee_to_joints_processor,
+        robot_observation_processor=robot_joints_to_ee_pose_processor,
     )
 
-    # Build Policy Processors
-    preprocessor, postprocessor = make_pre_post_processors(
-        policy_cfg=policy,
-        pretrained_path=HF_MODEL_ID,
-        dataset_stats=dataset.meta.stats,
-        # The inference device is automatically set to match the detected hardware, overriding any previous device settings from training to ensure compatibility.
-        preprocessor_overrides={"device_processor": {"device": str(policy.config.device)}},
-    )
-
-    # Connect the robot and teleoperator
-    robot.connect()
-
-    # Initialize the keyboard listener and rerun visualization
-    listener, events = init_keyboard_listener()
-    init_rerun(session_name="so100_so100_evaluate")
-
-    if not robot.is_connected:
-        raise ValueError("Robot is not connected!")
-
-    print("Starting evaluate loop...")
-    episode_idx = 0
-    for episode_idx in range(NUM_EPISODES):
-        log_say(f"Running inference, recording eval episode {episode_idx + 1} of {NUM_EPISODES}")
-
-        # Main record loop
+    # Reset the environment if not stopping or re-recording
+    if not events["stop_recording"] and ((episode_idx < NUM_EPISODES - 1) or events["rerecord_episode"]):
+        log_say("Reset the environment")
         record_loop(
             robot=robot,
             events=events,
             fps=FPS,
-            policy=policy,
-            preprocessor=preprocessor,  # Pass the pre and post policy processors
-            postprocessor=postprocessor,
-            dataset=dataset,
             control_time_s=EPISODE_TIME_SEC,
             single_task=TASK_DESCRIPTION,
             display_data=True,
@@ -168,40 +180,21 @@ def main():
             robot_observation_processor=robot_joints_to_ee_pose_processor,
         )
 
-        # Reset the environment if not stopping or re-recording
-        if not events["stop_recording"] and ((episode_idx < NUM_EPISODES - 1) or events["rerecord_episode"]):
-            log_say("Reset the environment")
-            record_loop(
-                robot=robot,
-                events=events,
-                fps=FPS,
-                control_time_s=EPISODE_TIME_SEC,
-                single_task=TASK_DESCRIPTION,
-                display_data=True,
-                teleop_action_processor=make_default_teleop_action_processor(),
-                robot_action_processor=robot_ee_to_joints_processor,
-                robot_observation_processor=robot_joints_to_ee_pose_processor,
-            )
+    if events["rerecord_episode"]:
+        log_say("Re-record episode")
+        events["rerecord_episode"] = False
+        events["exit_early"] = False
+        dataset.clear_episode_buffer()
+        continue
 
-        if events["rerecord_episode"]:
-            log_say("Re-record episode")
-            events["rerecord_episode"] = False
-            events["exit_early"] = False
-            dataset.clear_episode_buffer()
-            continue
+    # Save episode
+    dataset.save_episode()
+    episode_idx += 1
 
-        # Save episode
-        dataset.save_episode()
-        episode_idx += 1
+# Clean up
+log_say("Stop recording")
+robot.disconnect()
+listener.stop()
 
-    # Clean up
-    log_say("Stop recording")
-    robot.disconnect()
-    listener.stop()
-
-    dataset.finalize()
-    dataset.push_to_hub()
-
-
-if __name__ == "__main__":
-    main()
+dataset.finalize()
+dataset.push_to_hub()

examples/so100_to_so100_EE/record.py

@@ -48,122 +48,134 @@ RESET_TIME_SEC = 30
 TASK_DESCRIPTION = "My task description"
 HF_REPO_ID = "<hf_username>/<dataset_repo_id>"
 
+# Create the robot and teleoperator configurations
+camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
+follower_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", cameras=camera_config, use_degrees=True
+)
+leader_config = SO100LeaderConfig(port="/dev/tty.usbmodem5A460819811", id="my_awesome_leader_arm")
 
-def main():
-    # Create the robot and teleoperator configurations
-    camera_config = {"front": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=FPS)}
-    follower_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411",
-        id="my_awesome_follower_arm",
-        cameras=camera_config,
-        use_degrees=True,
-    )
-    leader_config = SO100LeaderConfig(port="/dev/tty.usbmodem5A460819811", id="my_awesome_leader_arm")
+# Initialize the robot and teleoperator
+follower = SO100Follower(follower_config)
+leader = SO100Leader(leader_config)
 
-    # Initialize the robot and teleoperator
-    follower = SO100Follower(follower_config)
-    leader = SO100Leader(leader_config)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+follower_kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(follower.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    follower_kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(follower.bus.motors.keys()),
-    )
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+leader_kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(leader.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    leader_kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(leader.bus.motors.keys()),
-    )
-
-    # Build pipeline to convert follower joints to EE observation
-    follower_joints_to_ee = RobotProcessorPipeline[RobotObservation, RobotObservation](
-        steps=[
-            ForwardKinematicsJointsToEE(
-                kinematics=follower_kinematics_solver, motor_names=list(follower.bus.motors.keys())
-            ),
-        ],
-        to_transition=observation_to_transition,
-        to_output=transition_to_observation,
-    )
-
-    # Build pipeline to convert leader joints to EE action
-    leader_joints_to_ee = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        steps=[
-            ForwardKinematicsJointsToEE(
-                kinematics=leader_kinematics_solver, motor_names=list(leader.bus.motors.keys())
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
-
-    # Build pipeline to convert EE action to follower joints
-    ee_to_follower_joints = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        [
-            EEBoundsAndSafety(
-                end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
-                max_ee_step_m=0.10,
-            ),
-            InverseKinematicsEEToJoints(
-                kinematics=follower_kinematics_solver,
-                motor_names=list(follower.bus.motors.keys()),
-                initial_guess_current_joints=True,
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
-
-    # Create the dataset
-    dataset = LeRobotDataset.create(
-        repo_id=HF_REPO_ID,
-        fps=FPS,
-        features=combine_feature_dicts(
-            # Run the feature contract of the pipelines
-            # This tells you how the features would look like after the pipeline steps
-            aggregate_pipeline_dataset_features(
-                pipeline=leader_joints_to_ee,
-                initial_features=create_initial_features(action=leader.action_features),
-                use_videos=True,
-            ),
-            aggregate_pipeline_dataset_features(
-                pipeline=follower_joints_to_ee,
-                initial_features=create_initial_features(observation=follower.observation_features),
-                use_videos=True,
-            ),
+# Build pipeline to convert follower joints to EE observation
+follower_joints_to_ee = RobotProcessorPipeline[RobotObservation, RobotObservation](
+    steps=[
+        ForwardKinematicsJointsToEE(
+            kinematics=follower_kinematics_solver, motor_names=list(follower.bus.motors.keys())
         ),
-        robot_type=follower.name,
-        use_videos=True,
-        image_writer_threads=4,
+    ],
+    to_transition=observation_to_transition,
+    to_output=transition_to_observation,
+)
+
+# Build pipeline to convert leader joints to EE action
+leader_joints_to_ee = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        ForwardKinematicsJointsToEE(
+            kinematics=leader_kinematics_solver, motor_names=list(leader.bus.motors.keys())
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
+
+# Build pipeline to convert EE action to follower joints
+ee_to_follower_joints = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    [
+        EEBoundsAndSafety(
+            end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
+            max_ee_step_m=0.10,
+        ),
+        InverseKinematicsEEToJoints(
+            kinematics=follower_kinematics_solver,
+            motor_names=list(follower.bus.motors.keys()),
+            initial_guess_current_joints=True,
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
+
+# Create the dataset
+dataset = LeRobotDataset.create(
+    repo_id=HF_REPO_ID,
+    fps=FPS,
+    features=combine_feature_dicts(
+        # Run the feature contract of the pipelines
+        # This tells you how the features would look like after the pipeline steps
+        aggregate_pipeline_dataset_features(
+            pipeline=leader_joints_to_ee,
+            initial_features=create_initial_features(action=leader.action_features),
+            use_videos=True,
+        ),
+        aggregate_pipeline_dataset_features(
+            pipeline=follower_joints_to_ee,
+            initial_features=create_initial_features(observation=follower.observation_features),
+            use_videos=True,
+        ),
+    ),
+    robot_type=follower.name,
+    use_videos=True,
+    image_writer_threads=4,
+)
+
+
+# Connect the robot and teleoperator
+leader.connect()
+follower.connect()
+
+# Initialize the keyboard listener and rerun visualization
+listener, events = init_keyboard_listener()
+init_rerun(session_name="recording_phone")
+
+if not leader.is_connected or not follower.is_connected:
+    raise ValueError("Robot or teleop is not connected!")
+
+print("Starting record loop...")
+episode_idx = 0
+while episode_idx < NUM_EPISODES and not events["stop_recording"]:
+    log_say(f"Recording episode {episode_idx + 1} of {NUM_EPISODES}")
+
+    # Main record loop
+    record_loop(
+        robot=follower,
+        events=events,
+        fps=FPS,
+        teleop=leader,
+        dataset=dataset,
+        control_time_s=EPISODE_TIME_SEC,
+        single_task=TASK_DESCRIPTION,
+        display_data=True,
+        teleop_action_processor=leader_joints_to_ee,
+        robot_action_processor=ee_to_follower_joints,
+        robot_observation_processor=follower_joints_to_ee,
     )
 
-    # Connect the robot and teleoperator
-    leader.connect()
-    follower.connect()
-
-    # Initialize the keyboard listener and rerun visualization
-    listener, events = init_keyboard_listener()
-    init_rerun(session_name="recording_phone")
-
-    if not leader.is_connected or not follower.is_connected:
-        raise ValueError("Robot or teleop is not connected!")
-
-    print("Starting record loop...")
-    episode_idx = 0
-    while episode_idx < NUM_EPISODES and not events["stop_recording"]:
-        log_say(f"Recording episode {episode_idx + 1} of {NUM_EPISODES}")
-
-        # Main record loop
+    # Reset the environment if not stopping or re-recording
+    if not events["stop_recording"] and (episode_idx < NUM_EPISODES - 1 or events["rerecord_episode"]):
+        log_say("Reset the environment")
         record_loop(
             robot=follower,
             events=events,
             fps=FPS,
             teleop=leader,
-            dataset=dataset,
-            control_time_s=EPISODE_TIME_SEC,
+            control_time_s=RESET_TIME_SEC,
             single_task=TASK_DESCRIPTION,
             display_data=True,
             teleop_action_processor=leader_joints_to_ee,
@@ -171,42 +183,22 @@ def main():
             robot_observation_processor=follower_joints_to_ee,
         )
 
-        # Reset the environment if not stopping or re-recording
-        if not events["stop_recording"] and (episode_idx < NUM_EPISODES - 1 or events["rerecord_episode"]):
-            log_say("Reset the environment")
-            record_loop(
-                robot=follower,
-                events=events,
-                fps=FPS,
-                teleop=leader,
-                control_time_s=RESET_TIME_SEC,
-                single_task=TASK_DESCRIPTION,
-                display_data=True,
-                teleop_action_processor=leader_joints_to_ee,
-                robot_action_processor=ee_to_follower_joints,
-                robot_observation_processor=follower_joints_to_ee,
-            )
+    if events["rerecord_episode"]:
+        log_say("Re-recording episode")
+        events["rerecord_episode"] = False
+        events["exit_early"] = False
+        dataset.clear_episode_buffer()
+        continue
 
-        if events["rerecord_episode"]:
-            log_say("Re-recording episode")
-            events["rerecord_episode"] = False
-            events["exit_early"] = False
-            dataset.clear_episode_buffer()
-            continue
+    # Save episode
+    dataset.save_episode()
+    episode_idx += 1
 
-        # Save episode
-        dataset.save_episode()
-        episode_idx += 1
+# Clean up
+log_say("Stop recording")
+leader.disconnect()
+follower.disconnect()
+listener.stop()
 
-    # Clean up
-    log_say("Stop recording")
-    leader.disconnect()
-    follower.disconnect()
-    listener.stop()
-
-    dataset.finalize()
-    dataset.push_to_hub()
-
-
-if __name__ == "__main__":
-    main()
+dataset.finalize()
+dataset.push_to_hub()

examples/so100_to_so100_EE/replay.py

@@ -30,78 +30,72 @@ from lerobot.robots.so100_follower.robot_kinematic_processor import (
 )
 from lerobot.robots.so100_follower.so100_follower import SO100Follower
 from lerobot.utils.constants import ACTION
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.utils import log_say
 
 EPISODE_IDX = 0
 HF_REPO_ID = "<hf_username>/<dataset_repo_id>"
 
+# Initialize the robot config
+robot_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
+)
 
-def main():
-    # Initialize the robot config
-    robot_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
-    )
+# Initialize the robot
+robot = SO100Follower(robot_config)
 
-    # Initialize the robot
-    robot = SO100Follower(robot_config)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(robot.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(robot.bus.motors.keys()),
-    )
+# Build pipeline to convert EE action to joints action
+robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    steps=[
+        InverseKinematicsEEToJoints(
+            kinematics=kinematics_solver,
+            motor_names=list(robot.bus.motors.keys()),
+            initial_guess_current_joints=False,  # Because replay is open loop
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
 
-    # Build pipeline to convert EE action to joints action
-    robot_ee_to_joints_processor = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        steps=[
-            InverseKinematicsEEToJoints(
-                kinematics=kinematics_solver,
-                motor_names=list(robot.bus.motors.keys()),
-                initial_guess_current_joints=False,  # Because replay is open loop
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
+# Fetch the dataset to replay
+dataset = LeRobotDataset(HF_REPO_ID, episodes=[EPISODE_IDX])
+# Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
+episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == EPISODE_IDX)
+actions = episode_frames.select_columns(ACTION)
 
-    # Fetch the dataset to replay
-    dataset = LeRobotDataset(HF_REPO_ID, episodes=[EPISODE_IDX])
-    # Filter dataset to only include frames from the specified episode since episodes are chunked in dataset V3.0
-    episode_frames = dataset.hf_dataset.filter(lambda x: x["episode_index"] == EPISODE_IDX)
-    actions = episode_frames.select_columns(ACTION)
+# Connect to the robot
+robot.connect()
 
-    # Connect to the robot
-    robot.connect()
+if not robot.is_connected:
+    raise ValueError("Robot is not connected!")
 
-    if not robot.is_connected:
-        raise ValueError("Robot is not connected!")
+print("Starting replay loop...")
+log_say(f"Replaying episode {EPISODE_IDX}")
+for idx in range(len(episode_frames)):
+    t0 = time.perf_counter()
 
-    print("Starting replay loop...")
-    log_say(f"Replaying episode {EPISODE_IDX}")
-    for idx in range(len(episode_frames)):
-        t0 = time.perf_counter()
+    # Get recorded action from dataset
+    ee_action = {
+        name: float(actions[idx][ACTION][i]) for i, name in enumerate(dataset.features[ACTION]["names"])
+    }
 
-        # Get recorded action from dataset
-        ee_action = {
-            name: float(actions[idx][ACTION][i]) for i, name in enumerate(dataset.features[ACTION]["names"])
-        }
+    # Get robot observation
+    robot_obs = robot.get_observation()
 
-        # Get robot observation
-        robot_obs = robot.get_observation()
+    # Dataset EE -> robot joints
+    joint_action = robot_ee_to_joints_processor((ee_action, robot_obs))
 
-        # Dataset EE -> robot joints
-        joint_action = robot_ee_to_joints_processor((ee_action, robot_obs))
+    # Send action to robot
+    _ = robot.send_action(joint_action)
 
-        # Send action to robot
-        _ = robot.send_action(joint_action)
+    busy_wait(1.0 / dataset.fps - (time.perf_counter() - t0))
 
-        precise_sleep(1.0 / dataset.fps - (time.perf_counter() - t0))
-
-    # Clean up
-    robot.disconnect()
-
-
-if __name__ == "__main__":
-    main()
+# Clean up
+robot.disconnect()

examples/so100_to_so100_EE/teleoperate.py

@@ -32,96 +32,90 @@ from lerobot.robots.so100_follower.robot_kinematic_processor import (
 from lerobot.robots.so100_follower.so100_follower import SO100Follower
 from lerobot.teleoperators.so100_leader.config_so100_leader import SO100LeaderConfig
 from lerobot.teleoperators.so100_leader.so100_leader import SO100Leader
-from lerobot.utils.robot_utils import precise_sleep
+from lerobot.utils.robot_utils import busy_wait
 from lerobot.utils.visualization_utils import init_rerun, log_rerun_data
 
 FPS = 30
 
+# Initialize the robot and teleoperator config
+follower_config = SO100FollowerConfig(
+    port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
+)
+leader_config = SO100LeaderConfig(port="/dev/tty.usbmodem5A460819811", id="my_awesome_leader_arm")
 
-def main():
-    # Initialize the robot and teleoperator config
-    follower_config = SO100FollowerConfig(
-        port="/dev/tty.usbmodem5A460814411", id="my_awesome_follower_arm", use_degrees=True
-    )
-    leader_config = SO100LeaderConfig(port="/dev/tty.usbmodem5A460819811", id="my_awesome_leader_arm")
+# Initialize the robot and teleoperator
+follower = SO100Follower(follower_config)
+leader = SO100Leader(leader_config)
 
-    # Initialize the robot and teleoperator
-    follower = SO100Follower(follower_config)
-    leader = SO100Leader(leader_config)
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+follower_kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(follower.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    follower_kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(follower.bus.motors.keys()),
-    )
+# NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
+leader_kinematics_solver = RobotKinematics(
+    urdf_path="./SO101/so101_new_calib.urdf",
+    target_frame_name="gripper_frame_link",
+    joint_names=list(leader.bus.motors.keys()),
+)
 
-    # NOTE: It is highly recommended to use the urdf in the SO-ARM100 repo: https://github.com/TheRobotStudio/SO-ARM100/blob/main/Simulation/SO101/so101_new_calib.urdf
-    leader_kinematics_solver = RobotKinematics(
-        urdf_path="./SO101/so101_new_calib.urdf",
-        target_frame_name="gripper_frame_link",
-        joint_names=list(leader.bus.motors.keys()),
-    )
+# Build pipeline to convert teleop joints to EE action
+leader_to_ee = RobotProcessorPipeline[RobotAction, RobotAction](
+    steps=[
+        ForwardKinematicsJointsToEE(
+            kinematics=leader_kinematics_solver, motor_names=list(leader.bus.motors.keys())
+        ),
+    ],
+    to_transition=robot_action_to_transition,
+    to_output=transition_to_robot_action,
+)
 
-    # Build pipeline to convert teleop joints to EE action
-    leader_to_ee = RobotProcessorPipeline[RobotAction, RobotAction](
-        steps=[
-            ForwardKinematicsJointsToEE(
-                kinematics=leader_kinematics_solver, motor_names=list(leader.bus.motors.keys())
-            ),
-        ],
-        to_transition=robot_action_to_transition,
-        to_output=transition_to_robot_action,
-    )
+# build pipeline to convert EE action to robot joints
+ee_to_follower_joints = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+    [
+        EEBoundsAndSafety(
+            end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
+            max_ee_step_m=0.10,
+        ),
+        InverseKinematicsEEToJoints(
+            kinematics=follower_kinematics_solver,
+            motor_names=list(follower.bus.motors.keys()),
+            initial_guess_current_joints=False,
+        ),
+    ],
+    to_transition=robot_action_observation_to_transition,
+    to_output=transition_to_robot_action,
+)
 
-    # build pipeline to convert EE action to robot joints
-    ee_to_follower_joints = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
-        [
-            EEBoundsAndSafety(
-                end_effector_bounds={"min": [-1.0, -1.0, -1.0], "max": [1.0, 1.0, 1.0]},
-                max_ee_step_m=0.10,
-            ),
-            InverseKinematicsEEToJoints(
-                kinematics=follower_kinematics_solver,
-                motor_names=list(follower.bus.motors.keys()),
-                initial_guess_current_joints=False,
-            ),
-        ],
-        to_transition=robot_action_observation_to_transition,
-        to_output=transition_to_robot_action,
-    )
+# Connect to the robot and teleoperator
+follower.connect()
+leader.connect()
 
-    # Connect to the robot and teleoperator
-    follower.connect()
-    leader.connect()
+# Init rerun viewer
+init_rerun(session_name="so100_so100_EE_teleop")
 
-    # Init rerun viewer
-    init_rerun(session_name="so100_so100_EE_teleop")
+print("Starting teleop loop...")
+while True:
+    t0 = time.perf_counter()
 
-    print("Starting teleop loop...")
-    while True:
-        t0 = time.perf_counter()
+    # Get robot observation
+    robot_obs = follower.get_observation()
 
-        # Get robot observation
-        robot_obs = follower.get_observation()
+    # Get teleop observation
+    leader_joints_obs = leader.get_action()
 
-        # Get teleop observation
-        leader_joints_obs = leader.get_action()
+    # teleop joints -> teleop EE action
+    leader_ee_act = leader_to_ee(leader_joints_obs)
 
-        # teleop joints -> teleop EE action
-        leader_ee_act = leader_to_ee(leader_joints_obs)
+    # teleop EE -> robot joints
+    follower_joints_act = ee_to_follower_joints((leader_ee_act, robot_obs))
 
-        # teleop EE -> robot joints
-        follower_joints_act = ee_to_follower_joints((leader_ee_act, robot_obs))
+    # Send action to robot
+    _ = follower.send_action(follower_joints_act)
 
-        # Send action to robot
-        _ = follower.send_action(follower_joints_act)
+    # Visualize
+    log_rerun_data(observation=leader_ee_act, action=follower_joints_act)
 
-        # Visualize
-        log_rerun_data(observation=leader_ee_act, action=follower_joints_act)
-
-        precise_sleep(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))
-
-
-if __name__ == "__main__":
-    main()
+    busy_wait(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))

examples/tutorial/act/act_training_example.py

@@ -19,86 +19,80 @@ def make_delta_timestamps(delta_indices: list[int] | None, fps: int) -> list[flo
     return [i / fps for i in delta_indices]
 
 
-def main():
-    output_directory = Path("outputs/robot_learning_tutorial/act")
-    output_directory.mkdir(parents=True, exist_ok=True)
+output_directory = Path("outputs/robot_learning_tutorial/act")
+output_directory.mkdir(parents=True, exist_ok=True)
 
-    # Select your device
-    device = torch.device("mps")  # or "cuda" or "cpu"
+# Select your device
+device = torch.device("mps")  # or "cuda" or "cpu"
 
-    dataset_id = "lerobot/svla_so101_pickplace"
+dataset_id = "lerobot/svla_so101_pickplace"
 
-    # This specifies the inputs the model will be expecting and the outputs it will produce
-    dataset_metadata = LeRobotDatasetMetadata(dataset_id)
-    features = dataset_to_policy_features(dataset_metadata.features)
+# This specifies the inputs the model will be expecting and the outputs it will produce
+dataset_metadata = LeRobotDatasetMetadata(dataset_id)
+features = dataset_to_policy_features(dataset_metadata.features)
 
-    output_features = {key: ft for key, ft in features.items() if ft.type is FeatureType.ACTION}
-    input_features = {key: ft for key, ft in features.items() if key not in output_features}
+output_features = {key: ft for key, ft in features.items() if ft.type is FeatureType.ACTION}
+input_features = {key: ft for key, ft in features.items() if key not in output_features}
 
-    cfg = ACTConfig(input_features=input_features, output_features=output_features)
-    policy = ACTPolicy(cfg)
-    preprocessor, postprocessor = make_pre_post_processors(cfg, dataset_stats=dataset_metadata.stats)
+cfg = ACTConfig(input_features=input_features, output_features=output_features)
+policy = ACTPolicy(cfg)
+preprocessor, postprocessor = make_pre_post_processors(cfg, dataset_stats=dataset_metadata.stats)
 
-    policy.train()
-    policy.to(device)
+policy.train()
+policy.to(device)
 
-    # To perform action chunking, ACT expects a given number of actions as targets
-    delta_timestamps = {
-        "action": make_delta_timestamps(cfg.action_delta_indices, dataset_metadata.fps),
-    }
+# To perform action chunking, ACT expects a given number of actions as targets
+delta_timestamps = {
+    "action": make_delta_timestamps(cfg.action_delta_indices, dataset_metadata.fps),
+}
 
-    # add image features if they are present
-    delta_timestamps |= {
-        k: make_delta_timestamps(cfg.observation_delta_indices, dataset_metadata.fps)
-        for k in cfg.image_features
-    }
+# add image features if they are present
+delta_timestamps |= {
+    k: make_delta_timestamps(cfg.observation_delta_indices, dataset_metadata.fps) for k in cfg.image_features
+}
 
-    # Instantiate the dataset
-    dataset = LeRobotDataset(dataset_id, delta_timestamps=delta_timestamps)
+# Instantiate the dataset
+dataset = LeRobotDataset(dataset_id, delta_timestamps=delta_timestamps)
 
-    # Create the optimizer and dataloader for offline training
-    optimizer = cfg.get_optimizer_preset().build(policy.parameters())
-    batch_size = 32
-    dataloader = torch.utils.data.DataLoader(
-        dataset,
-        batch_size=batch_size,
-        shuffle=True,
-        pin_memory=device.type != "cpu",
-        drop_last=True,
-    )
+# Create the optimizer and dataloader for offline training
+optimizer = cfg.get_optimizer_preset().build(policy.parameters())
+batch_size = 32
+dataloader = torch.utils.data.DataLoader(
+    dataset,
+    batch_size=batch_size,
+    shuffle=True,
+    pin_memory=device.type != "cpu",
+    drop_last=True,
+)
 
-    # Number of training steps and logging frequency
-    training_steps = 1
-    log_freq = 1
+# Number of training steps and logging frequency
+training_steps = 1
+log_freq = 1
 
-    # Run training loop
-    step = 0
-    done = False
-    while not done:
-        for batch in dataloader:
-            batch = preprocessor(batch)
-            loss, _ = policy.forward(batch)
-            loss.backward()
-            optimizer.step()
-            optimizer.zero_grad()
+# Run training loop
+step = 0
+done = False
+while not done:
+    for batch in dataloader:
+        batch = preprocessor(batch)
+        loss, _ = policy.forward(batch)
+        loss.backward()
+        optimizer.step()
+        optimizer.zero_grad()
 
-            if step % log_freq == 0:
-                print(f"step: {step} loss: {loss.item():.3f}")
-            step += 1
-            if step >= training_steps:
-                done = True
-                break
+        if step % log_freq == 0:
+            print(f"step: {step} loss: {loss.item():.3f}")
+        step += 1
+        if step >= training_steps:
+            done = True
+            break
 
-    # Save the policy checkpoint, alongside the pre/post processors
-    policy.save_pretrained(output_directory)
-    preprocessor.save_pretrained(output_directory)
-    postprocessor.save_pretrained(output_directory)
+# Save the policy checkpoint, alongside the pre/post processors
+policy.save_pretrained(output_directory)
+preprocessor.save_pretrained(output_directory)
+postprocessor.save_pretrained(output_directory)
 
-    # Save all assets to the Hub
-    policy.push_to_hub("<user>/robot_learning_tutorial_act")
-    preprocessor.push_to_hub("<user>/robot_learning_tutorial_act")
-    postprocessor.push_to_hub("<user>/robot_learning_tutorial_act")
-
-
-if __name__ == "__main__":
-    main()
+# Save all assets to the Hub
+policy.push_to_hub("fracapuano/robot_learning_tutorial_act")
+preprocessor.push_to_hub("fracapuano/robot_learning_tutorial_act")
+postprocessor.push_to_hub("fracapuano/robot_learning_tutorial_act")

examples/tutorial/act/act_using_example.py

@@ -8,56 +8,50 @@ from lerobot.policies.utils import build_inference_frame, make_robot_action
 from lerobot.robots.so100_follower.config_so100_follower import SO100FollowerConfig
 from lerobot.robots.so100_follower.so100_follower import SO100Follower
 
+device = torch.device("mps")  # or "cuda" or "cpu"
+model_id = "fracapuano/robot_learning_tutorial_act"
+model = ACTPolicy.from_pretrained(model_id)
+
+dataset_id = "lerobot/svla_so101_pickplace"
+# This only downloads the metadata for the dataset, ~10s of MB even for large-scale datasets
+dataset_metadata = LeRobotDatasetMetadata(dataset_id)
+preprocess, postprocess = make_pre_post_processors(model.config, dataset_stats=dataset_metadata.stats)
+
+# # find ports using lerobot-find-port
+follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
+
+# # the robot ids are used the load the right calibration files
+follower_id = ...  # something like "follower_so100"
+
 MAX_EPISODES = 5
 MAX_STEPS_PER_EPISODE = 20
 
+# Robot and environment configuration
+# Camera keys must match the name and resolutions of the ones used for training!
+# You can check the camera keys expected by a model in the info.json card on the model card on the Hub
+camera_config = {
+    "side": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
+    "up": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
+}
 
-def main():
-    device = torch.device("mps")  # or "cuda" or "cpu"
-    model_id = "<user>/robot_learning_tutorial_act"
-    model = ACTPolicy.from_pretrained(model_id)
+robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
+robot = SO100Follower(robot_cfg)
+robot.connect()
 
-    dataset_id = "lerobot/svla_so101_pickplace"
-    # This only downloads the metadata for the dataset, ~10s of MB even for large-scale datasets
-    dataset_metadata = LeRobotDatasetMetadata(dataset_id)
-    preprocess, postprocess = make_pre_post_processors(model.config, dataset_stats=dataset_metadata.stats)
+for _ in range(MAX_EPISODES):
+    for _ in range(MAX_STEPS_PER_EPISODE):
+        obs = robot.get_observation()
+        obs_frame = build_inference_frame(
+            observation=obs, ds_features=dataset_metadata.features, device=device
+        )
 
-    # # find ports using lerobot-find-port
-    follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
+        obs = preprocess(obs_frame)
 
-    # # the robot ids are used the load the right calibration files
-    follower_id = ...  # something like "follower_so100"
+        action = model.select_action(obs)
+        action = postprocess(action)
 
-    # Robot and environment configuration
-    # Camera keys must match the name and resolutions of the ones used for training!
-    # You can check the camera keys expected by a model in the info.json card on the model card on the Hub
-    camera_config = {
-        "side": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
-        "up": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
-    }
+        action = make_robot_action(action, dataset_metadata.features)
 
-    robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
-    robot = SO100Follower(robot_cfg)
-    robot.connect()
+        robot.send_action(action)
 
-    for _ in range(MAX_EPISODES):
-        for _ in range(MAX_STEPS_PER_EPISODE):
-            obs = robot.get_observation()
-            obs_frame = build_inference_frame(
-                observation=obs, ds_features=dataset_metadata.features, device=device
-            )
-
-            obs = preprocess(obs_frame)
-
-            action = model.select_action(obs)
-            action = postprocess(action)
-
-            action = make_robot_action(action, dataset_metadata.features)
-
-            robot.send_action(action)
-
-        print("Episode finished! Starting new episode...")
-
-
-if __name__ == "__main__":
-    main()
+    print("Episode finished! Starting new episode...")

examples/tutorial/async-inf/policy_server.py

@@ -1,17 +1,11 @@
 from lerobot.async_inference.configs import PolicyServerConfig
 from lerobot.async_inference.policy_server import serve
 
+host = ...  # something like "127.0.0.1" if you're exposing to localhost
+port = ...  # something like 8080
 
-def main():
-    host = ...  # something like "127.0.0.1" if you're exposing to localhost
-    port = ...  # something like 8080
-
-    config = PolicyServerConfig(
-        host=host,
-        port=port,
-    )
-    serve(config)
-
-
-if __name__ == "__main__":
-    main()
+config = PolicyServerConfig(
+    host=host,
+    port=port,
+)
+serve(config)

examples/tutorial/async-inf/robot_client.py

@@ -6,56 +6,50 @@ from lerobot.async_inference.robot_client import RobotClient
 from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig
 from lerobot.robots.so100_follower import SO100FollowerConfig
 
+# these cameras must match the ones expected by the policy - find your cameras with lerobot-find-cameras
+# check the config.json on the Hub for the policy you are using to see the expected camera specs
+camera_cfg = {
+    "up": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
+    "side": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
+}
 
-def main():
-    # these cameras must match the ones expected by the policy - find your cameras with lerobot-find-cameras
-    # check the config.json on the Hub for the policy you are using to see the expected camera specs
-    camera_cfg = {
-        "up": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
-        "side": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
-    }
+# # find ports using lerobot-find-port
+follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
 
-    # # find ports using lerobot-find-port
-    follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
+# # the robot ids are used the load the right calibration files
+follower_id = ...  # something like "follower_so100"
 
-    # # the robot ids are used the load the right calibration files
-    follower_id = ...  # something like "follower_so100"
+robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_cfg)
 
-    robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_cfg)
+server_address = ...  # something like "127.0.0.1:8080" if using localhost
 
-    server_address = ...  # something like "127.0.0.1:8080" if using localhost
+# 3. Create client configuration
+client_cfg = RobotClientConfig(
+    robot=robot_cfg,
+    server_address=server_address,
+    policy_device="mps",
+    policy_type="act",
+    pretrained_name_or_path="fracapuano/robot_learning_tutorial_act",
+    chunk_size_threshold=0.5,  # g
+    actions_per_chunk=50,  # make sure this is less than the max actions of the policy
+)
 
-    # 3. Create client configuration
-    client_cfg = RobotClientConfig(
-        robot=robot_cfg,
-        server_address=server_address,
-        policy_device="mps",
-        policy_type="act",
-        pretrained_name_or_path="<user>/robot_learning_tutorial_act",
-        chunk_size_threshold=0.5,  # g
-        actions_per_chunk=50,  # make sure this is less than the max actions of the policy
-    )
+# 4. Create and start client
+client = RobotClient(client_cfg)
 
-    # 4. Create and start client
-    client = RobotClient(client_cfg)
+# 5. Provide a textual description of the task
+task = ...
 
-    # 5. Provide a textual description of the task
-    task = ...
+if client.start():
+    # Start action receiver thread
+    action_receiver_thread = threading.Thread(target=client.receive_actions, daemon=True)
+    action_receiver_thread.start()
 
-    if client.start():
-        # Start action receiver thread
-        action_receiver_thread = threading.Thread(target=client.receive_actions, daemon=True)
-        action_receiver_thread.start()
-
-        try:
-            # Run the control loop
-            client.control_loop(task)
-        except KeyboardInterrupt:
-            client.stop()
-            action_receiver_thread.join()
-            # (Optionally) plot the action queue size
-            visualize_action_queue_size(client.action_queue_size)
-
-
-if __name__ == "__main__":
-    main()
+    try:
+        # Run the control loop
+        client.control_loop(task)
+    except KeyboardInterrupt:
+        client.stop()
+        action_receiver_thread.join()
+        # (Optionally) plot the action queue size
+        visualize_action_queue_size(client.action_queue_size)

examples/tutorial/diffusion/diffusion_training_example.py

@@ -19,87 +19,81 @@ def make_delta_timestamps(delta_indices: list[int] | None, fps: int) -> list[flo
     return [i / fps for i in delta_indices]
 
 
-def main():
-    output_directory = Path("outputs/robot_learning_tutorial/diffusion")
-    output_directory.mkdir(parents=True, exist_ok=True)
+output_directory = Path("outputs/robot_learning_tutorial/diffusion")
+output_directory.mkdir(parents=True, exist_ok=True)
 
-    # Select your device
-    device = torch.device("mps")  # or "cuda" or "cpu"
+# Select your device
+device = torch.device("mps")  # or "cuda" or "cpu"
 
-    dataset_id = "lerobot/svla_so101_pickplace"
+dataset_id = "lerobot/svla_so101_pickplace"
 
-    # This specifies the inputs the model will be expecting and the outputs it will produce
-    dataset_metadata = LeRobotDatasetMetadata(dataset_id)
-    features = dataset_to_policy_features(dataset_metadata.features)
+# This specifies the inputs the model will be expecting and the outputs it will produce
+dataset_metadata = LeRobotDatasetMetadata(dataset_id)
+features = dataset_to_policy_features(dataset_metadata.features)
 
-    output_features = {key: ft for key, ft in features.items() if ft.type is FeatureType.ACTION}
-    input_features = {key: ft for key, ft in features.items() if key not in output_features}
+output_features = {key: ft for key, ft in features.items() if ft.type is FeatureType.ACTION}
+input_features = {key: ft for key, ft in features.items() if key not in output_features}
 
-    cfg = DiffusionConfig(input_features=input_features, output_features=output_features)
-    policy = DiffusionPolicy(cfg)
-    preprocessor, postprocessor = make_pre_post_processors(cfg, dataset_stats=dataset_metadata.stats)
+cfg = DiffusionConfig(input_features=input_features, output_features=output_features)
+policy = DiffusionPolicy(cfg)
+preprocessor, postprocessor = make_pre_post_processors(cfg, dataset_stats=dataset_metadata.stats)
 
-    policy.train()
-    policy.to(device)
+policy.train()
+policy.to(device)
 
-    # To perform action chunking, ACT expects a given number of actions as targets
-    delta_timestamps = {
-        "observation.state": make_delta_timestamps(cfg.observation_delta_indices, dataset_metadata.fps),
-        "action": make_delta_timestamps(cfg.action_delta_indices, dataset_metadata.fps),
-    }
+# To perform action chunking, ACT expects a given number of actions as targets
+delta_timestamps = {
+    "observation.state": make_delta_timestamps(cfg.observation_delta_indices, dataset_metadata.fps),
+    "action": make_delta_timestamps(cfg.action_delta_indices, dataset_metadata.fps),
+}
 
-    # add image features if they are present
-    delta_timestamps |= {
-        k: make_delta_timestamps(cfg.observation_delta_indices, dataset_metadata.fps)
-        for k in cfg.image_features
-    }
+# add image features if they are present
+delta_timestamps |= {
+    k: make_delta_timestamps(cfg.observation_delta_indices, dataset_metadata.fps) for k in cfg.image_features
+}
 
-    # Instantiate the dataset
-    dataset = LeRobotDataset(dataset_id, delta_timestamps=delta_timestamps)
+# Instantiate the dataset
+dataset = LeRobotDataset(dataset_id, delta_timestamps=delta_timestamps)
 
-    # Create the optimizer and dataloader for offline training
-    optimizer = cfg.get_optimizer_preset().build(policy.parameters())
-    batch_size = 32
-    dataloader = torch.utils.data.DataLoader(
-        dataset,
-        batch_size=batch_size,
-        shuffle=True,
-        pin_memory=device.type != "cpu",
-        drop_last=True,
-    )
+# Create the optimizer and dataloader for offline training
+optimizer = cfg.get_optimizer_preset().build(policy.parameters())
+batch_size = 32
+dataloader = torch.utils.data.DataLoader(
+    dataset,
+    batch_size=batch_size,
+    shuffle=True,
+    pin_memory=device.type != "cpu",
+    drop_last=True,
+)
 
-    # Number of training steps and logging frequency
-    training_steps = 1
-    log_freq = 1
+# Number of training steps and logging frequency
+training_steps = 1
+log_freq = 1
 
-    # Run training loop
-    step = 0
-    done = False
-    while not done:
-        for batch in dataloader:
-            batch = preprocessor(batch)
-            loss, _ = policy.forward(batch)
-            loss.backward()
-            optimizer.step()
-            optimizer.zero_grad()
+# Run training loop
+step = 0
+done = False
+while not done:
+    for batch in dataloader:
+        batch = preprocessor(batch)
+        loss, _ = policy.forward(batch)
+        loss.backward()
+        optimizer.step()
+        optimizer.zero_grad()
 
-            if step % log_freq == 0:
-                print(f"step: {step} loss: {loss.item():.3f}")
-            step += 1
-            if step >= training_steps:
-                done = True
-                break
+        if step % log_freq == 0:
+            print(f"step: {step} loss: {loss.item():.3f}")
+        step += 1
+        if step >= training_steps:
+            done = True
+            break
 
-    # Save the policy checkpoint, alongside the pre/post processors
-    policy.save_pretrained(output_directory)
-    preprocessor.save_pretrained(output_directory)
-    postprocessor.save_pretrained(output_directory)
+# Save the policy checkpoint, alongside the pre/post processors
+policy.save_pretrained(output_directory)
+preprocessor.save_pretrained(output_directory)
+postprocessor.save_pretrained(output_directory)
 
-    # Save all assets to the Hub
-    policy.push_to_hub("<user>/robot_learning_tutorial_diffusion")
-    preprocessor.push_to_hub("<user>/robot_learning_tutorial_diffusion")
-    postprocessor.push_to_hub("<user>/robot_learning_tutorial_diffusion")
-
-
-if __name__ == "__main__":
-    main()
+# Save all assets to the Hub
+policy.push_to_hub("fracapuano/robot_learning_tutorial_diffusion")
+preprocessor.push_to_hub("fracapuano/robot_learning_tutorial_diffusion")
+postprocessor.push_to_hub("fracapuano/robot_learning_tutorial_diffusion")

examples/tutorial/diffusion/diffusion_using_example.py

@@ -8,57 +8,53 @@ from lerobot.policies.utils import build_inference_frame, make_robot_action
 from lerobot.robots.so100_follower.config_so100_follower import SO100FollowerConfig
 from lerobot.robots.so100_follower.so100_follower import SO100Follower
 
+device = torch.device("mps")  # or "cuda" or "cpu"
+model_id = "fracapuano/robot_learning_tutorial_diffusion"
+
+model = DiffusionPolicy.from_pretrained(model_id)
+
+dataset_id = "lerobot/svla_so101_pickplace"
+# This only downloads the metadata for the dataset, ~10s of MB even for large-scale datasets
+dataset_metadata = LeRobotDatasetMetadata(dataset_id)
+preprocess, postprocess = make_pre_post_processors(
+    model.config, model_id, dataset_stats=dataset_metadata.stats
+)
+
 MAX_EPISODES = 5
 MAX_STEPS_PER_EPISODE = 20
 
 
-def main():
-    device = torch.device("mps")  # or "cuda" or "cpu"
-    model_id = "<user>/robot_learning_tutorial_diffusion"
+# # find ports using lerobot-find-port
+follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
 
-    model = DiffusionPolicy.from_pretrained(model_id)
+# # the robot ids are used the load the right calibration files
+follower_id = ...  # something like "follower_so100"
 
-    dataset_id = "lerobot/svla_so101_pickplace"
-    # This only downloads the metadata for the dataset, ~10s of MB even for large-scale datasets
-    dataset_metadata = LeRobotDatasetMetadata(dataset_id)
-    preprocess, postprocess = make_pre_post_processors(
-        model.config, model_id, dataset_stats=dataset_metadata.stats
-    )
+# Robot and environment configuration
+# Camera keys must match the name and resolutions of the ones used for training!
+# You can check the camera keys expected by a model in the info.json card on the model card on the Hub
+camera_config = {
+    "side": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
+    "up": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
+}
 
-    # # find ports using lerobot-find-port
-    follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
-
-    # # the robot ids are used the load the right calibration files
-    follower_id = ...  # something like "follower_so100"
-
-    # Robot and environment configuration
-    # Camera keys must match the name and resolutions of the ones used for training!
-    # You can check the camera keys expected by a model in the info.json card on the model card on the Hub
-    camera_config = {
-        "side": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
-        "up": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
-    }
-
-    robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
-    robot = SO100Follower(robot_cfg)
-    robot.connect()
-
-    for _ in range(MAX_EPISODES):
-        for _ in range(MAX_STEPS_PER_EPISODE):
-            obs = robot.get_observation()
-            obs_frame = build_inference_frame(
-                observation=obs, ds_features=dataset_metadata.features, device=device
-            )
-
-            obs = preprocess(obs_frame)
-
-            action = model.select_action(obs)
-            action = postprocess(action)
-            action = make_robot_action(action, dataset_metadata.features)
-            robot.send_action(action)
-
-        print("Episode finished! Starting new episode...")
+robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
+robot = SO100Follower(robot_cfg)
+robot.connect()
 
 
-if __name__ == "__main__":
-    main()
+for _ in range(MAX_EPISODES):
+    for _ in range(MAX_STEPS_PER_EPISODE):
+        obs = robot.get_observation()
+        obs_frame = build_inference_frame(
+            observation=obs, ds_features=dataset_metadata.features, device=device
+        )
+
+        obs = preprocess(obs_frame)
+
+        action = model.select_action(obs)
+        action = postprocess(action)
+        action = make_robot_action(action, dataset_metadata.features)
+        robot.send_action(action)
+
+    print("Episode finished! Starting new episode...")

examples/tutorial/pi0/using_pi0_example.py

@@ -11,63 +11,57 @@ from lerobot.robots.so100_follower.so100_follower import SO100Follower
 MAX_EPISODES = 5
 MAX_STEPS_PER_EPISODE = 20
 
+device = torch.device("mps")  # or "cuda" or "cpu"
+model_id = "lerobot/pi0_base"
 
-def main():
-    device = torch.device("mps")  # or "cuda" or "cpu"
-    model_id = "lerobot/pi0_base"
+model = PI0Policy.from_pretrained(model_id)
 
-    model = PI0Policy.from_pretrained(model_id)
+preprocess, postprocess = make_pre_post_processors(
+    model.config,
+    model_id,
+    # This overrides allows to run on MPS, otherwise defaults to CUDA (if available)
+    preprocessor_overrides={"device_processor": {"device": str(device)}},
+)
 
-    preprocess, postprocess = make_pre_post_processors(
-        model.config,
-        model_id,
-        # This overrides allows to run on MPS, otherwise defaults to CUDA (if available)
-        preprocessor_overrides={"device_processor": {"device": str(device)}},
-    )
+# find ports using lerobot-find-port
+follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
 
-    # find ports using lerobot-find-port
-    follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
+# the robot ids are used the load the right calibration files
+follower_id = ...  # something like "follower_so100"
 
-    # the robot ids are used the load the right calibration files
-    follower_id = ...  # something like "follower_so100"
+# Robot and environment configuration
+# Camera keys must match the name and resolutions of the ones used for training!
+# You can check the camera keys expected by a model in the info.json card on the model card on the Hub
+camera_config = {
+    "base_0_rgb": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
+    "left_wrist_0_rgb": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
+    "right_wrist_0_rgb": OpenCVCameraConfig(index_or_path=2, width=640, height=480, fps=30),
+}
 
-    # Robot and environment configuration
-    # Camera keys must match the name and resolutions of the ones used for training!
-    # You can check the camera keys expected by a model in the info.json card on the model card on the Hub
-    camera_config = {
-        "base_0_rgb": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
-        "left_wrist_0_rgb": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
-        "right_wrist_0_rgb": OpenCVCameraConfig(index_or_path=2, width=640, height=480, fps=30),
-    }
+robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
+robot = SO100Follower(robot_cfg)
+robot.connect()
 
-    robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
-    robot = SO100Follower(robot_cfg)
-    robot.connect()
+task = ""  # something like "pick the red block"
+robot_type = ""  # something like "so100_follower" for multi-embodiment datasets
 
-    task = ""  # something like "pick the red block"
-    robot_type = ""  # something like "so100_follower" for multi-embodiment datasets
+# This is used to match the raw observation keys to the keys expected by the policy
+action_features = hw_to_dataset_features(robot.action_features, "action")
+obs_features = hw_to_dataset_features(robot.observation_features, "observation")
+dataset_features = {**action_features, **obs_features}
 
-    # This is used to match the raw observation keys to the keys expected by the policy
-    action_features = hw_to_dataset_features(robot.action_features, "action")
-    obs_features = hw_to_dataset_features(robot.observation_features, "observation")
-    dataset_features = {**action_features, **obs_features}
+for _ in range(MAX_EPISODES):
+    for _ in range(MAX_STEPS_PER_EPISODE):
+        obs = robot.get_observation()
+        obs_frame = build_inference_frame(
+            observation=obs, ds_features=dataset_features, device=device, task=task, robot_type=robot_type
+        )
 
-    for _ in range(MAX_EPISODES):
-        for _ in range(MAX_STEPS_PER_EPISODE):
-            obs = robot.get_observation()
-            obs_frame = build_inference_frame(
-                observation=obs, ds_features=dataset_features, device=device, task=task, robot_type=robot_type
-            )
+        obs = preprocess(obs_frame)
 
-            obs = preprocess(obs_frame)
+        action = model.select_action(obs)
+        action = postprocess(action)
+        action = make_robot_action(action, dataset_features)
+        robot.send_action(action)
 
-            action = model.select_action(obs)
-            action = postprocess(action)
-            action = make_robot_action(action, dataset_features)
-            robot.send_action(action)
-
-        print("Episode finished! Starting new episode...")
-
-
-if __name__ == "__main__":
-    main()
+    print("Episode finished! Starting new episode...")

examples/tutorial/rl/hilserl_example.py

@@ -20,8 +20,6 @@ from lerobot.teleoperators.utils import TeleopEvents
 
 LOG_EVERY = 10
 SEND_EVERY = 10
-MAX_EPISODES = 5
-MAX_STEPS_PER_EPISODE = 20
 
 
 def run_learner(
@@ -225,123 +223,123 @@ def make_policy_obs(obs, device: torch.device = "cpu"):
     }
 
 
-def main():
-    """Main function - coordinates actor and learner processes."""
+"""Main function - coordinates actor and learner processes."""
 
-    device = "mps"  # or "cuda" or "cpu"
-    output_directory = Path("outputs/robot_learning_tutorial/hil_serl")
-    output_directory.mkdir(parents=True, exist_ok=True)
+device = "mps"  # or "cuda" or "cpu"
+output_directory = Path("outputs/robot_learning_tutorial/hil_serl")
+output_directory.mkdir(parents=True, exist_ok=True)
 
-    # find ports using lerobot-find-port
-    follower_port = ...
-    leader_port = ...
+# find ports using lerobot-find-port
+follower_port = ...
+leader_port = ...
 
-    # the robot ids are used the load the right calibration files
-    follower_id = ...
-    leader_id = ...
+# the robot ids are used the load the right calibration files
+follower_id = ...
+leader_id = ...
 
-    # A pretrained model (to be used in-distribution!)
-    reward_classifier_id = "<user>/reward_classifier_hil_serl_example"
-    reward_classifier = Classifier.from_pretrained(reward_classifier_id)
+# A pretrained model (to be used in-distribution!)
+reward_classifier_id = "fracapuano/reward_classifier_hil_serl_example"
+reward_classifier = Classifier.from_pretrained(reward_classifier_id)
 
-    reward_classifier.to(device)
-    reward_classifier.eval()
+reward_classifier.to(device)
+reward_classifier.eval()
 
-    # Robot and environment configuration
-    robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id)
-    teleop_cfg = SO100LeaderConfig(port=leader_port, id=leader_id)
-    processor_cfg = HILSerlProcessorConfig(control_mode="leader")
+MAX_EPISODES = 5
+MAX_STEPS_PER_EPISODE = 20
 
-    env_cfg = HILSerlRobotEnvConfig(robot=robot_cfg, teleop=teleop_cfg, processor=processor_cfg)
+# Robot and environment configuration
+robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id)
+teleop_cfg = SO100LeaderConfig(port=leader_port, id=leader_id)
+processor_cfg = HILSerlProcessorConfig(control_mode="leader")
 
-    # Create robot environment
-    env, teleop_device = make_robot_env(env_cfg)
+env_cfg = HILSerlRobotEnvConfig(robot=robot_cfg, teleop=teleop_cfg, processor=processor_cfg)
 
-    obs_features = hw_to_dataset_features(env.robot.observation_features, "observation")
-    action_features = hw_to_dataset_features(env.robot.action_features, "action")
+# Create robot environment
+env, teleop_device = make_robot_env(env_cfg)
 
-    # Create SAC policy for action selection
-    policy_cfg = SACConfig(
-        device=device,
-        input_features=obs_features,
-        output_features=action_features,
-    )
+obs_features = hw_to_dataset_features(env.robot.observation_features, "observation")
+action_features = hw_to_dataset_features(env.robot.action_features, "action")
 
-    policy_actor = SACPolicy(policy_cfg)
-    policy_learner = SACPolicy(policy_cfg)
+# Create SAC policy for action selection
+policy_cfg = SACConfig(
+    device=device,
+    input_features=obs_features,
+    output_features=action_features,
+)
 
-    demonstrations_repo_id = "lerobot/example_hil_serl_dataset"
-    offline_dataset = LeRobotDataset(repo_id=demonstrations_repo_id)
+policy_actor = SACPolicy(policy_cfg)
+policy_learner = SACPolicy(policy_cfg)
 
-    # Online buffer: initialized from scratch
-    online_replay_buffer = ReplayBuffer(device=device, state_keys=list(obs_features.keys()))
-    # Offline buffer: Created from dataset (pre-populated it with demonstrations)
-    offline_replay_buffer = ReplayBuffer.from_lerobot_dataset(
-        lerobot_dataset=offline_dataset, device=device, state_keys=list(obs_features.keys())
-    )
+demonstrations_repo_id = "lerobot/example_hil_serl_dataset"
+offline_dataset = LeRobotDataset(repo_id=demonstrations_repo_id)
 
-    # Create communication channels between learner and actor processes
-    transitions_queue = mp.Queue(maxsize=10)
-    parameters_queue = mp.Queue(maxsize=2)
-    shutdown_event = mp.Event()
+# Online buffer: initialized from scratch
+online_replay_buffer = ReplayBuffer(device=device, state_keys=list(obs_features.keys()))
+# Offline buffer: Created from dataset (pre-populated it with demonstrations)
+offline_replay_buffer = ReplayBuffer.from_lerobot_dataset(
+    lerobot_dataset=offline_dataset, device=device, state_keys=list(obs_features.keys())
+)
 
-    # Signal handler for graceful shutdown
-    def signal_handler(sig):
-        print(f"\nSignal {sig} received, shutting down...")
-        shutdown_event.set()
-
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    # Create processes
-    learner_process = mp.Process(
-        target=run_learner,
-        args=(
-            transitions_queue,
-            parameters_queue,
-            shutdown_event,
-            policy_learner,
-            online_replay_buffer,
-            offline_replay_buffer,
-        ),
-        kwargs={"device": device},  # can run on accelerated hardware for training
-    )
-
-    actor_process = mp.Process(
-        target=run_actor,
-        args=(
-            transitions_queue,
-            parameters_queue,
-            shutdown_event,
-            policy_actor,
-            reward_classifier,
-            env_cfg,
-            output_directory,
-        ),
-        kwargs={"device": "cpu"},  # actor is frozen, can run on CPU or accelerate for inference
-    )
-
-    learner_process.start()
-    actor_process.start()
-
-    try:
-        # Wait for actor to finish (it controls the episode loop)
-        actor_process.join()
-        shutdown_event.set()
-        learner_process.join(timeout=10)
-
-    except KeyboardInterrupt:
-        print("Main process interrupted")
-        shutdown_event.set()
-        actor_process.join(timeout=5)
-        learner_process.join(timeout=10)
-
-    finally:
-        if learner_process.is_alive():
-            learner_process.terminate()
-        if actor_process.is_alive():
-            actor_process.terminate()
+# Create communication channels between learner and actor processes
+transitions_queue = mp.Queue(maxsize=10)
+parameters_queue = mp.Queue(maxsize=2)
+shutdown_event = mp.Event()
 
 
-if __name__ == "__main__":
-    main()
+# Signal handler for graceful shutdown
+def signal_handler(sig):
+    print(f"\nSignal {sig} received, shutting down...")
+    shutdown_event.set()
+
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+# Create processes
+learner_process = mp.Process(
+    target=run_learner,
+    args=(
+        transitions_queue,
+        parameters_queue,
+        shutdown_event,
+        policy_learner,
+        online_replay_buffer,
+        offline_replay_buffer,
+    ),
+    kwargs={"device": device},  # can run on accelerated hardware for training
+)
+
+actor_process = mp.Process(
+    target=run_actor,
+    args=(
+        transitions_queue,
+        parameters_queue,
+        shutdown_event,
+        policy_actor,
+        reward_classifier,
+        env_cfg,
+        output_directory,
+    ),
+    kwargs={"device": "cpu"},  # actor is frozen, can run on CPU or accelerate for inference
+)
+
+learner_process.start()
+actor_process.start()
+
+try:
+    # Wait for actor to finish (it controls the episode loop)
+    actor_process.join()
+    shutdown_event.set()
+    learner_process.join(timeout=10)
+
+except KeyboardInterrupt:
+    print("Main process interrupted")
+    shutdown_event.set()
+    actor_process.join(timeout=5)
+    learner_process.join(timeout=10)
+
+finally:
+    if learner_process.is_alive():
+        learner_process.terminate()
+    if actor_process.is_alive():
+        actor_process.terminate()

examples/tutorial/rl/reward_classifier_example.py

@@ -4,64 +4,59 @@ from lerobot.datasets.lerobot_dataset import LeRobotDataset
 from lerobot.policies.factory import make_policy, make_pre_post_processors
 from lerobot.policies.sac.reward_model.configuration_classifier import RewardClassifierConfig
 
+# Device to use for training
+device = "mps"  # or "cuda", or "cpu"
 
-def main():
-    # Device to use for training
-    device = "mps"  # or "cuda", or "cpu"
+# Load the dataset used for training
+repo_id = "lerobot/example_hil_serl_dataset"
+dataset = LeRobotDataset(repo_id)
 
-    # Load the dataset used for training
-    repo_id = "lerobot/example_hil_serl_dataset"
-    dataset = LeRobotDataset(repo_id)
+# Configure the policy to extract features from the image frames
+camera_keys = dataset.meta.camera_keys
 
-    # Configure the policy to extract features from the image frames
-    camera_keys = dataset.meta.camera_keys
+config = RewardClassifierConfig(
+    num_cameras=len(camera_keys),
+    device=device,
+    # backbone model to extract features from the image frames
+    model_name="microsoft/resnet-18",
+)
 
-    config = RewardClassifierConfig(
-        num_cameras=len(camera_keys),
-        device=device,
-        # backbone model to extract features from the image frames
-        model_name="microsoft/resnet-18",
-    )
-
-    # Make policy, preprocessor, and optimizer
-    policy = make_policy(config, ds_meta=dataset.meta)
-    optimizer = config.get_optimizer_preset().build(policy.parameters())
-    preprocessor, _ = make_pre_post_processors(policy_cfg=config, dataset_stats=dataset.meta.stats)
-
-    classifier_id = "<user>/reward_classifier_hil_serl_example"
-
-    # Instantiate a dataloader
-    dataloader = torch.utils.data.DataLoader(dataset, batch_size=16, shuffle=True)
-
-    # Training loop
-    num_epochs = 5
-    for epoch in range(num_epochs):
-        total_loss = 0
-        total_accuracy = 0
-        for batch in dataloader:
-            # Preprocess the batch and move it to the correct device.
-            batch = preprocessor(batch)
-
-            # Forward pass
-            loss, output_dict = policy.forward(batch)
-
-            # Backward pass and optimization
-            optimizer.zero_grad()
-            loss.backward()
-            optimizer.step()
-
-            total_loss += loss.item()
-            total_accuracy += output_dict["accuracy"]
-
-        avg_loss = total_loss / len(dataloader)
-        avg_accuracy = total_accuracy / len(dataloader)
-        print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {avg_loss:.4f}, Accuracy: {avg_accuracy:.2f}%")
-
-    print("Training finished!")
-
-    # You can now save the trained policy.
-    policy.push_to_hub(classifier_id)
+# Make policy, preprocessor, and optimizer
+policy = make_policy(config, ds_meta=dataset.meta)
+optimizer = config.get_optimizer_preset().build(policy.parameters())
+preprocessor, _ = make_pre_post_processors(policy_cfg=config, dataset_stats=dataset.meta.stats)
 
 
-if __name__ == "__main__":
-    main()
+classifier_id = "fracapuano/reward_classifier_hil_serl_example"
+
+# Instantiate a dataloader
+dataloader = torch.utils.data.DataLoader(dataset, batch_size=16, shuffle=True)
+
+# Training loop
+num_epochs = 5
+for epoch in range(num_epochs):
+    total_loss = 0
+    total_accuracy = 0
+    for batch in dataloader:
+        # Preprocess the batch and move it to the correct device.
+        batch = preprocessor(batch)
+
+        # Forward pass
+        loss, output_dict = policy.forward(batch)
+
+        # Backward pass and optimization
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        total_loss += loss.item()
+        total_accuracy += output_dict["accuracy"]
+
+    avg_loss = total_loss / len(dataloader)
+    avg_accuracy = total_accuracy / len(dataloader)
+    print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {avg_loss:.4f}, Accuracy: {avg_accuracy:.2f}%")
+
+print("Training finished!")
+
+# You can now save the trained policy.
+policy.push_to_hub(classifier_id)

examples/tutorial/smolvla/using_smolvla_example.py

@@ -11,62 +11,56 @@ from lerobot.robots.so100_follower.so100_follower import SO100Follower
 MAX_EPISODES = 5
 MAX_STEPS_PER_EPISODE = 20
 
+device = torch.device("mps")  # or "cuda" or "cpu"
+model_id = "lerobot/smolvla_base"
 
-def main():
-    device = torch.device("mps")  # or "cuda" or "cpu"
-    model_id = "lerobot/smolvla_base"
+model = SmolVLAPolicy.from_pretrained(model_id)
 
-    model = SmolVLAPolicy.from_pretrained(model_id)
+preprocess, postprocess = make_pre_post_processors(
+    model.config,
+    model_id,
+    # This overrides allows to run on MPS, otherwise defaults to CUDA (if available)
+    preprocessor_overrides={"device_processor": {"device": str(device)}},
+)
 
-    preprocess, postprocess = make_pre_post_processors(
-        model.config,
-        model_id,
-        # This overrides allows to run on MPS, otherwise defaults to CUDA (if available)
-        preprocessor_overrides={"device_processor": {"device": str(device)}},
-    )
+# find ports using lerobot-find-port
+follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
 
-    # find ports using lerobot-find-port
-    follower_port = ...  # something like "/dev/tty.usbmodem58760431631"
+# the robot ids are used the load the right calibration files
+follower_id = ...  # something like "follower_so100"
 
-    # the robot ids are used the load the right calibration files
-    follower_id = ...  # something like "follower_so100"
+# Robot and environment configuration
+# Camera keys must match the name and resolutions of the ones used for training!
+# You can check the camera keys expected by a model in the info.json card on the model card on the Hub
+camera_config = {
+    "camera1": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
+    "camera2": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
+}
 
-    # Robot and environment configuration
-    # Camera keys must match the name and resolutions of the ones used for training!
-    # You can check the camera keys expected by a model in the info.json card on the model card on the Hub
-    camera_config = {
-        "camera1": OpenCVCameraConfig(index_or_path=0, width=640, height=480, fps=30),
-        "camera2": OpenCVCameraConfig(index_or_path=1, width=640, height=480, fps=30),
-    }
+robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
+robot = SO100Follower(robot_cfg)
+robot.connect()
 
-    robot_cfg = SO100FollowerConfig(port=follower_port, id=follower_id, cameras=camera_config)
-    robot = SO100Follower(robot_cfg)
-    robot.connect()
+task = ""  # something like "pick the red block"
+robot_type = ""  # something like "so100_follower" for multi-embodiment datasets
 
-    task = ""  # something like "pick the red block"
-    robot_type = ""  # something like "so100_follower" for multi-embodiment datasets
+# This is used to match the raw observation keys to the keys expected by the policy
+action_features = hw_to_dataset_features(robot.action_features, "action")
+obs_features = hw_to_dataset_features(robot.observation_features, "observation")
+dataset_features = {**action_features, **obs_features}
 
-    # This is used to match the raw observation keys to the keys expected by the policy
-    action_features = hw_to_dataset_features(robot.action_features, "action")
-    obs_features = hw_to_dataset_features(robot.observation_features, "observation")
-    dataset_features = {**action_features, **obs_features}
+for _ in range(MAX_EPISODES):
+    for _ in range(MAX_STEPS_PER_EPISODE):
+        obs = robot.get_observation()
+        obs_frame = build_inference_frame(
+            observation=obs, ds_features=dataset_features, device=device, task=task, robot_type=robot_type
+        )
 
-    for _ in range(MAX_EPISODES):
-        for _ in range(MAX_STEPS_PER_EPISODE):
-            obs = robot.get_observation()
-            obs_frame = build_inference_frame(
-                observation=obs, ds_features=dataset_features, device=device, task=task, robot_type=robot_type
-            )
+        obs = preprocess(obs_frame)
 
-            obs = preprocess(obs_frame)
+        action = model.select_action(obs)
+        action = postprocess(action)
+        action = make_robot_action(action, dataset_features)
+        robot.send_action(action)
 
-            action = model.select_action(obs)
-            action = postprocess(action)
-            action = make_robot_action(action, dataset_features)
-            robot.send_action(action)
-
-        print("Episode finished! Starting new episode...")
-
-
-if __name__ == "__main__":
-    main()
+    print("Episode finished! Starting new episode...")

pyproject.toml

@@ -25,7 +25,7 @@ discord = "https://discord.gg/s3KuuzsPFb"
 
 [project]
 name = "lerobot"
-version = "0.4.3"
+version = "0.4.1"
 description = "🤗 LeRobot: State-of-the-art Machine Learning for Real-World Robotics in Pytorch"
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -102,8 +102,10 @@ grpcio-dep = ["grpcio==1.73.1", "protobuf==6.31.0"] # TODO: Bumb dependency (com
 # Motors
 feetech = ["feetech-servo-sdk>=1.0.0,<2.0.0"]
 dynamixel = ["dynamixel-sdk>=3.7.31,<3.9.0"]
+damiao = ["python-can>=4.2.0,<5.0.0"]
 
 # Robots
+openarms = ["lerobot[damiao]"]
 gamepad = ["lerobot[pygame-dep]", "hidapi>=0.14.0,<0.15.0"]
 hopejr = ["lerobot[feetech]", "lerobot[pygame-dep]"]
 lekiwi = ["lerobot[feetech]", "pyzmq>=26.2.1,<28.0.0"]
@@ -129,7 +131,6 @@ groot = [
     "ninja>=1.11.1,<2.0.0",
     "flash-attn>=2.5.9,<3.0.0 ; sys_platform != 'darwin'"
 ]
-xvla = ["lerobot[transformers-dep]"]
 hilserl = ["lerobot[transformers-dep]", "gym-hil>=0.1.13,<0.2.0", "lerobot[grpcio-dep]", "lerobot[placo-dep]"]
 
 # Features
@@ -143,12 +144,13 @@ video_benchmark = ["scikit-image>=0.23.2,<0.26.0", "pandas>=2.2.2,<2.4.0"]
 # Simulation
 aloha = ["gym-aloha>=0.1.2,<0.2.0"]
 pusht = ["gym-pusht>=0.1.5,<0.2.0", "pymunk>=6.6.0,<7.0.0"] # TODO: Fix pymunk version in gym-pusht instead
-libero = ["lerobot[transformers-dep]", "hf-libero>=0.1.3,<0.2.0"]
+libero = ["lerobot[transformers-dep]", "libero @ git+https://github.com/huggingface/lerobot-libero.git@main#egg=libero"]
 metaworld = ["metaworld==3.0.0"]
 
 # All
 all = [
     "lerobot[dynamixel]",
+    "lerobot[openarms]",
     "lerobot[gamepad]",
     "lerobot[hopejr]",
     "lerobot[lekiwi]",
@@ -158,7 +160,6 @@ all = [
     "lerobot[pi]",
     "lerobot[smolvla]",
     # "lerobot[groot]", TODO(Steven): Gr00t requires specific installation instructions for flash-attn
-    "lerobot[xvla]",
     "lerobot[hilserl]",
     "lerobot[async]",
     "lerobot[dev]",

src/lerobot/configs/types.py

@@ -43,10 +43,3 @@ class NormalizationMode(str, Enum):
 class PolicyFeature:
     type: FeatureType
     shape: tuple[int, ...]
-
-
-class RTCAttentionSchedule(str, Enum):
-    ZEROS = "ZEROS"
-    ONES = "ONES"
-    LINEAR = "LINEAR"
-    EXP = "EXP"

src/lerobot/datasets/dataset_tools.py

@@ -39,7 +39,6 @@ from lerobot.datasets.aggregate import aggregate_datasets
 from lerobot.datasets.compute_stats import aggregate_stats
 from lerobot.datasets.lerobot_dataset import LeRobotDataset, LeRobotDatasetMetadata
 from lerobot.datasets.utils import (
-    DATA_DIR,
     DEFAULT_CHUNK_SIZE,
     DEFAULT_DATA_FILE_SIZE_IN_MB,
     DEFAULT_DATA_PATH,
@@ -963,23 +962,28 @@ def _copy_data_with_feature_changes(
     remove_features: list[str] | None = None,
 ) -> None:
     """Copy data while adding or removing features."""
-    data_dir = dataset.root / DATA_DIR
-    parquet_files = sorted(data_dir.glob("*/*.parquet"))
+    if dataset.meta.episodes is None:
+        dataset.meta.episodes = load_episodes(dataset.meta.root)
 
-    if not parquet_files:
-        raise ValueError(f"No parquet files found in {data_dir}")
+    # Map file paths to episode indices to extract chunk/file indices
+    file_to_episodes: dict[Path, set[int]] = {}
+    for ep_idx in range(dataset.meta.total_episodes):
+        file_path = dataset.meta.get_data_file_path(ep_idx)
+        if file_path not in file_to_episodes:
+            file_to_episodes[file_path] = set()
+        file_to_episodes[file_path].add(ep_idx)
 
     frame_idx = 0
 
-    for src_path in tqdm(parquet_files, desc="Processing data files"):
-        df = pd.read_parquet(src_path).reset_index(drop=True)
+    for src_path in tqdm(sorted(file_to_episodes.keys()), desc="Processing data files"):
+        df = pd.read_parquet(dataset.root / src_path).reset_index(drop=True)
 
-        relative_path = src_path.relative_to(dataset.root)
-        chunk_dir = relative_path.parts[1]
-        file_name = relative_path.parts[2]
-
-        chunk_idx = int(chunk_dir.split("-")[1])
-        file_idx = int(file_name.split("-")[1].split(".")[0])
+        # Get chunk_idx and file_idx from the source file's first episode
+        episodes_in_file = file_to_episodes[src_path]
+        first_ep_idx = min(episodes_in_file)
+        src_ep = dataset.meta.episodes[first_ep_idx]
+        chunk_idx = src_ep["data/chunk_index"]
+        file_idx = src_ep["data/file_index"]
 
         if remove_features:
             df = df.drop(columns=remove_features, errors="ignore")
@@ -1005,7 +1009,7 @@ def _copy_data_with_feature_changes(
                         df[feature_name] = feature_slice
             frame_idx = end_idx
 
-        # Write using the same chunk/file structure as source
+        # Write using the preserved chunk_idx and file_idx from source
         dst_path = new_meta.root / DEFAULT_DATA_PATH.format(chunk_index=chunk_idx, file_index=file_idx)
         dst_path.parent.mkdir(parents=True, exist_ok=True)
 

src/lerobot/datasets/image_writer.py

@@ -110,8 +110,8 @@ def worker_thread_loop(queue: queue.Queue):
         if item is None:
             queue.task_done()
             break
-        image_array, fpath, compress_level = item
-        write_image(image_array, fpath, compress_level)
+        image_array, fpath = item
+        write_image(image_array, fpath)
         queue.task_done()
 
 
@@ -169,13 +169,11 @@ class AsyncImageWriter:
                 p.start()
                 self.processes.append(p)
 
-    def save_image(
-        self, image: torch.Tensor | np.ndarray | PIL.Image.Image, fpath: Path, compress_level: int = 1
-    ):
+    def save_image(self, image: torch.Tensor | np.ndarray | PIL.Image.Image, fpath: Path):
         if isinstance(image, torch.Tensor):
             # Convert tensor to numpy array to minimize main process time
             image = image.cpu().numpy()
-        self.queue.put((image, fpath, compress_level))
+        self.queue.put((image, fpath))
 
     def wait_until_done(self):
         self.queue.join()

src/lerobot/datasets/lerobot_dataset.py

@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import concurrent.futures
 import contextlib
 import logging
 import shutil
@@ -431,7 +430,9 @@ class LeRobotDatasetMetadata:
         video_keys = [video_key] if video_key is not None else self.video_keys
         for key in video_keys:
             if not self.features[key].get("info", None):
-                video_path = self.root / self.video_path.format(video_key=key, chunk_index=0, file_index=0)
+                video_path = self.root / self.video_path.format(
+                    video_key=video_key, chunk_index=0, file_index=0
+                )
                 self.info["features"][key]["info"] = get_video_info(video_path)
 
     def update_chunk_settings(
@@ -540,15 +541,6 @@ class LeRobotDatasetMetadata:
         return obj
 
 
-def _encode_video_worker(video_key: str, episode_index: int, root: Path, fps: int) -> Path:
-    temp_path = Path(tempfile.mkdtemp(dir=root)) / f"{video_key}_{episode_index:03d}.mp4"
-    fpath = DEFAULT_IMAGE_PATH.format(image_key=video_key, episode_index=episode_index, frame_index=0)
-    img_dir = (root / fpath).parent
-    encode_video_frames(img_dir, temp_path, fps, overwrite=True)
-    shutil.rmtree(img_dir)
-    return temp_path
-
-
 class LeRobotDataset(torch.utils.data.Dataset):
     def __init__(
         self,
@@ -722,15 +714,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
             self.download(download_videos)
             self.hf_dataset = self.load_hf_dataset()
 
-        # Create mapping from absolute indices to relative indices when only a subset of the episodes are loaded
-        # Build a mapping: absolute_index -> relative_index_in_filtered_dataset
-        self._absolute_to_relative_idx = None
-        if self.episodes is not None:
-            self._absolute_to_relative_idx = {
-                abs_idx.item() if isinstance(abs_idx, torch.Tensor) else abs_idx: rel_idx
-                for rel_idx, abs_idx in enumerate(self.hf_dataset["index"])
-            }
-
         # Setup delta_indices
         if self.delta_timestamps is not None:
             check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s)
@@ -849,7 +832,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
     def load_hf_dataset(self) -> datasets.Dataset:
         """hf_dataset contains all the observations, states, actions, rewards, etc."""
         features = get_hf_features_from_features(self.features)
-        hf_dataset = load_nested_dataset(self.root / "data", features=features, episodes=self.episodes)
+        hf_dataset = load_nested_dataset(self.root / "data", features=features)
         hf_dataset.set_transform(hf_transform_to_torch)
         return hf_dataset
 
@@ -866,8 +849,10 @@ class LeRobotDataset(torch.utils.data.Dataset):
 
         # Determine requested episodes
         if self.episodes is None:
+            # Requesting all episodes - check if we have all episodes from metadata
             requested_episodes = set(range(self.meta.total_episodes))
         else:
+            # Requesting specific episodes
             requested_episodes = set(self.episodes)
 
         # Check if all requested episodes are available in cached data
@@ -949,11 +934,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
         query_timestamps = {}
         for key in self.meta.video_keys:
             if query_indices is not None and key in query_indices:
-                if self._absolute_to_relative_idx is not None:
-                    relative_indices = [self._absolute_to_relative_idx[idx] for idx in query_indices[key]]
-                    timestamps = self.hf_dataset[relative_indices]["timestamp"]
-                else:
-                    timestamps = self.hf_dataset[query_indices[key]]["timestamp"]
+                timestamps = self.hf_dataset[query_indices[key]]["timestamp"]
                 query_timestamps[key] = torch.stack(timestamps).tolist()
             else:
                 query_timestamps[key] = [current_ts]
@@ -961,32 +942,11 @@ class LeRobotDataset(torch.utils.data.Dataset):
         return query_timestamps
 
     def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict:
-        """
-        Query dataset for indices across keys, skipping video keys.
-
-        Tries column-first [key][indices] for speed, falls back to row-first.
-
-        Args:
-            query_indices: Dict mapping keys to index lists to retrieve
-
-        Returns:
-            Dict with stacked tensors of queried data (video keys excluded)
-        """
-        result: dict = {}
-        for key, q_idx in query_indices.items():
-            if key in self.meta.video_keys:
-                continue
-            # Map absolute indices to relative indices if needed
-            relative_indices = (
-                q_idx
-                if self._absolute_to_relative_idx is None
-                else [self._absolute_to_relative_idx[idx] for idx in q_idx]
-            )
-            try:
-                result[key] = torch.stack(self.hf_dataset[key][relative_indices])
-            except (KeyError, TypeError, IndexError):
-                result[key] = torch.stack(self.hf_dataset[relative_indices][key])
-        return result
+        return {
+            key: torch.stack(self.hf_dataset[q_idx][key])
+            for key, q_idx in query_indices.items()
+            if key not in self.meta.video_keys
+        }
 
     def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]:
         """Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
@@ -1081,7 +1041,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
             ep_buffer[key] = current_ep_idx if key == "episode_index" else []
         return ep_buffer
 
-    # TODO(Steven): consider move this to utils
     def _get_image_file_path(self, episode_index: int, image_key: str, frame_index: int) -> Path:
         fpath = DEFAULT_IMAGE_PATH.format(
             image_key=image_key, episode_index=episode_index, frame_index=frame_index
@@ -1091,15 +1050,13 @@ class LeRobotDataset(torch.utils.data.Dataset):
     def _get_image_file_dir(self, episode_index: int, image_key: str) -> Path:
         return self._get_image_file_path(episode_index, image_key, frame_index=0).parent
 
-    def _save_image(
-        self, image: torch.Tensor | np.ndarray | PIL.Image.Image, fpath: Path, compress_level: int = 1
-    ) -> None:
+    def _save_image(self, image: torch.Tensor | np.ndarray | PIL.Image.Image, fpath: Path) -> None:
         if self.image_writer is None:
             if isinstance(image, torch.Tensor):
                 image = image.cpu().numpy()
-            write_image(image, fpath, compress_level=compress_level)
+            write_image(image, fpath)
         else:
-            self.image_writer.save_image(image=image, fpath=fpath, compress_level=compress_level)
+            self.image_writer.save_image(image=image, fpath=fpath)
 
     def add_frame(self, frame: dict) -> None:
         """
@@ -1137,19 +1094,14 @@ class LeRobotDataset(torch.utils.data.Dataset):
                 )
                 if frame_index == 0:
                     img_path.parent.mkdir(parents=True, exist_ok=True)
-                compress_level = 1 if self.features[key]["dtype"] == "video" else 6
-                self._save_image(frame[key], img_path, compress_level)
+                self._save_image(frame[key], img_path)
                 self.episode_buffer[key].append(str(img_path))
             else:
                 self.episode_buffer[key].append(frame[key])
 
         self.episode_buffer["size"] += 1
 
-    def save_episode(
-        self,
-        episode_data: dict | None = None,
-        parallel_encoding: bool = True,
-    ) -> None:
+    def save_episode(self, episode_data: dict | None = None) -> None:
         """
         This will save to disk the current episode in self.episode_buffer.
 
@@ -1161,8 +1113,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
             episode_data (dict | None, optional): Dict containing the episode data to save. If None, this will
                 save the current episode in self.episode_buffer, which is filled with 'add_frame'. Defaults to
                 None.
-            parallel_encoding (bool, optional): If True, encode videos in parallel using ProcessPoolExecutor.
-                Defaults to True on Linux, False on macOS as it tends to use all the CPU available already.
         """
         episode_buffer = episode_data if episode_data is not None else self.episode_buffer
 
@@ -1199,40 +1149,8 @@ class LeRobotDataset(torch.utils.data.Dataset):
         use_batched_encoding = self.batch_encoding_size > 1
 
         if has_video_keys and not use_batched_encoding:
-            num_cameras = len(self.meta.video_keys)
-            if parallel_encoding and num_cameras > 1:
-                # TODO(Steven): Ideally we would like to control the number of threads per encoding such that:
-                # num_cameras * num_threads = (total_cpu -1)
-                with concurrent.futures.ProcessPoolExecutor(max_workers=num_cameras) as executor:
-                    future_to_key = {
-                        executor.submit(
-                            _encode_video_worker,
-                            video_key,
-                            episode_index,
-                            self.root,
-                            self.fps,
-                        ): video_key
-                        for video_key in self.meta.video_keys
-                    }
-
-                    results = {}
-                    for future in concurrent.futures.as_completed(future_to_key):
-                        video_key = future_to_key[future]
-                        try:
-                            temp_path = future.result()
-                            results[video_key] = temp_path
-                        except Exception as exc:
-                            logging.error(f"Video encoding failed for {video_key}: {exc}")
-                            raise exc
-
-                for video_key in self.meta.video_keys:
-                    temp_path = results[video_key]
-                    ep_metadata.update(
-                        self._save_episode_video(video_key, episode_index, temp_path=temp_path)
-                    )
-            else:
-                for video_key in self.meta.video_keys:
-                    ep_metadata.update(self._save_episode_video(video_key, episode_index))
+            for video_key in self.meta.video_keys:
+                ep_metadata.update(self._save_episode_video(video_key, episode_index))
 
         # `meta.save_episode` need to be executed after encoding the videos
         self.meta.save_episode(episode_index, episode_length, episode_tasks, ep_stats, ep_metadata)
@@ -1397,18 +1315,9 @@ class LeRobotDataset(torch.utils.data.Dataset):
 
         return metadata
 
-    def _save_episode_video(
-        self,
-        video_key: str,
-        episode_index: int,
-        temp_path: Path | None = None,
-    ) -> dict:
+    def _save_episode_video(self, video_key: str, episode_index: int) -> dict:
         # Encode episode frames into a temporary video
-        if temp_path is None:
-            ep_path = self._encode_temporary_episode_video(video_key, episode_index)
-        else:
-            ep_path = temp_path
-
+        ep_path = self._encode_temporary_episode_video(video_key, episode_index)
         ep_size_in_mb = get_file_size_in_mb(ep_path)
         ep_duration_in_s = get_video_duration_in_s(ep_path)
 
@@ -1526,7 +1435,11 @@ class LeRobotDataset(torch.utils.data.Dataset):
         Note: `encode_video_frames` is a blocking call. Making it asynchronous shouldn't speedup encoding,
         since video encoding with ffmpeg is already using multithreading.
         """
-        return _encode_video_worker(video_key, episode_index, self.root, self.fps)
+        temp_path = Path(tempfile.mkdtemp(dir=self.root)) / f"{video_key}_{episode_index:03d}.mp4"
+        img_dir = self._get_image_file_dir(episode_index, video_key)
+        encode_video_frames(img_dir, temp_path, self.fps, overwrite=True)
+        shutil.rmtree(img_dir)
+        return temp_path
 
     @classmethod
     def create(
@@ -1572,7 +1485,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
         obj.image_transforms = None
         obj.delta_timestamps = None
         obj.delta_indices = None
-        obj._absolute_to_relative_idx = None
         obj.video_backend = video_backend if video_backend is not None else get_safe_default_codec()
         obj.writer = None
         obj.latest_episode = None

src/lerobot/datasets/utils.py

@@ -28,7 +28,6 @@ import numpy as np
 import packaging.version
 import pandas
 import pandas as pd
-import pyarrow.dataset as pa_ds
 import pyarrow.parquet as pq
 import torch
 from datasets import Dataset
@@ -49,7 +48,7 @@ from lerobot.utils.utils import SuppressProgressBars, is_valid_numpy_dtype_strin
 
 DEFAULT_CHUNK_SIZE = 1000  # Max number of files per chunk
 DEFAULT_DATA_FILE_SIZE_IN_MB = 100  # Max size per file
-DEFAULT_VIDEO_FILE_SIZE_IN_MB = 200  # Max size per file
+DEFAULT_VIDEO_FILE_SIZE_IN_MB = 500  # Max size per file
 
 INFO_PATH = "meta/info.json"
 STATS_PATH = "meta/stats.json"
@@ -104,9 +103,7 @@ def update_chunk_file_indices(chunk_idx: int, file_idx: int, chunks_size: int) -
     return chunk_idx, file_idx
 
 
-def load_nested_dataset(
-    pq_dir: Path, features: datasets.Features | None = None, episodes: list[int] | None = None
-) -> Dataset:
+def load_nested_dataset(pq_dir: Path, features: datasets.Features | None = None) -> Dataset:
     """Find parquet files in provided directory {pq_dir}/chunk-xxx/file-xxx.parquet
     Convert parquet files to pyarrow memory mapped in a cache folder for efficient RAM usage
     Concatenate all pyarrow references to return HF Dataset format
@@ -114,26 +111,15 @@ def load_nested_dataset(
     Args:
         pq_dir: Directory containing parquet files
         features: Optional features schema to ensure consistent loading of complex types like images
-        episodes: Optional list of episode indices to filter. Uses PyArrow predicate pushdown for efficiency.
     """
     paths = sorted(pq_dir.glob("*/*.parquet"))
     if len(paths) == 0:
         raise FileNotFoundError(f"Provided directory does not contain any parquet file: {pq_dir}")
 
+    # TODO(rcadene): set num_proc to accelerate conversion to pyarrow
     with SuppressProgressBars():
-        # When no filtering needed, Dataset uses memory-mapped loading for efficiency
-        # PyArrow loads the entire dataset into memory
-        if episodes is None:
-            return Dataset.from_parquet([str(path) for path in paths], features=features)
-
-        arrow_dataset = pa_ds.dataset(paths, format="parquet")
-        filter_expr = pa_ds.field("episode_index").isin(episodes)
-        table = arrow_dataset.to_table(filter=filter_expr)
-
-        if features is not None:
-            table = table.cast(features.arrow_schema)
-
-        return Dataset(table)
+        datasets = Dataset.from_parquet([str(path) for path in paths], features=features)
+    return datasets
 
 
 def get_parquet_num_frames(parquet_path: str | Path) -> int:

src/lerobot/datasets/video_utils.py

@@ -311,7 +311,6 @@ def encode_video_frames(
     fast_decode: int = 0,
     log_level: int | None = av.logging.ERROR,
     overwrite: bool = False,
-    preset: int | None = None,
 ) -> None:
     """More info on ffmpeg arguments tuning on `benchmark/video/README.md`"""
     # Check encoder availability
@@ -360,9 +359,6 @@ def encode_video_frames(
         value = f"fast-decode={fast_decode}" if vcodec == "libsvtav1" else "fastdecode"
         video_options[key] = value
 
-    if vcodec == "libsvtav1":
-        video_options["preset"] = str(preset) if preset is not None else "12"
-
     # Set logging level
     if log_level is not None:
         # "While less efficient, it is generally preferable to modify logging with Python's logging"

src/lerobot/envs/configs.py

@@ -21,22 +21,7 @@ import draccus
 from lerobot.configs.types import FeatureType, PolicyFeature
 from lerobot.robots import RobotConfig
 from lerobot.teleoperators.config import TeleoperatorConfig
-from lerobot.utils.constants import (
-    ACTION,
-    LIBERO_KEY_EEF_MAT,
-    LIBERO_KEY_EEF_POS,
-    LIBERO_KEY_EEF_QUAT,
-    LIBERO_KEY_GRIPPER_QPOS,
-    LIBERO_KEY_GRIPPER_QVEL,
-    LIBERO_KEY_JOINTS_POS,
-    LIBERO_KEY_JOINTS_VEL,
-    LIBERO_KEY_PIXELS_AGENTVIEW,
-    LIBERO_KEY_PIXELS_EYE_IN_HAND,
-    OBS_ENV_STATE,
-    OBS_IMAGE,
-    OBS_IMAGES,
-    OBS_STATE,
-)
+from lerobot.utils.constants import ACTION, OBS_ENV_STATE, OBS_IMAGE, OBS_IMAGES, OBS_STATE
 
 
 @dataclass
@@ -245,7 +230,7 @@ class HILSerlRobotEnvConfig(EnvConfig):
 class LiberoEnv(EnvConfig):
     task: str = "libero_10"  # can also choose libero_spatial, libero_object, etc.
     fps: int = 30
-    episode_length: int | None = None
+    episode_length: int = 520
     obs_type: str = "pixels_agent_pos"
     render_mode: str = "rgb_array"
     camera_name: str = "agentview_image,robot0_eye_in_hand_image"
@@ -261,62 +246,28 @@ class LiberoEnv(EnvConfig):
     features_map: dict[str, str] = field(
         default_factory=lambda: {
             ACTION: ACTION,
-            LIBERO_KEY_EEF_POS: f"{OBS_STATE}.eef_pos",
-            LIBERO_KEY_EEF_QUAT: f"{OBS_STATE}.eef_quat",
-            LIBERO_KEY_EEF_MAT: f"{OBS_STATE}.eef_mat",
-            LIBERO_KEY_GRIPPER_QPOS: f"{OBS_STATE}.gripper_qpos",
-            LIBERO_KEY_GRIPPER_QVEL: f"{OBS_STATE}.gripper_qvel",
-            LIBERO_KEY_JOINTS_POS: f"{OBS_STATE}.joint_pos",
-            LIBERO_KEY_JOINTS_VEL: f"{OBS_STATE}.joint_vel",
-            LIBERO_KEY_PIXELS_AGENTVIEW: f"{OBS_IMAGES}.image",
-            LIBERO_KEY_PIXELS_EYE_IN_HAND: f"{OBS_IMAGES}.image2",
+            "agent_pos": OBS_STATE,
+            "pixels/agentview_image": f"{OBS_IMAGES}.image",
+            "pixels/robot0_eye_in_hand_image": f"{OBS_IMAGES}.image2",
         }
     )
-    control_mode: str = "relative"  # or "absolute"
 
     def __post_init__(self):
         if self.obs_type == "pixels":
-            self.features[LIBERO_KEY_PIXELS_AGENTVIEW] = PolicyFeature(
+            self.features["pixels/agentview_image"] = PolicyFeature(
                 type=FeatureType.VISUAL, shape=(self.observation_height, self.observation_width, 3)
             )
-            self.features[LIBERO_KEY_PIXELS_EYE_IN_HAND] = PolicyFeature(
+            self.features["pixels/robot0_eye_in_hand_image"] = PolicyFeature(
                 type=FeatureType.VISUAL, shape=(self.observation_height, self.observation_width, 3)
             )
         elif self.obs_type == "pixels_agent_pos":
-            self.features[LIBERO_KEY_PIXELS_AGENTVIEW] = PolicyFeature(
+            self.features["agent_pos"] = PolicyFeature(type=FeatureType.STATE, shape=(8,))
+            self.features["pixels/agentview_image"] = PolicyFeature(
                 type=FeatureType.VISUAL, shape=(self.observation_height, self.observation_width, 3)
             )
-            self.features[LIBERO_KEY_PIXELS_EYE_IN_HAND] = PolicyFeature(
+            self.features["pixels/robot0_eye_in_hand_image"] = PolicyFeature(
                 type=FeatureType.VISUAL, shape=(self.observation_height, self.observation_width, 3)
             )
-            self.features[LIBERO_KEY_EEF_POS] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(3,),
-            )
-            self.features[LIBERO_KEY_EEF_QUAT] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(4,),
-            )
-            self.features[LIBERO_KEY_EEF_MAT] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(3, 3),
-            )
-            self.features[LIBERO_KEY_GRIPPER_QPOS] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(2,),
-            )
-            self.features[LIBERO_KEY_GRIPPER_QVEL] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(2,),
-            )
-            self.features[LIBERO_KEY_JOINTS_POS] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(7,),
-            )
-            self.features[LIBERO_KEY_JOINTS_VEL] = PolicyFeature(
-                type=FeatureType.STATE,
-                shape=(7,),
-            )
         else:
             raise ValueError(f"Unsupported obs_type: {self.obs_type}")
 

src/lerobot/envs/factory.py

@@ -14,18 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import importlib
-from typing import Any
 
 import gymnasium as gym
 from gymnasium.envs.registration import registry as gym_registry
 
-from lerobot.configs.policies import PreTrainedConfig
 from lerobot.envs.configs import AlohaEnv, EnvConfig, LiberoEnv, PushtEnv
-from lerobot.envs.utils import _call_make_env, _download_hub_file, _import_hub_module, _normalize_hub_result
-from lerobot.policies.xvla.configuration_xvla import XVLAConfig
-from lerobot.processor import ProcessorStep
-from lerobot.processor.env_processor import LiberoProcessorStep
-from lerobot.processor.pipeline import PolicyProcessorPipeline
 
 
 def make_env_config(env_type: str, **kwargs) -> EnvConfig:
@@ -39,65 +32,16 @@ def make_env_config(env_type: str, **kwargs) -> EnvConfig:
         raise ValueError(f"Policy type '{env_type}' is not available.")
 
 
-def make_env_pre_post_processors(
-    env_cfg: EnvConfig,
-    policy_cfg: PreTrainedConfig,
-) -> tuple[
-    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
-    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
-]:
-    """
-    Create preprocessor and postprocessor pipelines for environment observations.
-
-    This function creates processor pipelines that transform raw environment
-    observations and actions. By default, it returns identity processors that do nothing.
-    For specific environments like LIBERO, it adds environment-specific processing steps.
-
-    Args:
-        env_cfg: The configuration of the environment.
-
-    Returns:
-        A tuple containing:
-            - preprocessor: Pipeline that processes environment observations
-            - postprocessor: Pipeline that processes environment outputs (currently identity)
-    """
-    # Preprocessor and Postprocessor steps are Identity for most environments
-    preprocessor_steps: list[ProcessorStep] = []
-    postprocessor_steps: list[ProcessorStep] = []
-    if isinstance(policy_cfg, XVLAConfig):
-        from lerobot.policies.xvla.processor_xvla import make_xvla_libero_pre_post_processors
-
-        return make_xvla_libero_pre_post_processors()
-
-    # For LIBERO environments, add the LiberoProcessorStep to preprocessor
-    if isinstance(env_cfg, LiberoEnv) or "libero" in env_cfg.type:
-        preprocessor_steps.append(LiberoProcessorStep())
-
-    preprocessor = PolicyProcessorPipeline(steps=preprocessor_steps)
-    postprocessor = PolicyProcessorPipeline(steps=postprocessor_steps)
-
-    return preprocessor, postprocessor
-
-
 def make_env(
-    cfg: EnvConfig | str,
-    n_envs: int = 1,
-    use_async_envs: bool = False,
-    hub_cache_dir: str | None = None,
-    trust_remote_code: bool = False,
+    cfg: EnvConfig, n_envs: int = 1, use_async_envs: bool = False
 ) -> dict[str, dict[int, gym.vector.VectorEnv]]:
-    """Makes a gym vector environment according to the config or Hub reference.
+    """Makes a gym vector environment according to the config.
 
     Args:
-        cfg (EnvConfig | str): Either an `EnvConfig` object describing the environment to build locally,
-            or a Hugging Face Hub repository identifier (e.g. `"username/repo"`). In the latter case,
-            the repo must include a Python file (usually `env.py`).
+        cfg (EnvConfig): the config of the environment to instantiate.
         n_envs (int, optional): The number of parallelized env to return. Defaults to 1.
         use_async_envs (bool, optional): Whether to return an AsyncVectorEnv or a SyncVectorEnv. Defaults to
             False.
-        hub_cache_dir (str | None): Optional cache path for downloaded hub files.
-        trust_remote_code (bool): **Explicit consent** to execute remote code from the Hub.
-            Default False — must be set to True to import/exec hub `env.py`.
 
     Raises:
         ValueError: if n_envs < 1
@@ -110,21 +54,6 @@ def make_env(
             - For single-task environments: a single suite entry (cfg.type) with task_id=0.
 
     """
-    # if user passed a hub id string (e.g., "username/repo", "username/repo@main:env.py")
-    # simplified: only support hub-provided `make_env`
-    if isinstance(cfg, str):
-        # _download_hub_file will raise the same RuntimeError if trust_remote_code is False
-        repo_id, file_path, local_file, revision = _download_hub_file(cfg, trust_remote_code, hub_cache_dir)
-
-        # import and surface clear import errors
-        module = _import_hub_module(local_file, repo_id)
-
-        # call the hub-provided make_env
-        raw_result = _call_make_env(module, n_envs=n_envs, use_async_envs=use_async_envs)
-
-        # normalize the return into {suite: {task_id: vec_env}}
-        return _normalize_hub_result(raw_result)
-
     if n_envs < 1:
         raise ValueError("`n_envs` must be at least 1")
 
@@ -143,8 +72,6 @@ def make_env(
             init_states=cfg.init_states,
             gym_kwargs=cfg.gym_kwargs,
             env_cls=env_cls,
-            control_mode=cfg.control_mode,
-            episode_length=cfg.episode_length,
         )
     elif "metaworld" in cfg.type:
         from lerobot.envs.metaworld import create_metaworld_envs

src/lerobot/envs/libero.py

@@ -28,6 +28,7 @@ import torch
 from gymnasium import spaces
 from libero.libero import benchmark, get_libero_path
 from libero.libero.envs import OffScreenRenderEnv
+from robosuite.utils.transform_utils import quat2axisangle
 
 
 def _parse_camera_names(camera_name: str | Sequence[str]) -> list[str]:
@@ -80,7 +81,10 @@ def get_libero_dummy_action():
     return [0, 0, 0, 0, 0, 0, -1]
 
 
+OBS_STATE_DIM = 8
 ACTION_DIM = 7
+AGENT_POS_LOW = -1000.0
+AGENT_POS_HIGH = 1000.0
 ACTION_LOW = -1.0
 ACTION_HIGH = 1.0
 TASK_SUITE_MAX_STEPS: dict[str, int] = {
@@ -100,7 +104,6 @@ class LiberoEnv(gym.Env):
         task_suite: Any,
         task_id: int,
         task_suite_name: str,
-        episode_length: int | None = None,
         camera_name: str | Sequence[str] = "agentview_image,robot0_eye_in_hand_image",
         obs_type: str = "pixels",
         render_mode: str = "rgb_array",
@@ -112,7 +115,6 @@ class LiberoEnv(gym.Env):
         episode_index: int = 0,
         camera_name_mapping: dict[str, str] | None = None,
         num_steps_wait: int = 10,
-        control_mode: str = "relative",
     ):
         super().__init__()
         self.task_id = task_id
@@ -140,19 +142,14 @@ class LiberoEnv(gym.Env):
         self.camera_name_mapping = camera_name_mapping
         self.num_steps_wait = num_steps_wait
         self.episode_index = episode_index
-        self.episode_length = episode_length
         # Load once and keep
         self._init_states = get_task_init_states(task_suite, self.task_id) if self.init_states else None
         self._init_state_id = self.episode_index  # tie each sub-env to a fixed init state
 
         self._env = self._make_envs_task(task_suite, self.task_id)
         default_steps = 500
-        self._max_episode_steps = (
-            TASK_SUITE_MAX_STEPS.get(task_suite_name, default_steps)
-            if self.episode_length is None
-            else self.episode_length
-        )
-        self.control_mode = control_mode
+        self._max_episode_steps = TASK_SUITE_MAX_STEPS.get(task_suite_name, default_steps)
+
         images = {}
         for cam in self.camera_name:
             images[self.camera_name_mapping[cam]] = spaces.Box(
@@ -178,36 +175,11 @@ class LiberoEnv(gym.Env):
             self.observation_space = spaces.Dict(
                 {
                     "pixels": spaces.Dict(images),
-                    "robot_state": spaces.Dict(
-                        {
-                            "eef": spaces.Dict(
-                                {
-                                    "pos": spaces.Box(low=-np.inf, high=np.inf, shape=(3,), dtype=np.float64),
-                                    "quat": spaces.Box(
-                                        low=-np.inf, high=np.inf, shape=(4,), dtype=np.float64
-                                    ),
-                                    "mat": spaces.Box(
-                                        low=-np.inf, high=np.inf, shape=(3, 3), dtype=np.float64
-                                    ),
-                                }
-                            ),
-                            "gripper": spaces.Dict(
-                                {
-                                    "qpos": spaces.Box(
-                                        low=-np.inf, high=np.inf, shape=(2,), dtype=np.float64
-                                    ),
-                                    "qvel": spaces.Box(
-                                        low=-np.inf, high=np.inf, shape=(2,), dtype=np.float64
-                                    ),
-                                }
-                            ),
-                            "joints": spaces.Dict(
-                                {
-                                    "pos": spaces.Box(low=-np.inf, high=np.inf, shape=(7,), dtype=np.float64),
-                                    "vel": spaces.Box(low=-np.inf, high=np.inf, shape=(7,), dtype=np.float64),
-                                }
-                            ),
-                        }
+                    "agent_pos": spaces.Box(
+                        low=AGENT_POS_LOW,
+                        high=AGENT_POS_HIGH,
+                        shape=(OBS_STATE_DIM,),
+                        dtype=np.float64,
                     ),
                 }
             )
@@ -219,7 +191,6 @@ class LiberoEnv(gym.Env):
     def render(self):
         raw_obs = self._env.env._get_observations()
         image = self._format_raw_obs(raw_obs)["pixels"]["image"]
-        image = image[::-1, ::-1]  # flip both H and W for visualization
         return image
 
     def _make_envs_task(self, task_suite: Any, task_id: int = 0):
@@ -241,48 +212,23 @@ class LiberoEnv(gym.Env):
         images = {}
         for camera_name in self.camera_name:
             image = raw_obs[camera_name]
+            image = image[::-1, ::-1]  # rotate 180 degrees
             images[self.camera_name_mapping[camera_name]] = image
-
-        eef_pos = raw_obs.get("robot0_eef_pos")
-        eef_quat = raw_obs.get("robot0_eef_quat")
-
-        # rotation matrix from controller
-        eef_mat = self._env.robots[0].controller.ee_ori_mat if eef_pos is not None else None
-        gripper_qpos = raw_obs.get("robot0_gripper_qpos")
-        gripper_qvel = raw_obs.get("robot0_gripper_qvel")
-        joint_pos = raw_obs.get("robot0_joint_pos")
-        joint_vel = raw_obs.get("robot0_joint_vel")
-        obs = {
-            "pixels": images,
-            "robot_state": {
-                "eef": {
-                    "pos": eef_pos,  # (3,)
-                    "quat": eef_quat,  # (4,)
-                    "mat": eef_mat,  # (3, 3)
-                },
-                "gripper": {
-                    "qpos": gripper_qpos,  # (2,)
-                    "qvel": gripper_qvel,  # (2,)
-                },
-                "joints": {
-                    "pos": joint_pos,  # (7,)
-                    "vel": joint_vel,  # (7,)
-                },
-            },
-        }
+        state = np.concatenate(
+            (
+                raw_obs["robot0_eef_pos"],
+                quat2axisangle(raw_obs["robot0_eef_quat"]),
+                raw_obs["robot0_gripper_qpos"],
+            )
+        )
+        agent_pos = state
         if self.obs_type == "pixels":
             return {"pixels": images.copy()}
-
         if self.obs_type == "pixels_agent_pos":
-            # Validate required fields are present
-            if eef_pos is None or eef_quat is None or gripper_qpos is None:
-                raise ValueError(
-                    f"Missing required robot state fields in raw observation. "
-                    f"Got eef_pos={eef_pos is not None}, eef_quat={eef_quat is not None}, "
-                    f"gripper_qpos={gripper_qpos is not None}"
-                )
-            return obs
-
+            return {
+                "pixels": images.copy(),
+                "agent_pos": agent_pos,
+            }
         raise NotImplementedError(
             f"The observation type '{self.obs_type}' is not supported in LiberoEnv. "
             "Please switch to an image-based obs_type (e.g. 'pixels', 'pixels_agent_pos')."
@@ -300,15 +246,6 @@ class LiberoEnv(gym.Env):
         # Increasing this value can improve determinism and reproducibility across resets.
         for _ in range(self.num_steps_wait):
             raw_obs, _, _, _ = self._env.step(get_libero_dummy_action())
-
-        if self.control_mode == "absolute":
-            for robot in self._env.robots:
-                robot.controller.use_delta = False
-        elif self.control_mode == "relative":
-            for robot in self._env.robots:
-                robot.controller.use_delta = True
-        else:
-            raise ValueError(f"Invalid control mode: {self.control_mode}")
         observation = self._format_raw_obs(raw_obs)
         info = {"is_success": False}
         return observation, info
@@ -354,10 +291,8 @@ def _make_env_fns(
     task_id: int,
     n_envs: int,
     camera_names: list[str],
-    episode_length: int | None,
     init_states: bool,
     gym_kwargs: Mapping[str, Any],
-    control_mode: str,
 ) -> list[Callable[[], LiberoEnv]]:
     """Build n_envs factory callables for a single (suite, task_id)."""
 
@@ -369,9 +304,7 @@ def _make_env_fns(
             task_suite_name=suite_name,
             camera_name=camera_names,
             init_states=init_states,
-            episode_length=episode_length,
             episode_index=episode_index,
-            control_mode=control_mode,
             **local_kwargs,
         )
 
@@ -391,8 +324,6 @@ def create_libero_envs(
     camera_name: str | Sequence[str] = "agentview_image,robot0_eye_in_hand_image",
     init_states: bool = True,
     env_cls: Callable[[Sequence[Callable[[], Any]]], Any] | None = None,
-    control_mode: str = "relative",
-    episode_length: int | None = None,
 ) -> dict[str, dict[int, Any]]:
     """
     Create vectorized LIBERO environments with a consistent return shape.
@@ -424,24 +355,24 @@ def create_libero_envs(
         print(f"Restricting to task_ids={task_ids_filter}")
 
     out: dict[str, dict[int, Any]] = defaultdict(dict)
+
     for suite_name in suite_names:
         suite = _get_suite(suite_name)
         total = len(suite.tasks)
         selected = _select_task_ids(total, task_ids_filter)
+
         if not selected:
             raise ValueError(f"No tasks selected for suite '{suite_name}' (available: {total}).")
 
         for tid in selected:
             fns = _make_env_fns(
                 suite=suite,
-                episode_length=episode_length,
                 suite_name=suite_name,
                 task_id=tid,
                 n_envs=n_envs,
                 camera_names=camera_names,
                 init_states=init_states,
                 gym_kwargs=gym_kwargs,
-                control_mode=control_mode,
             )
             out[suite_name][tid] = env_cls(fns)
             print(f"Built vec env | suite={suite_name} | task_id={tid} | n_envs={n_envs}")

src/lerobot/envs/utils.py

@@ -13,8 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import importlib.util
-import os
 import warnings
 from collections.abc import Mapping, Sequence
 from functools import singledispatch
@@ -24,27 +22,14 @@ import einops
 import gymnasium as gym
 import numpy as np
 import torch
-from huggingface_hub import hf_hub_download, snapshot_download
 from torch import Tensor
 
 from lerobot.configs.types import FeatureType, PolicyFeature
 from lerobot.envs.configs import EnvConfig
-from lerobot.utils.constants import OBS_ENV_STATE, OBS_IMAGE, OBS_IMAGES, OBS_STATE, OBS_STR
+from lerobot.utils.constants import OBS_ENV_STATE, OBS_IMAGE, OBS_IMAGES, OBS_STATE
 from lerobot.utils.utils import get_channel_first_image_shape
 
 
-def _convert_nested_dict(d):
-    result = {}
-    for k, v in d.items():
-        if isinstance(v, dict):
-            result[k] = _convert_nested_dict(v)
-        elif isinstance(v, np.ndarray):
-            result[k] = torch.from_numpy(v)
-        else:
-            result[k] = v
-    return result
-
-
 def preprocess_observation(observations: dict[str, np.ndarray]) -> dict[str, Tensor]:
     # TODO(aliberts, rcadene): refactor this to use features from the environment (no hardcoding)
     """Convert environment observation to LeRobot format observation.
@@ -90,14 +75,12 @@ def preprocess_observation(observations: dict[str, np.ndarray]) -> dict[str, Ten
 
         return_observations[OBS_ENV_STATE] = env_state
 
-    if "agent_pos" in observations:
-        agent_pos = torch.from_numpy(observations["agent_pos"]).float()
-        if agent_pos.dim() == 1:
-            agent_pos = agent_pos.unsqueeze(0)
-        return_observations[OBS_STATE] = agent_pos
+    # TODO(rcadene): enable pixels only baseline with `obs_type="pixels"` in environment by removing
+    agent_pos = torch.from_numpy(observations["agent_pos"]).float()
+    if agent_pos.dim() == 1:
+        agent_pos = agent_pos.unsqueeze(0)
+    return_observations[OBS_STATE] = agent_pos
 
-    if "robot_state" in observations:
-        return_observations[f"{OBS_STR}.robot_state"] = _convert_nested_dict(observations["robot_state"])
     return return_observations
 
 
@@ -212,132 +195,3 @@ def _(envs: Sequence) -> None:
 @close_envs.register
 def _(env: gym.Env) -> None:
     _close_single_env(env)
-
-
-# helper to safely load a python file as a module
-def _load_module_from_path(path: str, module_name: str | None = None):
-    module_name = module_name or f"hub_env_{os.path.basename(path).replace('.', '_')}"
-    spec = importlib.util.spec_from_file_location(module_name, path)
-    if spec is None:
-        raise ImportError(f"Could not load module spec for {module_name} from {path}")
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)  # type: ignore
-    return module
-
-
-# helper to parse hub string (supports "user/repo", "user/repo@rev", optional path)
-# examples:
-#   "user/repo" -> will look for env.py at repo root
-#   "user/repo@main:envs/my_env.py" -> explicit revision and path
-def _parse_hub_url(hub_uri: str):
-    # very small parser: [repo_id][@revision][:path]
-    # repo_id is required (user/repo or org/repo)
-    revision = None
-    file_path = "env.py"
-    if "@" in hub_uri:
-        repo_and_rev, *rest = hub_uri.split(":", 1)
-        repo_id, rev = repo_and_rev.split("@", 1)
-        revision = rev
-        if rest:
-            file_path = rest[0]
-    else:
-        repo_id, *rest = hub_uri.split(":", 1)
-        if rest:
-            file_path = rest[0]
-    return repo_id, revision, file_path
-
-
-def _download_hub_file(
-    cfg_str: str,
-    trust_remote_code: bool,
-    hub_cache_dir: str | None,
-) -> tuple[str, str, str, str]:
-    """
-    Parse `cfg_str` (hub URL), enforce `trust_remote_code`, and return
-    (repo_id, file_path, local_file, revision).
-    """
-    if not trust_remote_code:
-        raise RuntimeError(
-            f"Refusing to execute remote code from the Hub for '{cfg_str}'. "
-            "Executing hub env modules runs arbitrary Python code from third-party repositories. "
-            "If you trust this repo and understand the risks, call `make_env(..., trust_remote_code=True)` "
-            "and prefer pinning to a specific revision: 'user/repo@<commit-hash>:env.py'."
-        )
-
-    repo_id, revision, file_path = _parse_hub_url(cfg_str)
-
-    try:
-        local_file = hf_hub_download(
-            repo_id=repo_id, filename=file_path, revision=revision, cache_dir=hub_cache_dir
-        )
-    except Exception as e:
-        # fallback to snapshot download
-        snapshot_dir = snapshot_download(repo_id=repo_id, revision=revision, cache_dir=hub_cache_dir)
-        local_file = os.path.join(snapshot_dir, file_path)
-        if not os.path.exists(local_file):
-            raise FileNotFoundError(
-                f"Could not find {file_path} in repository {repo_id}@{revision or 'main'}"
-            ) from e
-
-    return repo_id, file_path, local_file, revision
-
-
-def _import_hub_module(local_file: str, repo_id: str) -> Any:
-    """
-    Import the downloaded file as a module and surface helpful import error messages.
-    """
-    module_name = f"hub_env_{repo_id.replace('/', '_')}"
-    try:
-        module = _load_module_from_path(local_file, module_name=module_name)
-    except ModuleNotFoundError as e:
-        missing = getattr(e, "name", None) or str(e)
-        raise ModuleNotFoundError(
-            f"Hub env '{repo_id}:{os.path.basename(local_file)}' failed to import because the dependency "
-            f"'{missing}' is not installed locally.\n\n"
-        ) from e
-    except ImportError as e:
-        raise ImportError(
-            f"Failed to load hub env module '{repo_id}:{os.path.basename(local_file)}'. Import error: {e}\n\n"
-        ) from e
-    return module
-
-
-def _call_make_env(module: Any, n_envs: int, use_async_envs: bool) -> Any:
-    """
-    Ensure module exposes make_env and call it.
-    """
-    if not hasattr(module, "make_env"):
-        raise AttributeError(
-            f"The hub module {getattr(module, '__name__', 'hub_module')} must expose `make_env(n_envs=int, use_async_envs=bool)`."
-        )
-    entry_fn = module.make_env
-    return entry_fn(n_envs=n_envs, use_async_envs=use_async_envs)
-
-
-def _normalize_hub_result(result: Any) -> dict[str, dict[int, gym.vector.VectorEnv]]:
-    """
-    Normalize possible return types from hub `make_env` into the mapping:
-      { suite_name: { task_id: vector_env } }
-    Accepts:
-      - dict (assumed already correct)
-      - gym.vector.VectorEnv
-      - gym.Env (will be wrapped into SyncVectorEnv)
-    """
-    if isinstance(result, dict):
-        return result
-
-    # VectorEnv: use its spec.id if available
-    if isinstance(result, gym.vector.VectorEnv):
-        suite_name = getattr(result, "spec", None) and getattr(result.spec, "id", None) or "hub_env"
-        return {suite_name: {0: result}}
-
-    # Single Env: wrap into SyncVectorEnv
-    if isinstance(result, gym.Env):
-        vec = gym.vector.SyncVectorEnv([lambda: result])
-        suite_name = getattr(result, "spec", None) and getattr(result.spec, "id", None) or "hub_env"
-        return {suite_name: {0: vec}}
-
-    raise ValueError(
-        "Hub `make_env` must return either a mapping {suite: {task_id: vec_env}}, "
-        "a gym.vector.VectorEnv, or a single gym.Env."
-    )

src/lerobot/motors/__init__.py

@@ -14,4 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .motors_bus import Motor, MotorCalibration, MotorNormMode, MotorsBus
+from .motors_bus import (
+    Motor,
+    MotorCalibration,
+    MotorNormMode,
+    MotorsBus,  # Backward compatibility (alias for SerialMotorsBus)
+    MotorsBusBase,
+    SerialMotorsBus,
+)

src/lerobot/motors/damiao/__init__.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .damiao import DamiaoMotorsBus
+from .tables import *

src/lerobot/motors/damiao/damiao.py

@@ -0,0 +1,787 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO(pepijn): add license of: https://github.com/cmjang/DM_Control_Python?tab=MIT-1-ov-file#readme
+
+import logging
+import time
+from contextlib import contextmanager
+from copy import deepcopy
+from functools import cached_property
+from typing import Dict, List, Optional, Tuple, Union
+
+import can
+import numpy as np
+
+from lerobot.motors import Motor, MotorCalibration, MotorNormMode, MotorsBusBase
+from lerobot.utils.errors import DeviceAlreadyConnectedError, DeviceNotConnectedError
+from lerobot.utils.utils import enter_pressed, move_cursor_up
+
+from .tables import (
+    AVAILABLE_BAUDRATES,
+    CAN_CMD_DISABLE,
+    CAN_CMD_ENABLE,
+    CAN_CMD_REFRESH,
+    CAN_CMD_SET_ZERO,
+    CAN_PARAM_ID,
+    DEFAULT_BAUDRATE,
+    DEFAULT_TIMEOUT_MS,
+    MODEL_RESOLUTION,
+    MOTOR_LIMIT_PARAMS,
+    NORMALIZED_DATA,
+    MotorType,
+)
+
+logger = logging.getLogger(__name__)
+
+NameOrID = Union[str, int]
+Value = Union[int, float]
+
+
+class DamiaoMotorsBus(MotorsBusBase):
+    """
+    The Damiao implementation for a MotorsBus using CAN bus communication.
+    
+    This class uses python-can for CAN bus communication with Damiao motors.
+    For more info, see:
+    - python-can documentation: https://python-can.readthedocs.io/en/stable/
+    - Seedstudio documentation: https://wiki.seeedstudio.com/damiao_series/
+    - DM_Control_Python repo: https://github.com/cmjang/DM_Control_Python
+    """
+
+    # CAN-specific settings
+    available_baudrates = deepcopy(AVAILABLE_BAUDRATES)
+    default_baudrate = DEFAULT_BAUDRATE
+    default_timeout = DEFAULT_TIMEOUT_MS
+    
+    # Motor configuration
+    model_resolution_table = deepcopy(MODEL_RESOLUTION)
+    normalized_data = deepcopy(NORMALIZED_DATA)
+
+    def __init__(
+        self,
+        port: str,
+        motors: dict[str, Motor],
+        calibration: dict[str, MotorCalibration] | None = None,
+        can_interface: str = "auto",
+        use_can_fd: bool = True,
+        bitrate: int = 1000000,
+        data_bitrate: int | None = 5000000,
+    ):
+        """
+        Initialize the Damiao motors bus.
+
+        Args:
+            port: CAN interface name (e.g., "can0" for Linux, "/dev/cu.usbmodem*" for macOS)
+            motors: Dictionary mapping motor names to Motor objects
+            calibration: Optional calibration data
+            can_interface: CAN interface type - "auto" (default), "socketcan" (Linux), or "slcan" (macOS/serial)
+            use_can_fd: Whether to use CAN FD mode (default: True for OpenArms)
+            bitrate: Nominal bitrate in bps (default: 1000000 = 1 Mbps)
+            data_bitrate: Data bitrate for CAN FD in bps (default: 5000000 = 5 Mbps), ignored if use_can_fd is False
+        """
+        super().__init__(port, motors, calibration)
+        self.port = port
+        self.can_interface = can_interface
+        self.use_can_fd = use_can_fd
+        self.bitrate = bitrate
+        self.data_bitrate = data_bitrate
+        self.canbus = None
+        self._is_connected = False
+        
+        # Map motor names to CAN IDs
+        self._motor_can_ids = {}
+        self._recv_id_to_motor = {}
+        
+        # Store motor types and recv IDs
+        self._motor_types = {}
+        for name, motor in self.motors.items():
+            if hasattr(motor, "motor_type"):
+                self._motor_types[name] = motor.motor_type
+            else:
+                # Default to DM4310 if not specified
+                self._motor_types[name] = MotorType.DM4310
+            
+            # Map recv_id to motor name for filtering responses
+            if hasattr(motor, "recv_id"):
+                self._recv_id_to_motor[motor.recv_id] = name
+
+    @property
+    def is_connected(self) -> bool:
+        """Check if the CAN bus is connected."""
+        return self._is_connected and self.canbus is not None
+
+    def connect(self, handshake: bool = True) -> None:
+        """
+        Open the CAN bus and initialize communication.
+
+        Args:
+            handshake: If True, ping all motors to verify they're present
+        """
+        if self.is_connected:
+            raise DeviceAlreadyConnectedError(
+                f"{self.__class__.__name__}('{self.port}') is already connected."
+            )
+
+        try:
+            # Auto-detect interface type based on port name
+            if self.can_interface == "auto":
+                if self.port.startswith("/dev/"):
+                    # Serial device (macOS/Windows)
+                    self.can_interface = "slcan"
+                    logger.info(f"Auto-detected slcan interface for port {self.port}")
+                else:
+                    # Network interface (Linux)
+                    self.can_interface = "socketcan"
+                    logger.info(f"Auto-detected socketcan interface for port {self.port}")
+            
+            # Connect to CAN bus
+            if self.can_interface == "socketcan":
+                # Linux SocketCAN with CAN FD support
+                if self.use_can_fd and self.data_bitrate is not None:
+                    self.canbus = can.interface.Bus(
+                        channel=self.port,
+                        interface="socketcan",
+                        bitrate=self.bitrate,
+                        data_bitrate=self.data_bitrate,
+                        fd=True
+                    )
+                    logger.info(f"Connected to {self.port} with CAN FD (bitrate={self.bitrate}, data_bitrate={self.data_bitrate})")
+                else:
+                    self.canbus = can.interface.Bus(
+                        channel=self.port,
+                        interface="socketcan",
+                        bitrate=self.bitrate
+                    )
+                    logger.info(f"Connected to {self.port} with CAN 2.0 (bitrate={self.bitrate})")
+            elif self.can_interface == "slcan":
+                # Serial Line CAN (macOS, Windows, or USB adapters)
+                # Note: SLCAN typically doesn't support CAN FD
+                self.canbus = can.interface.Bus(
+                    channel=self.port,
+                    interface="slcan",
+                    bitrate=self.bitrate
+                )
+                logger.info(f"Connected to {self.port} with SLCAN (bitrate={self.bitrate})")
+            else:
+                # Generic interface (vector, pcan, etc.)
+                if self.use_can_fd and self.data_bitrate is not None:
+                    self.canbus = can.interface.Bus(
+                        channel=self.port,
+                        interface=self.can_interface,
+                        bitrate=self.bitrate,
+                        data_bitrate=self.data_bitrate,
+                        fd=True
+                    )
+                else:
+                    self.canbus = can.interface.Bus(
+                        channel=self.port,
+                        interface=self.can_interface,
+                        bitrate=self.bitrate
+                    )
+            
+            self._is_connected = True
+            
+            if handshake:
+                self._handshake()
+                
+            logger.debug(f"{self.__class__.__name__} connected via {self.can_interface}.")
+        except Exception as e:
+            self._is_connected = False
+            raise ConnectionError(f"Failed to connect to CAN bus: {e}")
+
+    def _handshake(self) -> None:
+        """Verify all motors are present by refreshing their status."""
+        for motor_name in self.motors:
+            self._refresh_motor(motor_name)
+            time.sleep(0.01)  # Small delay between motors
+
+    def disconnect(self, disable_torque: bool = True) -> None:
+        """
+        Close the CAN bus connection.
+
+        Args:
+            disable_torque: If True, disable torque on all motors before disconnecting
+        """
+        if not self.is_connected:
+            raise DeviceNotConnectedError(
+                f"{self.__class__.__name__}('{self.port}') is not connected."
+            )
+
+        if disable_torque:
+            try:
+                self.disable_torque()
+            except Exception as e:
+                logger.warning(f"Failed to disable torque during disconnect: {e}")
+
+        if self.canbus:
+            self.canbus.shutdown()
+            self.canbus = None
+        self._is_connected = False
+        logger.debug(f"{self.__class__.__name__} disconnected.")
+
+    def configure_motors(self) -> None:
+        """Configure all motors with default settings."""
+        # Damiao motors don't require much configuration in MIT mode
+        # Just ensure they're enabled
+        for motor in self.motors:
+            self._enable_motor(motor)
+            time.sleep(0.01)
+
+    def _enable_motor(self, motor: NameOrID) -> None:
+        """Enable a single motor."""
+        motor_id = self._get_motor_id(motor)
+        recv_id = self._get_motor_recv_id(motor)
+        data = [0xFF] * 7 + [CAN_CMD_ENABLE]
+        msg = can.Message(arbitration_id=motor_id, data=data, is_extended_id=False)
+        self.canbus.send(msg)
+        self._recv_motor_response(expected_recv_id=recv_id)
+
+    def _disable_motor(self, motor: NameOrID) -> None:
+        """Disable a single motor."""
+        motor_id = self._get_motor_id(motor)
+        recv_id = self._get_motor_recv_id(motor)
+        data = [0xFF] * 7 + [CAN_CMD_DISABLE]
+        msg = can.Message(arbitration_id=motor_id, data=data, is_extended_id=False)
+        self.canbus.send(msg)
+        self._recv_motor_response(expected_recv_id=recv_id)
+
+    def enable_torque(self, motors: str | list[str] | None = None, num_retry: int = 0) -> None:
+        """Enable torque on selected motors."""
+        motors = self._get_motors_list(motors)
+        for motor in motors:
+            for _ in range(num_retry + 1):
+                try:
+                    self._enable_motor(motor)
+                    break
+                except Exception as e:
+                    if _ == num_retry:
+                        raise e
+                    time.sleep(0.01)
+
+    def disable_torque(self, motors: str | list[str] | None = None, num_retry: int = 0) -> None:
+        """Disable torque on selected motors."""
+        motors = self._get_motors_list(motors)
+        for motor in motors:
+            for _ in range(num_retry + 1):
+                try:
+                    self._disable_motor(motor)
+                    break
+                except Exception as e:
+                    if _ == num_retry:
+                        raise e
+                    time.sleep(0.01)
+
+    @contextmanager
+    def torque_disabled(self, motors: str | list[str] | None = None):
+        """
+        Context manager that guarantees torque is re-enabled.
+        
+        This helper is useful to temporarily disable torque when configuring motors.
+        
+        Examples:
+            >>> with bus.torque_disabled():
+            ...     # Safe operations here with torque disabled
+            ...     pass
+        """
+        self.disable_torque(motors)
+        try:
+            yield
+        finally:
+            self.enable_torque(motors)
+
+    def set_zero_position(self, motors: str | list[str] | None = None) -> None:
+        """Set current position as zero for selected motors."""
+        motors = self._get_motors_list(motors)
+        for motor in motors:
+            motor_id = self._get_motor_id(motor)
+            recv_id = self._get_motor_recv_id(motor)
+            data = [0xFF] * 7 + [CAN_CMD_SET_ZERO]
+            msg = can.Message(arbitration_id=motor_id, data=data, is_extended_id=False)
+            self.canbus.send(msg)
+            self._recv_motor_response(expected_recv_id=recv_id)
+            time.sleep(0.01)
+
+    def _refresh_motor(self, motor: NameOrID) -> Optional[can.Message]:
+        """Refresh motor status and return the response."""
+        motor_id = self._get_motor_id(motor)
+        recv_id = self._get_motor_recv_id(motor)
+        data = [motor_id & 0xFF, (motor_id >> 8) & 0xFF, CAN_CMD_REFRESH, 0, 0, 0, 0, 0]
+        msg = can.Message(arbitration_id=CAN_PARAM_ID, data=data, is_extended_id=False)
+        self.canbus.send(msg)
+        return self._recv_motor_response(expected_recv_id=recv_id)
+
+    def _recv_motor_response(self, expected_recv_id: Optional[int] = None, timeout: float = 0.001) -> Optional[can.Message]:
+        """
+        Receive a response from a motor.
+        
+        Args:
+            expected_recv_id: If provided, only return messages from this CAN ID
+            timeout: Timeout in seconds (default: 1ms for high-speed operation)
+        
+        Returns:
+            CAN message if received, None otherwise
+        """
+        try:
+            start_time = time.time()
+            messages_seen = []
+            while time.time() - start_time < timeout:
+                msg = self.canbus.recv(timeout=0.0001)  # 100us timeout for fast polling
+                if msg:
+                    messages_seen.append(f"0x{msg.arbitration_id:02X}")
+                    # If no filter specified, return any message
+                    if expected_recv_id is None:
+                        return msg
+                    # Otherwise, only return if it matches the expected recv_id
+                    if msg.arbitration_id == expected_recv_id:
+                        return msg
+                    else:
+                        logger.debug(f"Ignoring message from CAN ID 0x{msg.arbitration_id:02X}, expected 0x{expected_recv_id:02X}")
+            
+            # Only log warnings if we're in debug mode to reduce overhead
+            if logger.isEnabledFor(logging.DEBUG):
+                if messages_seen:
+                    logger.debug(f"Received {len(messages_seen)} message(s) from IDs {set(messages_seen)}, but expected 0x{expected_recv_id:02X}")
+                else:
+                    logger.debug(f"No CAN messages received (expected from 0x{expected_recv_id:02X})")
+        except Exception as e:
+            logger.debug(f"Failed to receive CAN message: {e}")
+        return None
+    
+    def _recv_all_responses(self, expected_recv_ids: list[int], timeout: float = 0.002) -> dict[int, can.Message]:
+        """
+        Efficiently receive responses from multiple motors at once.
+        Uses the OpenArms pattern: collect all available messages within timeout.
+        
+        Args:
+            expected_recv_ids: List of CAN IDs we expect responses from
+            timeout: Total timeout in seconds (default: 2ms)
+        
+        Returns:
+            Dictionary mapping recv_id to CAN message
+        """
+        responses = {}
+        expected_set = set(expected_recv_ids)
+        start_time = time.time()
+        
+        try:
+            while len(responses) < len(expected_recv_ids) and (time.time() - start_time) < timeout:
+                msg = self.canbus.recv(timeout=0.0001)  # 100us poll timeout
+                if msg and msg.arbitration_id in expected_set:
+                    responses[msg.arbitration_id] = msg
+                    if len(responses) == len(expected_recv_ids):
+                        break  # Got all responses, exit early
+        except Exception as e:
+            logger.debug(f"Error receiving responses: {e}")
+        
+        return responses
+
+    def _mit_control(
+        self,
+        motor: NameOrID,
+        kp: float,
+        kd: float,
+        position_degrees: float,
+        velocity_deg_per_sec: float,
+        torque: float,
+    ) -> None:
+        """
+        Send MIT control command to a motor.
+
+        Args:
+            motor: Motor name or ID
+            kp: Position gain
+            kd: Velocity gain
+            position_degrees: Target position (degrees)
+            velocity_deg_per_sec: Target velocity (degrees/s)
+            torque: Target torque (N·m)
+        """
+        motor_id = self._get_motor_id(motor)
+        motor_name = self._get_motor_name(motor)
+        motor_type = self._motor_types.get(motor_name, MotorType.DM4310)
+        
+        # Convert degrees to radians for motor control
+        position_rad = np.radians(position_degrees)
+        velocity_rad_per_sec = np.radians(velocity_deg_per_sec)
+        
+        # Get motor limits
+        pmax, vmax, tmax = MOTOR_LIMIT_PARAMS[motor_type]
+        
+        # Encode parameters
+        kp_uint = self._float_to_uint(kp, 0, 500, 12)
+        kd_uint = self._float_to_uint(kd, 0, 5, 12)
+        q_uint = self._float_to_uint(position_rad, -pmax, pmax, 16)
+        dq_uint = self._float_to_uint(velocity_rad_per_sec, -vmax, vmax, 12)
+        tau_uint = self._float_to_uint(torque, -tmax, tmax, 12)
+        
+        # Pack data
+        data = [0] * 8
+        data[0] = (q_uint >> 8) & 0xFF
+        data[1] = q_uint & 0xFF
+        data[2] = dq_uint >> 4
+        data[3] = ((dq_uint & 0xF) << 4) | ((kp_uint >> 8) & 0xF)
+        data[4] = kp_uint & 0xFF
+        data[5] = kd_uint >> 4
+        data[6] = ((kd_uint & 0xF) << 4) | ((tau_uint >> 8) & 0xF)
+        data[7] = tau_uint & 0xFF
+        
+        msg = can.Message(arbitration_id=motor_id, data=data, is_extended_id=False)
+        self.canbus.send(msg)
+        recv_id = self._get_motor_recv_id(motor)
+        self._recv_motor_response(expected_recv_id=recv_id)
+
+    def _float_to_uint(self, x: float, x_min: float, x_max: float, bits: int) -> int:
+        """Convert float to unsigned integer for CAN transmission."""
+        x = max(x_min, min(x_max, x))  # Clamp to range
+        span = x_max - x_min
+        data_norm = (x - x_min) / span
+        return int(data_norm * ((1 << bits) - 1))
+
+    def _uint_to_float(self, x: int, x_min: float, x_max: float, bits: int) -> float:
+        """Convert unsigned integer from CAN to float."""
+        span = x_max - x_min
+        data_norm = float(x) / ((1 << bits) - 1)
+        return data_norm * span + x_min
+
+    def _decode_motor_state(self, data: bytes, motor_type: MotorType) -> Tuple[float, float, float, int, int]:
+        """
+        Decode motor state from CAN data.
+
+        Returns:
+            Tuple of (position_degrees, velocity_deg_per_sec, torque, temp_mos, temp_rotor)
+        """
+        if len(data) < 8:
+            raise ValueError("Invalid motor state data")
+        
+        # Extract encoded values
+        q_uint = (data[1] << 8) | data[2]
+        dq_uint = (data[3] << 4) | (data[4] >> 4)
+        tau_uint = ((data[4] & 0x0F) << 8) | data[5]
+        t_mos = data[6]
+        t_rotor = data[7]
+        
+        # Get motor limits
+        pmax, vmax, tmax = MOTOR_LIMIT_PARAMS[motor_type]
+        
+        # Decode to physical values (radians)
+        position_rad = self._uint_to_float(q_uint, -pmax, pmax, 16)
+        velocity_rad_per_sec = self._uint_to_float(dq_uint, -vmax, vmax, 12)
+        torque = self._uint_to_float(tau_uint, -tmax, tmax, 12)
+        
+        # Convert to degrees
+        position_degrees = np.degrees(position_rad)
+        velocity_deg_per_sec = np.degrees(velocity_rad_per_sec)
+        
+        return position_degrees, velocity_deg_per_sec, torque, t_mos, t_rotor
+
+    def read(
+        self,
+        data_name: str,
+        motor: str,
+        *,
+        normalize: bool = True,
+        num_retry: int = 0,
+    ) -> Value:
+        """Read a value from a single motor. Positions are always in degrees."""
+        if not self.is_connected:
+            raise DeviceNotConnectedError(f"{self} is not connected.")
+        
+        # Refresh motor to get latest state
+        msg = self._refresh_motor(motor)
+        if msg is None:
+            motor_id = self._get_motor_id(motor)
+            recv_id = self._get_motor_recv_id(motor)
+            raise ConnectionError(
+                f"No response from motor '{motor}' (send ID: 0x{motor_id:02X}, recv ID: 0x{recv_id:02X}). "
+                f"Check that: 1) Motor is powered (24V), 2) CAN wiring is correct, "
+                f"3) Motor IDs are configured correctly using Damiao Debugging Tools"
+            )
+        
+        motor_type = self._motor_types.get(motor, MotorType.DM4310)
+        position_degrees, velocity_deg_per_sec, torque, t_mos, t_rotor = self._decode_motor_state(msg.data, motor_type)
+        
+        # Return requested data (already in degrees for position/velocity)
+        if data_name == "Present_Position":
+            value = position_degrees
+        elif data_name == "Present_Velocity":
+            value = velocity_deg_per_sec
+        elif data_name == "Present_Torque":
+            value = torque
+        elif data_name == "Temperature_MOS":
+            value = t_mos
+        elif data_name == "Temperature_Rotor":
+            value = t_rotor
+        else:
+            raise ValueError(f"Unknown data_name: {data_name}")
+        
+        # For Damiao, positions are always in degrees, no normalization needed
+        # We keep the normalize parameter for compatibility but don't use it
+        return value
+
+    def write(
+        self,
+        data_name: str,
+        motor: str,
+        value: Value,
+        *,
+        normalize: bool = True,
+        num_retry: int = 0,
+    ) -> None:
+        """Write a value to a single motor. Positions are always in degrees."""
+        if not self.is_connected:
+            raise DeviceNotConnectedError(f"{self} is not connected.")
+        
+        # Value is expected to be in degrees for positions
+        if data_name == "Goal_Position":
+            # Use MIT control with position in degrees
+            self._mit_control(motor, 10.0, 0.5, value, 0, 0)
+        else:
+            raise ValueError(f"Writing {data_name} not supported in MIT mode")
+
+    def sync_read(
+        self,
+        data_name: str,
+        motors: str | list[str] | None = None,
+        *,
+        normalize: bool = True,
+        num_retry: int = 0,
+    ) -> Dict[str, Value]:
+        """
+        Read the same value from multiple motors simultaneously.
+        Uses batched operations: sends all refresh commands, then collects all responses.
+        This is MUCH faster than sequential reads (OpenArms pattern).
+        """
+        motors = self._get_motors_list(motors)
+        result = {}
+
+        # Step 1: Send refresh commands to ALL motors first (no waiting)
+        for motor in motors:
+            motor_id = self._get_motor_id(motor)
+            data = [motor_id & 0xFF, (motor_id >> 8) & 0xFF, CAN_CMD_REFRESH, 0, 0, 0, 0, 0]
+            msg = can.Message(arbitration_id=CAN_PARAM_ID, data=data, is_extended_id=False)
+            self.canbus.send(msg)
+        
+        # Step 2: Collect all responses at once (batch receive)
+        expected_recv_ids = [self._get_motor_recv_id(motor) for motor in motors]
+        responses = self._recv_all_responses(expected_recv_ids, timeout=0.003)  # 3ms total timeout
+        
+        # Step 3: Parse responses
+        for motor in motors:
+            try:
+                recv_id = self._get_motor_recv_id(motor)
+                msg = responses.get(recv_id)
+                
+                if msg is None:
+                    logger.warning(f"No response from motor '{motor}' (recv ID: 0x{recv_id:02X})")
+                    result[motor] = 0.0
+                    continue
+                
+                motor_type = self._motor_types.get(motor, MotorType.DM4310)
+                position_degrees, velocity_deg_per_sec, torque, t_mos, t_rotor = self._decode_motor_state(msg.data, motor_type)
+                
+                # Return requested data
+                if data_name == "Present_Position":
+                    value = position_degrees
+                elif data_name == "Present_Velocity":
+                    value = velocity_deg_per_sec
+                elif data_name == "Present_Torque":
+                    value = torque
+                elif data_name == "Temperature_MOS":
+                    value = t_mos
+                elif data_name == "Temperature_Rotor":
+                    value = t_rotor
+                else:
+                    raise ValueError(f"Unknown data_name: {data_name}")
+                
+                result[motor] = value
+                
+            except Exception as e:
+                logger.warning(f"Failed to read {data_name} from {motor}: {e}")
+                result[motor] = 0.0
+        
+        return result
+
+    def sync_write(
+        self,
+        data_name: str,
+        values: Dict[str, Value],
+        *,
+        normalize: bool = True,
+        num_retry: int = 0,
+    ) -> None:
+        """
+        Write different values to multiple motors simultaneously. Positions are always in degrees.
+        Uses batched operations: sends all commands first, then collects responses (OpenArms pattern).
+        """
+        if data_name == "Goal_Position":
+            # Step 1: Send all MIT control commands first (no waiting)
+            for motor, value_degrees in values.items():
+                motor_id = self._get_motor_id(motor)
+                motor_name = self._get_motor_name(motor)
+                motor_type = self._motor_types.get(motor_name, MotorType.DM4310)
+                
+                # Convert degrees to radians
+                position_rad = np.radians(value_degrees)
+                
+                # Default gains for position control
+                kp, kd = 10.0, 0.5
+                
+                # Get motor limits and encode parameters
+                pmax, vmax, tmax = MOTOR_LIMIT_PARAMS[motor_type]
+                kp_uint = self._float_to_uint(kp, 0, 500, 12)
+                kd_uint = self._float_to_uint(kd, 0, 5, 12)
+                q_uint = self._float_to_uint(position_rad, -pmax, pmax, 16)
+                dq_uint = self._float_to_uint(0, -vmax, vmax, 12)
+                tau_uint = self._float_to_uint(0, -tmax, tmax, 12)
+                
+                # Pack data
+                data = [0] * 8
+                data[0] = (q_uint >> 8) & 0xFF
+                data[1] = q_uint & 0xFF
+                data[2] = dq_uint >> 4
+                data[3] = ((dq_uint & 0xF) << 4) | ((kp_uint >> 8) & 0xF)
+                data[4] = kp_uint & 0xFF
+                data[5] = kd_uint >> 4
+                data[6] = ((kd_uint & 0xF) << 4) | ((tau_uint >> 8) & 0xF)
+                data[7] = tau_uint & 0xFF
+                
+                msg = can.Message(arbitration_id=motor_id, data=data, is_extended_id=False)
+                self.canbus.send(msg)
+            
+            # Step 2: Collect all responses at once
+            expected_recv_ids = [self._get_motor_recv_id(motor) for motor in values.keys()]
+            self._recv_all_responses(expected_recv_ids, timeout=0.002)  # 2ms timeout
+        else:
+            # Fall back to individual writes for other data types
+            for motor, value in values.items():
+                self.write(data_name, motor, value, normalize=normalize, num_retry=num_retry)
+
+    def read_calibration(self) -> dict[str, MotorCalibration]:
+        """Read calibration data from motors."""
+        # Damiao motors don't store calibration internally
+        # Return existing calibration or empty dict
+        return self.calibration if self.calibration else {}
+
+    def write_calibration(self, calibration_dict: dict[str, MotorCalibration], cache: bool = True) -> None:
+        """Write calibration data to motors."""
+        # Damiao motors don't store calibration internally
+        # Just cache it in memory
+        if cache:
+            self.calibration = calibration_dict
+
+    def record_ranges_of_motion(
+        self, motors: NameOrID | list[NameOrID] | None = None, display_values: bool = True
+    ) -> tuple[dict[NameOrID, Value], dict[NameOrID, Value]]:
+        """
+        Interactively record the min/max values of each motor in degrees.
+
+        Move the joints by hand (with torque disabled) while the method streams live positions.
+        Press Enter to finish.
+        """
+        if motors is None:
+            motors = list(self.motors.keys())
+        elif isinstance(motors, (str, int)):
+            motors = [motors]
+
+        # Disable torque for manual movement
+        self.disable_torque(motors)
+        time.sleep(0.1)
+
+        # Get initial positions (already in degrees)
+        start_positions = self.sync_read("Present_Position", motors, normalize=False)
+        mins = start_positions.copy()
+        maxes = start_positions.copy()
+
+        print("\nMove joints through their full range of motion. Press ENTER when done.")
+        user_pressed_enter = False
+        
+        while not user_pressed_enter:
+            positions = self.sync_read("Present_Position", motors, normalize=False)
+            
+            for motor in motors:
+                if motor in positions:
+                    mins[motor] = min(positions[motor], mins.get(motor, positions[motor]))
+                    maxes[motor] = max(positions[motor], maxes.get(motor, positions[motor]))
+
+            if display_values:
+                print("\n" + "=" * 50)
+                print(f"{'MOTOR':<20} | {'MIN (deg)':>12} | {'POS (deg)':>12} | {'MAX (deg)':>12}")
+                print("-" * 50)
+                for motor in motors:
+                    if motor in positions:
+                        print(f"{motor:<20} | {mins[motor]:>12.1f} | {positions[motor]:>12.1f} | {maxes[motor]:>12.1f}")
+
+            if enter_pressed():
+                user_pressed_enter = True
+
+            if display_values and not user_pressed_enter:
+                # Move cursor up to overwrite the previous output
+                move_cursor_up(len(motors) + 4)
+
+            time.sleep(0.05)
+
+        # Re-enable torque
+        self.enable_torque(motors)
+        
+        # Validate ranges
+        for motor in motors:
+            if motor in mins and motor in maxes:
+                if abs(maxes[motor] - mins[motor]) < 5.0:  # At least 5 degrees of range
+                    raise ValueError(f"Motor {motor} has insufficient range of motion (< 5 degrees)")
+
+        return mins, maxes
+
+    def _get_motors_list(self, motors: str | list[str] | None) -> list[str]:
+        """Convert motor specification to list of motor names."""
+        if motors is None:
+            return list(self.motors.keys())
+        elif isinstance(motors, str):
+            return [motors]
+        elif isinstance(motors, list):
+            return motors
+        else:
+            raise TypeError(f"Invalid motors type: {type(motors)}")
+
+    def _get_motor_id(self, motor: NameOrID) -> int:
+        """Get CAN ID for a motor."""
+        if isinstance(motor, str):
+            if motor in self.motors:
+                return self.motors[motor].id
+            else:
+                raise ValueError(f"Unknown motor: {motor}")
+        else:
+            return motor
+
+    def _get_motor_name(self, motor: NameOrID) -> str:
+        """Get motor name from name or ID."""
+        if isinstance(motor, str):
+            return motor
+        else:
+            for name, m in self.motors.items():
+                if m.id == motor:
+                    return name
+            raise ValueError(f"Unknown motor ID: {motor}")
+    
+    def _get_motor_recv_id(self, motor: NameOrID) -> Optional[int]:
+        """Get motor recv_id from name or ID."""
+        motor_name = self._get_motor_name(motor)
+        motor_obj = self.motors.get(motor_name)
+        if motor_obj and hasattr(motor_obj, "recv_id"):
+            return motor_obj.recv_id
+        return None
+
+    @cached_property
+    def is_calibrated(self) -> bool:
+        """Check if motors are calibrated."""
+        return bool(self.calibration)

src/lerobot/motors/damiao/seed_damiao.py

@@ -0,0 +1,833 @@
+## This is a derivative of the following software.
+## https://github.com/cmjang/DM_Control_Python/blob/main/DM_CAN.py
+
+import can
+from time import sleep, time
+import numpy as np
+from enum import IntEnum
+from struct import unpack
+from struct import pack
+
+class Motor:
+    def __init__(self, MotorType, SlaveID, MasterID):
+        """
+        define Motor object 定义电机对象
+        :param MotorType: Motor type 电机类型
+        :param SlaveID: CANID 电机ID
+        :param MasterID: MasterID 主机ID 建议不要设为0
+        """
+        self.Pd = float(0)
+        self.Vd = float(0)
+        self.goal_position = float(0)
+        self.goal_tau = float(0)
+        self.state_q = float(0)
+        self.state_dq = float(0)
+        self.state_tau = float(0)
+        self.state_tmos = int(0)
+        self.state_trotor = int(0)
+        self.SlaveID = SlaveID
+        self.MasterID = MasterID
+        self.MotorType = MotorType
+        self.isEnable = False
+        self.NowControlMode = Control_Type.MIT
+        self.temp_param_dict = {}
+
+    def recv_data(self, q: float, dq: float, tau: float, tmos: int, trotor: int):
+        self.state_q = q
+        self.state_dq = dq
+        self.state_tau = tau
+        self.state_tmos = tmos
+        self.state_trotor = trotor
+
+    def getPosition(self):
+        """
+        get the position of the motor 获取电机位置
+        :return: the position of the motor 电机位置
+        """
+        return self.state_q
+
+    def getVelocity(self):
+        """
+        get the velocity of the motor 获取电机速度
+        :return: the velocity of the motor 电机速度
+        """
+        return self.state_dq
+
+    def getTorque(self):
+        """
+        get the torque of the motor 获取电机力矩
+        :return: the torque of the motor 电机力矩
+        """
+        return self.state_tau
+
+    def getParam(self, RID):
+        """
+        get the parameter of the motor 获取电机内部的参数，需要提前读取
+        :param RID: DM_variable 电机参数
+        :return: the parameter of the motor 电机参数
+        """
+        if RID in self.temp_param_dict:
+            return self.temp_param_dict[RID]
+        else:
+            return None
+
+
+class MotorControl:
+    #send_data_frame = np.array(
+    #    [0x55, 0xAA, 0x1e, 0x03, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0, 0, 0, 0, 0x00, 0x08, 0x00,
+    #     0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0x00], np.uint8)
+    #                4310           4310_48        4340           4340_48
+    Limit_Param = [[12.5, 30, 10], [12.5, 50, 10], [12.5, 8, 28], [12.5, 10, 28],
+                   # 6006           8006           8009            10010L         10010
+                   [12.5, 45, 20], [12.5, 45, 40], [12.5, 45, 54], [12.5, 25, 200], [12.5, 20, 200],
+                   # H3510            DMG62150      DMH6220
+                   [12.5 , 280 , 1],[12.5 , 45 , 10],[12.5 , 45 , 10]]
+
+    def __init__(self, channel: str, bitrate: int = 1000000):
+        """
+        define MotorControl object 定义电机控制对象
+        :param serial_device: serial object 串口对象
+        """
+        #self.serial_ = serial_device
+        self.motors_map = dict()
+        self.data_save = bytes()  # save data
+        #if self.serial_.is_open:  # open the serial port
+        #    print("Serial port is open")
+        #    serial_device.close()
+        #self.serial_.open()
+        self.canbus = can.interface.Bus(channel=channel, interface='socketcan', bitrate=bitrate)
+
+        #print("can is open")
+        
+
+    def controlMIT(self, DM_Motor, kp: float, kd: float, q: float, dq: float, tau: float):
+        """
+        MIT Control Mode Function 达妙电机MIT控制模式函数
+        :param DM_Motor: Motor object 电机对象
+        :param kp: kp
+        :param kd:  kd
+        :param q:  position  期望位置
+        :param dq:  velocity  期望速度
+        :param tau: torque  期望力矩
+        :return: None
+        """
+        if DM_Motor.SlaveID not in self.motors_map:
+            print("controlMIT ERROR : Motor ID not found")
+            return
+        kp_uint = float_to_uint(kp, 0, 500, 12)
+        kd_uint = float_to_uint(kd, 0, 5, 12)
+        MotorType = DM_Motor.MotorType
+        Q_MAX = self.Limit_Param[MotorType][0]
+        DQ_MAX = self.Limit_Param[MotorType][1]
+        TAU_MAX = self.Limit_Param[MotorType][2]
+        q_uint = float_to_uint(q, -Q_MAX, Q_MAX, 16)
+        dq_uint = float_to_uint(dq, -DQ_MAX, DQ_MAX, 12)
+        tau_uint = float_to_uint(tau, -TAU_MAX, TAU_MAX, 12)
+        data_buf = np.array([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], np.uint8)
+        data_buf[0] = (q_uint >> 8) & 0xff
+        data_buf[1] = q_uint & 0xff
+        data_buf[2] = dq_uint >> 4
+        data_buf[3] = ((dq_uint & 0xf) << 4) | ((kp_uint >> 8) & 0xf)
+        data_buf[4] = kp_uint & 0xff
+        data_buf[5] = kd_uint >> 4
+        data_buf[6] = ((kd_uint & 0xf) << 4) | ((tau_uint >> 8) & 0xf)
+        data_buf[7] = tau_uint & 0xff
+        self.__send_data(DM_Motor.SlaveID, data_buf)
+        self.recv()  # receive the data from serial port
+
+    def control_delay(self, DM_Motor, kp: float, kd: float, q: float, dq: float, tau: float, delay: float):
+        """
+        MIT Control Mode Function with delay 达妙电机MIT控制模式函数带延迟
+        :param DM_Motor: Motor object 电机对象
+        :param kp: kp
+        :param kd: kd
+        :param q:  position  期望位置
+        :param dq:  velocity  期望速度
+        :param tau: torque  期望力矩
+        :param delay: delay time 延迟时间 单位秒
+        """
+        self.controlMIT(DM_Motor, kp, kd, q, dq, tau)
+        sleep(delay)
+
+    def control_Pos_Vel(self, Motor, P_desired: float, V_desired: float):
+        """
+        control the motor in position and velocity control mode 电机位置速度控制模式
+        :param Motor: Motor object 电机对象
+        :param P_desired: desired position 期望位置
+        :param V_desired: desired velocity 期望速度
+        :return: None
+        """
+        if Motor.SlaveID not in self.motors_map:
+            print("Control Pos_Vel Error : Motor ID not found")
+            return
+        motorid = 0x100 + Motor.SlaveID
+        data_buf = np.array([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], np.uint8)
+        P_desired_uint8s = float_to_uint8s(P_desired)
+        V_desired_uint8s = float_to_uint8s(V_desired)
+        data_buf[0:4] = P_desired_uint8s
+        data_buf[4:8] = V_desired_uint8s
+        self.__send_data(motorid, data_buf)
+        self.recv()  # receive the data from serial port
+
+    def control_Vel(self, Motor, Vel_desired):
+        """
+        control the motor in velocity control mode 电机速度控制模式
+        :param Motor: Motor object 电机对象
+        :param Vel_desired: desired velocity 期望速度
+        """
+        if Motor.SlaveID not in self.motors_map:
+            print("control_VEL ERROR : Motor ID not found")
+            return
+        motorid = 0x200 + Motor.SlaveID
+        data_buf = np.array([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], np.uint8)
+        Vel_desired_uint8s = float_to_uint8s(Vel_desired)
+        data_buf[0:4] = Vel_desired_uint8s
+        self.__send_data(motorid, data_buf)
+        self.recv()  # receive the data from serial port
+
+    def control_pos_force(self, Motor, Pos_des: float, Vel_des, i_des):
+        """
+        control the motor in EMIT control mode 电机力位混合模式
+        :param Pos_des: desired position rad  期望位置 单位为rad
+        :param Vel_des: desired velocity rad/s  期望速度 为放大100倍
+        :param i_des: desired current rang 0-10000 期望电流标幺值放大10000倍
+        电流标幺值：实际电流值除以最大电流值，最大电流见上电打印
+        """
+        if Motor.SlaveID not in self.motors_map:
+            print("control_pos_vel ERROR : Motor ID not found")
+            return
+        motorid = 0x300 + Motor.SlaveID
+        data_buf = np.array([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], np.uint8)
+        Pos_desired_uint8s = float_to_uint8s(Pos_des)
+        data_buf[0:4] = Pos_desired_uint8s
+        Vel_uint = np.uint16(Vel_des)
+        ides_uint = np.uint16(i_des)
+        data_buf[4] = Vel_uint & 0xff
+        data_buf[5] = Vel_uint >> 8
+        data_buf[6] = ides_uint & 0xff
+        data_buf[7] = ides_uint >> 8
+        self.__send_data(motorid, data_buf)
+        self.recv()  # receive the data from serial port
+
+    def enable(self, Motor):
+        """
+        enable motor 使能电机
+        最好在上电后几秒后再使能电机
+        :param Motor: Motor object 电机对象
+        """
+        self.__control_cmd(Motor, np.uint8(0xFC))
+        sleep(0.1)
+        self.recv()  # receive the data from serial port
+
+    def enable_old(self, Motor ,ControlMode):
+        """
+        enable motor old firmware 使能电机旧版本固件，这个是为了旧版本电机固件的兼容性
+        可恶的旧版本固件使能需要加上偏移量
+        最好在上电后几秒后再使能电机
+        :param Motor: Motor object 电机对象
+        """
+        data_buf = np.array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc], np.uint8)
+        enable_id = ((int(ControlMode)-1) << 2) + Motor.SlaveID
+        self.__send_data(enable_id, data_buf)
+        sleep(0.1)
+        self.recv()  # receive the data from serial port
+
+    def disable(self, Motor):
+        """
+        disable motor 失能电机
+        :param Motor: Motor object 电机对象
+        """
+        self.__control_cmd(Motor, np.uint8(0xFD))
+        sleep(0.1)
+        self.recv()  # receive the data from serial port
+
+    def set_zero_position(self, Motor):
+        """
+        set the zero position of the motor 设置电机0位
+        :param Motor: Motor object 电机对象
+        """
+        self.__control_cmd(Motor, np.uint8(0xFE))
+        sleep(0.1)
+        self.recv()  # receive the data from serial port
+
+    def recv(self):
+        # 把上次没有解析完的剩下的也放进来
+        # data_recv = b''.join([self.data_save, self.serial_.read_all()])
+        #data_recv = b''.join([self.data_save, self.canbus.recv()])
+
+
+        # packets = self.__extract_packets(data_recv)
+        # for packet in packets:
+        #     data = packet[7:15]
+        #     CANID = (packet[6] << 24) | (packet[5] << 16) | (packet[4] << 8) | packet[3]
+        #     CMD = packet[1]
+        #     self.__process_packet(data, CANID, CMD)
+        
+        data_recv = self.canbus.recv(0.1)
+
+        if data_recv is not None:
+            # data = data_recv.data
+            # err = data[0] >> 12
+            # id = data[0] & 0x7f
+            # pos = (data[1] << 8) + data[2]
+            # vel = (data[3] << 4) + (data[4] >> 4)
+            # tau = ((data[4] & 0x0f) << 8) + data[5]
+            # t_mos = data[6]
+            # t_rotor = data[7]
+            # print(hex(id), err, id, pos, vel, tau, goal_tau, t_mos, t_rotor)
+            # CANID = data_recv.arbitration_id
+            CANID = data_recv.data[0]
+            # CMD = data_recv.data[3]
+            CMD = 0x11                  # 飯田：修正の必要あり
+            self.__process_packet(data_recv.data, CANID, CMD)
+
+            # 飯田：Debug print
+            # print(hex(CANID),hex(CMD))
+            # print(hex(data_recv.data[0]),hex(data_recv.data[1]),hex(data_recv.data[2]),hex(data_recv.data[3]),hex(data_recv.data[4]),hex(data_recv.data[5]),hex(data_recv.data[6]),hex(data_recv.data[7]))
+            #return data
+
+    def recv_set_param_data(self):
+        #data_recv = self.serial_.read_all()
+
+        # packets = self.__extract_packets(data_recv)
+        # for packet in packets:
+        #     data = packet[7:15]
+        #     CANID = (packet[6] << 24) | (packet[5] << 16) | (packet[4] << 8) | packet[3]
+        #     CMD = packet[1]
+        #     self.__process_set_param_packet(data, CANID, CMD)
+        
+        data_recv = self.canbus.recv(0.1)
+
+
+        if data_recv is not None:
+            data = data_recv.data
+            CANID = data_recv.arbitration_id
+            # CANID = data_recv.data[0]
+            # CMD = data_recv.data[3]  
+            CMD = 0x11                  # 飯田：修正の必要あり
+            self.__process_packet(data, CANID, CMD)
+
+
+            # 飯田：Debug print
+            print(hex(CANID),hex(CMD))
+            print(hex(data_recv.data[0]),hex(data_recv.data[1]),hex(data_recv.data[2]),hex(data_recv.data[3]),hex(data_recv.data[4]),hex(data_recv.data[5]),hex(data_recv.data[6]),hex(data_recv.data[7]))
+
+    def __process_packet(self, data, CANID, CMD):
+        if CMD == 0x11:
+            if CANID != 0x00:
+                if CANID in self.motors_map:
+                    q_uint = np.uint16((np.uint16(data[1]) << 8) | data[2])
+                    dq_uint = np.uint16((np.uint16(data[3]) << 4) | (data[4] >> 4))
+                    tau_uint = np.uint16(((data[4] & 0xf) << 8) | data[5])
+                    t_mos = data[6]
+                    t_rotor = data[7]
+                    MotorType_recv = self.motors_map[CANID].MotorType
+                    Q_MAX = self.Limit_Param[MotorType_recv][0]
+                    DQ_MAX = self.Limit_Param[MotorType_recv][1]
+                    TAU_MAX = self.Limit_Param[MotorType_recv][2]
+                    recv_q = uint_to_float(q_uint, -Q_MAX, Q_MAX, 16)
+                    recv_dq = uint_to_float(dq_uint, -DQ_MAX, DQ_MAX, 12)
+                    recv_tau = uint_to_float(tau_uint, -TAU_MAX, TAU_MAX, 12)
+                    self.motors_map[CANID].recv_data(recv_q, recv_dq, recv_tau, t_mos, t_rotor)
+            else:
+                MasterID=data[0] & 0x0f
+                if MasterID in self.motors_map:
+                    q_uint = np.uint16((np.uint16(data[1]) << 8) | data[2])
+                    dq_uint = np.uint16((np.uint16(data[3]) << 4) | (data[4] >> 4))
+                    tau_uint = np.uint16(((data[4] & 0xf) << 8) | data[5])
+                    t_mos = data[6]
+                    t_rotor = data[7]
+                    MotorType_recv = self.motors_map[MasterID].MotorType
+                    Q_MAX = self.Limit_Param[MotorType_recv][0]
+                    DQ_MAX = self.Limit_Param[MotorType_recv][1]
+                    TAU_MAX = self.Limit_Param[MotorType_recv][2]
+                    recv_q = uint_to_float(q_uint, -Q_MAX, Q_MAX, 16)
+                    recv_dq = uint_to_float(dq_uint, -DQ_MAX, DQ_MAX, 12)
+                    recv_tau = uint_to_float(tau_uint, -TAU_MAX, TAU_MAX, 12)
+                    self.motors_map[MasterID].recv_data(recv_q, recv_dq, recv_tau, t_mos, t_rotor)
+
+
+    def __process_set_param_packet(self, data, CANID, CMD):
+        if CMD == 0x11 and (data[2] == 0x33 or data[2] == 0x55):
+            masterid=CANID
+            slaveId = ((data[1] << 8) | data[0])
+            if CANID==0x00:  #防止有人把MasterID设为0稳一手
+                masterid=slaveId
+
+            if masterid not in self.motors_map:
+                if slaveId not in self.motors_map:
+                    return
+                else:
+                    masterid=slaveId
+
+            RID = data[3]
+            # 读取参数得到的数据
+            if is_in_ranges(RID):
+                #uint32类型
+                num = uint8s_to_uint32(data[4], data[5], data[6], data[7])
+                self.motors_map[masterid].temp_param_dict[RID] = num
+
+            else:
+                #float类型
+                num = uint8s_to_float(data[4], data[5], data[6], data[7])
+                self.motors_map[masterid].temp_param_dict[RID] = num
+
+
+    def addMotor(self, Motor):
+        """
+        add motor to the motor control object 添加电机到电机控制对象
+        :param Motor: Motor object 电机对象
+        """
+        self.motors_map[Motor.SlaveID] = Motor
+        if Motor.MasterID != 0:
+            self.motors_map[Motor.MasterID] = Motor
+        return True
+
+    def __control_cmd(self, Motor, cmd: np.uint8):     # 飯田：コマンドは通ります
+        data_buf = np.array([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, cmd], np.uint8)
+        self.__send_data(Motor.SlaveID, data_buf)
+
+    def __send_data(self, motor_id, data):
+        """
+        send data to the motor 发送数据到电机
+        :param motor_id:
+        :param data:
+        :return:
+        """
+        #self.send_data_frame[13] = motor_id & 0xff
+        #self.send_data_frame[14] = (motor_id >> 8)& 0xff  #id high 8 bits
+        #self.send_data_frame[21:29] = data
+        #self.serial_.write(bytes(self.send_data_frame.T))
+
+        msg =can.Message(is_extended_id=False,arbitration_id=motor_id,data=data,is_remote_frame = False)
+        self.canbus.send(msg)
+
+    def __read_RID_param(self, Motor, RID):             # 飯田：修正の必要あり?
+        can_id_l = Motor.SlaveID & 0xff #id low 8 bits
+        can_id_h = (Motor.SlaveID >> 8)& 0xff  #id high 8 bits
+        data_buf = np.array([np.uint8(can_id_l), np.uint8(can_id_h), 0x33, np.uint8(RID), 0x00, 0x00, 0x00, 0x00], np.uint8)
+        self.__send_data(0x7FF, data_buf)
+        
+
+
+    def __write_motor_param(self, Motor, RID, data):             # 飯田：修正の必要あり?
+        can_id_l = Motor.SlaveID & 0xff #id low 8 bits
+        can_id_h = (Motor.SlaveID >> 8)& 0xff  #id high 8 bits
+        data_buf = np.array([np.uint8(can_id_l), np.uint8(can_id_h), 0x55, np.uint8(RID), 0x00, 0x00, 0x00, 0x00], np.uint8)
+        if not is_in_ranges(RID):
+            # data is float
+            data_buf[4:8] = float_to_uint8s(data)
+        else:
+            # data is int
+            data_buf[4:8] = data_to_uint8s(int(data))
+        self.__send_data(0x7FF, data_buf)
+
+    def switchControlMode(self, Motor, ControlMode):
+        """
+        switch the control mode of the motor 切换电机控制模式
+        :param Motor: Motor object 电机对象
+        :param ControlMode: Control_Type 电机控制模式 example:MIT:Control_Type.MIT MIT模式
+        """
+        max_retries = 20
+        retry_interval = 0.1  #retry times
+        RID = 10
+        self.__write_motor_param(Motor, RID, np.uint8(ControlMode))
+        for _ in range(max_retries):
+            sleep(retry_interval)
+            self.recv_set_param_data()
+            if Motor.SlaveID in self.motors_map:
+                if RID in self.motors_map[Motor.SlaveID].temp_param_dict:
+                    if abs(self.motors_map[Motor.SlaveID].temp_param_dict[RID] - ControlMode) < 0.1:
+                        return True
+                    else:
+                        return False
+        return False
+
+    def save_motor_param(self, Motor):
+        """
+        save the all parameter  to flash 保存所有电机参数
+        :param Motor: Motor object 电机对象
+        :return:
+        """
+        can_id_l = Motor.SlaveID & 0xff #id low 8 bits
+        can_id_h = (Motor.SlaveID >> 8)& 0xff  #id high 8 bits
+        data_buf = np.array([np.uint8(can_id_l), np.uint8(can_id_h), 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00], np.uint8)
+        self.disable(Motor)  # before save disable the motor
+        self.__send_data(0x7FF, data_buf)
+        sleep(0.001)
+
+    def change_limit_param(self, Motor_Type, PMAX, VMAX, TMAX):
+        """
+        change the PMAX VMAX TMAX of the motor 改变电机的PMAX VMAX TMAX
+        :param Motor_Type:
+        :param PMAX: 电机的PMAX
+        :param VMAX: 电机的VMAX
+        :param TMAX: 电机的TMAX
+        :return:
+        """
+        self.Limit_Param[Motor_Type][0] = PMAX
+        self.Limit_Param[Motor_Type][1] = VMAX
+        self.Limit_Param[Motor_Type][2] = TMAX
+
+    def refresh_motor_status(self,Motor):
+        """
+        get the motor status 获得电机状态
+        """
+        can_id_l = Motor.SlaveID & 0xff #id low 8 bits
+        can_id_h = (Motor.SlaveID >> 8) & 0xff  #id high 8 bits
+        data_buf = np.array([np.uint8(can_id_l), np.uint8(can_id_h), 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00], np.uint8)
+        self.__send_data(0x7FF, data_buf)
+        self.recv()  # receive the data from serial port
+
+    def change_motor_param(self, Motor, RID, data):
+        """
+        change the RID of the motor 改变电机的参数
+        :param Motor: Motor object 电机对象
+        :param RID: DM_variable 电机参数
+        :param data: 电机参数的值
+        :return: True or False ,True means success, False means fail
+        """
+        max_retries = 20
+        retry_interval = 0.05  #retry times
+
+        self.__write_motor_param(Motor, RID, data)
+        for _ in range(max_retries):
+            self.recv_set_param_data()
+            if Motor.SlaveID in self.motors_map and RID in self.motors_map[Motor.SlaveID].temp_param_dict:
+                if abs(self.motors_map[Motor.SlaveID].temp_param_dict[RID] - data) < 0.1:
+                    return True
+                else:
+                    return False
+            sleep(retry_interval)
+        return False
+
+    def read_motor_param(self, Motor, RID):
+        """
+        read only the RID of the motor 读取电机的内部信息例如 版本号等
+        :param Motor: Motor object 电机对象
+        :param RID: DM_variable 电机参数
+        :return: 电机参数的值
+        """
+        max_retries = 5
+        retry_interval = 0.05  #retry times
+        self.__read_RID_param(Motor, RID)
+        for _ in range(max_retries):
+            sleep(retry_interval)
+            self.recv_set_param_data()
+            if Motor.SlaveID in self.motors_map:
+                if RID in self.motors_map[Motor.SlaveID].temp_param_dict:
+                    return self.motors_map[Motor.SlaveID].temp_param_dict[RID]
+        return None
+
+    # -------------------------------------------------
+    # Extract packets from the serial data
+    def __extract_packets(self, data):          
+        frames = []
+        header = 0xAA
+        tail = 0x55
+        frame_length = 16
+        i = 0
+        remainder_pos = 0
+
+        while i <= len(data) - frame_length:
+            if data[i] == header and data[i + frame_length - 1] == tail:
+                frame = data[i:i + frame_length]
+                frames.append(frame)
+                i += frame_length
+                remainder_pos = i
+            else:
+                i += 1
+        self.data_save = data[remainder_pos:]
+        return frames
+
+
+def LIMIT_MIN_MAX(x, min, max):
+    if x <= min:
+        x = min
+    elif x > max:
+        x = max
+
+
+def float_to_uint(x: float, x_min: float, x_max: float, bits):
+    LIMIT_MIN_MAX(x, x_min, x_max)
+    span = x_max - x_min
+    data_norm = (x - x_min) / span
+    return np.uint16(data_norm * ((1 << bits) - 1))
+
+
+def uint_to_float(x: np.uint16, min: float, max: float, bits):
+    span = max - min
+    data_norm = float(x) / ((1 << bits) - 1)
+    temp = data_norm * span + min
+    return np.float32(temp)
+
+
+def float_to_uint8s(value):
+    # Pack the float into 4 bytes
+    packed = pack('f', value)
+    # Unpack the bytes into four uint8 values
+    return unpack('4B', packed)
+
+
+def data_to_uint8s(value):
+    # Check if the value is within the range of uint32
+    if isinstance(value, int) and (0 <= value <= 0xFFFFFFFF):
+        # Pack the uint32 into 4 bytes
+        packed = pack('I', value)
+    else:
+        raise ValueError("Value must be an integer within the range of uint32")
+
+    # Unpack the bytes into four uint8 values
+    return unpack('4B', packed)
+
+
+def is_in_ranges(number):
+    """
+    check if the number is in the range of uint32
+    :param number:
+    :return:
+    """
+    if (7 <= number <= 10) or (13 <= number <= 16) or (35 <= number <= 36):
+        return True
+    return False
+
+
+def uint8s_to_uint32(byte1, byte2, byte3, byte4):
+    # Pack the four uint8 values into a single uint32 value in little-endian order
+    packed = pack('<4B', byte1, byte2, byte3, byte4)
+    # Unpack the packed bytes into a uint32 value
+    return unpack('<I', packed)[0]
+
+
+def uint8s_to_float(byte1, byte2, byte3, byte4):
+    # Pack the four uint8 values into a single float value in little-endian order
+    packed = pack('<4B', byte1, byte2, byte3, byte4)
+    # Unpack the packed bytes into a float value
+    return unpack('<f', packed)[0]
+
+
+def print_hex(data):
+    hex_values = [f'{byte:02X}' for byte in data]
+    print(' '.join(hex_values))
+
+
+def get_enum_by_index(index, enum_class):
+    try:
+        return enum_class(index)
+    except ValueError:
+        return None
+
+
+class DM_Motor_Type(IntEnum):
+    DM4310 = 0
+    DM4310_48V = 1
+    DM4340 = 2
+    DM4340_48V = 3
+    DM6006 = 4
+    DM8006 = 5
+    DM8009 = 6
+    DM10010L = 7
+    DM10010 = 8
+    DMH3510 = 9
+    DMH6215 = 10
+    DMG6220 = 11
+
+
+class DM_variable(IntEnum):
+    UV_Value = 0
+    KT_Value = 1
+    OT_Value = 2
+    OC_Value = 3
+    ACC = 4
+    DEC = 5
+    MAX_SPD = 6
+    MST_ID = 7
+    ESC_ID = 8
+    TIMEOUT = 9
+    CTRL_MODE = 10
+    Damp = 11
+    Inertia = 12
+    hw_ver = 13
+    sw_ver = 14
+    SN = 15
+    NPP = 16
+    Rs = 17
+    LS = 18
+    Flux = 19
+    Gr = 20
+    PMAX = 21
+    VMAX = 22
+    TMAX = 23
+    I_BW = 24
+    KP_ASR = 25
+    KI_ASR = 26
+    KP_APR = 27
+    KI_APR = 28
+    OV_Value = 29
+    GREF = 30
+    Deta = 31
+    V_BW = 32
+    IQ_c1 = 33
+    VL_c1 = 34
+    can_br = 35
+    sub_ver = 36
+    u_off = 50
+    v_off = 51
+    k1 = 52
+    k2 = 53
+    m_off = 54
+    dir = 55
+    p_m = 80
+    xout = 81
+
+
+class Control_Type(IntEnum):
+    MIT = 1
+    POS_VEL = 2
+    VEL = 3
+    Torque_Pos = 4
+
+class DamiaoPort:
+    def __init__(self, device, types, can_ids, master_ids, motor_with_torque, control_mode=Control_Type.MIT):
+        self.device = device
+        self.types = types
+        self.can_ids = can_ids
+        self.master_ids = master_ids
+        self.control = MotorControl(self.device, bitrate=4000000)
+        self.motors = [Motor(type, can_id, master_id) for type, can_id, master_id in zip(types, can_ids, master_ids)]
+        self.stat_data = []
+        self.stat_time = []
+        for motor in self.motors:
+            self.control.addMotor(motor)
+            self.control.enable(motor)
+
+    def get_present_status(self):
+        self.stat_time.append(time())
+        stat = [[
+            motor.goal_position,
+            motor.goal_tau,
+            motor.getPosition(),
+            motor.getVelocity(),
+            motor.getTorque(),
+            motor.state_tmos,
+            motor.state_trotor,
+        ] for motor in self.motors]
+        self.stat_data.append(stat)
+
+        return stat
+
+    def save_status(self, filename):
+        np.savez(filename, np.array(self.stat_time), np.array(self.stat_data))
+
+    def disable(self):
+        for motor in self.motors:
+            self.control.disable(motor)
+
+    def shutdown(self):
+        for motor in self.motors:
+            self.control.controlMIT(motor, 0, 0, 0, 0, 0)
+        self.control.canbus.shutdown()
+
+    def set_zero_position(self):
+        for motor in self.motors:
+            self.control.disable(motor)
+        sleep(1)
+        for motor in self.motors:
+            self.control.set_zero_position(motor)
+        sleep(1)
+        for motor in self.motors:
+            self.control.enable(motor)
+        return 0
+
+    async def move_towards(self, goal_positions, kps, kds):
+        for motor, goal_position, kp, kd in zip(self.motors, goal_positions, kps, kds):
+            delta = goal_position - motor.getPosition()
+            v = motor.getVelocity()
+            tau = kp * delta - kd * v
+            motor.goal_position = goal_position
+            motor.goal_tau = tau
+            self.control.controlMIT(motor, 0, 0, 0, 0, tau)
+            await asyncio.sleep(0.00003)
+
+    def move_regressor_sync(self, regs, search_range, search_step, goal_positions, kps, kds):
+        TORQUE_SCALER=30
+        if len(self.stat_data) == 0:
+            return self.move_towards_sync(goal_positions, kps, kds)
+        for motor, reg, goal_position, kp, kd, stat in zip(
+                self.motors, regs, goal_positions, kps, kds, self.stat_data[-1]):
+            pos = motor.getPosition()
+            vel = motor.getVelocity()
+            delta = goal_position - pos
+            goal_tau = kp * delta - kd * vel
+            _goal_pos, _goal_tau, _pos, _vel, _tau = stat
+            x = np.array([[_pos, _vel, _tau, _goal_pos, _goal_tau],
+                          [pos, vel, motor.getTorque(), goal_position, goal_tau]])
+            x /= np.array([[np.pi, 10, TORQUE_SCALER, np.pi, TORQUE_SCALER]])
+            xs = []
+            for tau in np.linspace(goal_tau/TORQUE_SCALER - search_range,
+                                   goal_tau/TORQUE_SCALER + search_range,
+                                   num=search_step):
+                x_ = x.copy()
+                x_[0,4] = tau
+                xs.append(x_.flatten())
+            h = reg.predict(xs)
+            diff = h - goal_position
+            tau = TORQUE_SCALER * xs[np.argmin(diff ** 2)][4]
+            goal_tau = tau
+            motor.goal_position = goal_position
+            motor.goal_tau = goal_tau
+            self.control.controlMIT(motor, 0, 0, 0, 0, goal_tau)
+            sleep(0.00003)
+
+    def move_towards_sync(self, goal_positions, kps, kds):
+        for motor, goal_position, kp, kd in zip(self.motors, goal_positions, kps, kds):
+            delta = goal_position - motor.getPosition()
+            v = motor.getVelocity()
+            tau = kp * delta - kd * v
+            motor.goal_position = goal_position
+            motor.goal_tau = tau
+            self.control.controlMIT(motor, 0, 0, 0, 0, tau)
+            # sleep(0.00003)
+
+    def set_goal_torque_sync(self, goal_taus):
+        for motor, goal_tau in zip(self.motors, goal_taus):
+            motor.goal_position = 0
+            motor.goal_tau = goal_tau
+            self.control.controlMIT(motor, 0, 0, 0, 0, motor.goal_tau)
+            sleep(0.00003)
+
+    def move_torque_sync(self, taus):
+        for motor,tau in zip(self.motors, taus):
+            motor.goal_position = 0
+            motor.goal_tau = tau
+            self.control.controlMIT(motor, 0, 0, 0, 0, motor.goal_tau)
+            sleep(0.00003)
+
+    def keep_torque_sync(self):
+        for motor in self.motors:
+            self.control.controlMIT(motor, 0, 0, 0, 0, motor.goal_tau)
+            sleep(0.00003)
+
+    async def set_goal_positions(self, goal_positions, kps):
+        for motor, goal_position, kp in zip(self.motors, goal_positions, kps):
+            motor.goal_position = goal_position
+            motor.goal_tau = 0
+            self.control.controlMIT(motor, kp, 1.2, goal_position, 0, 0)
+            await asyncio.sleep(0.00003)
+
+    def set_goal_positions_sync(self, goal_positions, kps, kds):
+        for motor, goal_position, kp, kd in zip(self.motors, goal_positions, kps, kds):
+            motor.goal_position = goal_position
+            motor.goal_tau = 0
+            self.control.controlMIT(motor, kp, kd, goal_position, 0, 0)
+            sleep(0.00003)
+
+    def set_goal_posvel(self, goal_positions):
+        for motor, goal_position in zip(self.motors, goal_positions):
+            motor.goal_position = goal_position
+            motor.goal_tau = 0
+            self.control.control_pos_force(motor, goal_position, 1, 1)
+
+    def controlMIT(self, motor, kp, kd, q, dq, tau):
+        self.control.controlMIT(self.motors[motor], kp, kd, q, dq, tau)
+

src/lerobot/motors/damiao/tables.py

@@ -0,0 +1,209 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Configuration tables for Damiao motors."""
+
+from enum import IntEnum
+from typing import Dict, List, Tuple
+
+# Motor type definitions
+class MotorType(IntEnum):
+    DM3507 = 0
+    DM4310 = 1
+    DM4310_48V = 2
+    DM4340 = 3
+    DM4340_48V = 4
+    DM6006 = 5
+    DM8006 = 6
+    DM8009 = 7
+    DM10010L = 8
+    DM10010 = 9
+    DMH3510 = 10
+    DMH6215 = 11
+    DMG6220 = 12
+
+# Control modes
+class ControlMode(IntEnum):
+    MIT = 1
+    POS_VEL = 2
+    VEL = 3
+    TORQUE_POS = 4
+
+# Motor variable IDs (RID)
+class MotorVariable(IntEnum):
+    UV_VALUE = 0
+    KT_VALUE = 1
+    OT_VALUE = 2
+    OC_VALUE = 3
+    ACC = 4
+    DEC = 5
+    MAX_SPD = 6
+    MST_ID = 7
+    ESC_ID = 8
+    TIMEOUT = 9
+    CTRL_MODE = 10
+    DAMP = 11
+    INERTIA = 12
+    HW_VER = 13
+    SW_VER = 14
+    SN = 15
+    NPP = 16
+    RS = 17
+    LS = 18
+    FLUX = 19
+    GR = 20
+    PMAX = 21
+    VMAX = 22
+    TMAX = 23
+    I_BW = 24
+    KP_ASR = 25
+    KI_ASR = 26
+    KP_APR = 27
+    KI_APR = 28
+    OV_VALUE = 29
+    GREF = 30
+    DETA = 31
+    V_BW = 32
+    IQ_C1 = 33
+    VL_C1 = 34
+    CAN_BR = 35
+    SUB_VER = 36
+    U_OFF = 50
+    V_OFF = 51
+    K1 = 52
+    K2 = 53
+    M_OFF = 54
+    DIR = 55
+    P_M = 80
+    XOUT = 81
+
+# Motor limit parameters [PMAX, VMAX, TMAX] 
+# PMAX: Maximum position (rad)
+# VMAX: Maximum velocity (rad/s)
+# TMAX: Maximum torque (N·m)
+MOTOR_LIMIT_PARAMS = {
+    MotorType.DM3507: (12.5, 30, 10),
+    MotorType.DM4310: (12.5, 30, 10),
+    MotorType.DM4310_48V: (12.5, 50, 10),
+    MotorType.DM4340: (12.5, 8, 28),
+    MotorType.DM4340_48V: (12.5, 10, 28),
+    MotorType.DM6006: (12.5, 45, 20),
+    MotorType.DM8006: (12.5, 45, 40),
+    MotorType.DM8009: (12.5, 45, 54),
+    MotorType.DM10010L: (12.5, 25, 200),
+    MotorType.DM10010: (12.5, 20, 200),
+    MotorType.DMH3510: (12.5, 280, 1),
+    MotorType.DMH6215: (12.5, 45, 10),
+    MotorType.DMG6220: (12.5, 45, 10),
+}
+
+# Motor model names
+MODEL_NAMES = {
+    MotorType.DM3507: "dm3507",
+    MotorType.DM4310: "dm4310",
+    MotorType.DM4310_48V: "dm4310_48v",
+    MotorType.DM4340: "dm4340",
+    MotorType.DM4340_48V: "dm4340_48v",
+    MotorType.DM6006: "dm6006",
+    MotorType.DM8006: "dm8006",
+    MotorType.DM8009: "dm8009",
+    MotorType.DM10010L: "dm10010l",
+    MotorType.DM10010: "dm10010",
+    MotorType.DMH3510: "dmh3510",
+    MotorType.DMH6215: "dmh6215",
+    MotorType.DMG6220: "dmg6220",
+}
+
+# Motor resolution table (encoder counts per revolution)
+MODEL_RESOLUTION = {
+    "dm3507": 65536,
+    "dm4310": 65536,
+    "dm4310_48v": 65536,
+    "dm4340": 65536,
+    "dm4340_48v": 65536,
+    "dm6006": 65536,
+    "dm8006": 65536,
+    "dm8009": 65536,
+    "dm10010l": 65536,
+    "dm10010": 65536,
+    "dmh3510": 65536,
+    "dmh6215": 65536,
+    "dmg6220": 65536,
+}
+
+# CAN baudrates supported by Damiao motors
+AVAILABLE_BAUDRATES = [
+    125000,   # 0: 125 kbps
+    200000,   # 1: 200 kbps
+    250000,   # 2: 250 kbps
+    500000,   # 3: 500 kbps
+    1000000,  # 4: 1 mbps (default for OpenArms)
+    2000000,  # 5: 2 mbps
+    2500000,  # 6: 2.5 mbps
+    3200000,  # 7: 3.2 mbps
+    4000000,  # 8: 4 mbps
+    5000000,  # 9: 5 mbps
+]
+DEFAULT_BAUDRATE = 1000000  # 1 Mbps is standard for OpenArms
+
+# Default timeout in milliseconds
+DEFAULT_TIMEOUT_MS = 1000
+
+# Data that should be normalized
+NORMALIZED_DATA = ["Present_Position", "Goal_Position"]
+
+# OpenArms specific configurations
+# Based on: https://docs.openarm.dev/software/setup/configure-test
+# OpenArms has 7 DOF per arm (14 total for dual arm)
+OPENARMS_ARM_MOTOR_IDS = {
+    "joint_1": {"send": 0x01, "recv": 0x11},  # J1 - Shoulder pan
+    "joint_2": {"send": 0x02, "recv": 0x12},  # J2 - Shoulder lift
+    "joint_3": {"send": 0x03, "recv": 0x13},  # J3 - Elbow flex
+    "joint_4": {"send": 0x04, "recv": 0x14},  # J4 - Wrist flex
+    "joint_5": {"send": 0x05, "recv": 0x15},  # J5 - Wrist roll
+    "joint_6": {"send": 0x06, "recv": 0x16},  # J6 - Wrist pitch
+    "joint_7": {"send": 0x07, "recv": 0x17},  # J7 - Wrist rotation
+}
+
+OPENARMS_GRIPPER_MOTOR_IDS = {
+    "gripper": {"send": 0x08, "recv": 0x18},  # J8 - Gripper
+}
+
+# Default motor types for OpenArms
+OPENARMS_DEFAULT_MOTOR_TYPES = {
+    "joint_1": MotorType.DM8009,   # Shoulder pan - high torque
+    "joint_2": MotorType.DM8009,   # Shoulder lift - high torque
+    "joint_3": MotorType.DM4340,   # Shoulder rotation
+    "joint_4": MotorType.DM4340,   # Elbow flex
+    "joint_5": MotorType.DM4310,   # Wrist roll
+    "joint_6": MotorType.DM4310,   # Wrist pitch
+    "joint_7": MotorType.DM4310,   # Wrist rotation
+    "gripper": MotorType.DM4310,   # Gripper
+}
+
+# MIT control parameter ranges
+MIT_KP_RANGE = (0.0, 500.0)
+MIT_KD_RANGE = (0.0, 5.0)
+
+# CAN frame command IDs
+CAN_CMD_ENABLE = 0xFC
+CAN_CMD_DISABLE = 0xFD
+CAN_CMD_SET_ZERO = 0xFE
+CAN_CMD_REFRESH = 0xCC
+CAN_CMD_QUERY_PARAM = 0x33
+CAN_CMD_WRITE_PARAM = 0x55
+CAN_CMD_SAVE_PARAM = 0xAA
+
+# CAN ID for parameter operations
+CAN_PARAM_ID = 0x7FF

src/lerobot/motors/dynamixel/dynamixel.py

@@ -24,7 +24,7 @@ from enum import Enum
 
 from lerobot.motors.encoding_utils import decode_twos_complement, encode_twos_complement
 
-from ..motors_bus import Motor, MotorCalibration, MotorsBus, NameOrID, Value, get_address
+from ..motors_bus import Motor, MotorCalibration, NameOrID, SerialMotorsBus, Value, get_address
 from .tables import (
     AVAILABLE_BAUDRATES,
     MODEL_BAUDRATE_TABLE,
@@ -100,7 +100,7 @@ def _split_into_byte_chunks(value: int, length: int) -> list[int]:
     return data
 
 
-class DynamixelMotorsBus(MotorsBus):
+class DynamixelMotorsBus(SerialMotorsBus):
     """
     The Dynamixel implementation for a MotorsBus. It relies on the python dynamixel sdk to communicate with
     the motors. For more info, see the Dynamixel SDK Documentation:

src/lerobot/motors/feetech/feetech.py

@@ -19,7 +19,7 @@ from pprint import pformat
 
 from lerobot.motors.encoding_utils import decode_sign_magnitude, encode_sign_magnitude
 
-from ..motors_bus import Motor, MotorCalibration, MotorsBus, NameOrID, Value, get_address
+from ..motors_bus import Motor, MotorCalibration, NameOrID, SerialMotorsBus, Value, get_address
 from .tables import (
     FIRMWARE_MAJOR_VERSION,
     FIRMWARE_MINOR_VERSION,
@@ -96,7 +96,7 @@ def patch_setPacketTimeout(self, packet_length):  # noqa: N802
     self.packet_timeout = (self.tx_time_per_byte * packet_length) + (self.tx_time_per_byte * 3.0) + 50
 
 
-class FeetechMotorsBus(MotorsBus):
+class FeetechMotorsBus(SerialMotorsBus):
     """
     The FeetechMotorsBus class allows to efficiently read and write to the attached motors. It relies on the
     python feetech sdk to communicate with the motors, which is itself based on the dynamixel sdk.

src/lerobot/motors/motors_bus.py

@@ -19,6 +19,8 @@
 # TODO(aliberts): Add block noqa when feature below is available
 # https://github.com/astral-sh/ruff/issues/3711
 
+from __future__ import annotations
+
 import abc
 import logging
 from contextlib import contextmanager
@@ -41,6 +43,92 @@ Value: TypeAlias = int | float
 logger = logging.getLogger(__name__)
 
 
+class MotorsBusBase(abc.ABC):
+    """
+    Base class for all motor bus implementations.
+    
+    This is a minimal interface that all motor buses must implement, regardless of their
+    communication protocol (serial, CAN, etc.).
+    """
+
+    def __init__(
+        self,
+        port: str,
+        motors: dict[str, Motor],
+        calibration: dict[str, MotorCalibration] | None = None,
+    ):
+        self.port = port
+        self.motors = motors
+        self.calibration = calibration if calibration else {}
+
+    @abc.abstractmethod
+    def connect(self, handshake: bool = True) -> None:
+        """Establish connection to the motors."""
+        pass
+
+    @abc.abstractmethod
+    def disconnect(self, disable_torque: bool = True) -> None:
+        """Disconnect from the motors."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def is_connected(self) -> bool:
+        """Check if connected to the motors."""
+        pass
+
+    @abc.abstractmethod
+    def read(self, data_name: str, motor: str, *, normalize: bool = True, num_retry: int = 0) -> Value:
+        """Read a value from a single motor."""
+        pass
+
+    @abc.abstractmethod
+    def write(
+        self, data_name: str, motor: str, value: Value, *, normalize: bool = True, num_retry: int = 0
+    ) -> None:
+        """Write a value to a single motor."""
+        pass
+
+    @abc.abstractmethod
+    def sync_read(
+        self, data_name: str, motors: str | list[str] | None = None, *, normalize: bool = True
+    ) -> dict[str, Value]:
+        """Read a value from multiple motors."""
+        pass
+
+    @abc.abstractmethod
+    def sync_write(
+        self,
+        data_name: str,
+        values: Value | dict[str, Value],
+        motors: str | list[str] | None = None,
+        *,
+        normalize: bool = True,
+    ) -> None:
+        """Write values to multiple motors."""
+        pass
+
+    @abc.abstractmethod
+    def enable_torque(self, motors: str | list[str] | None = None, num_retry: int = 0) -> None:
+        """Enable torque on selected motors."""
+        pass
+
+    @abc.abstractmethod
+    def disable_torque(self, motors: int | str | list[str] | None = None, num_retry: int = 0) -> None:
+        """Disable torque on selected motors."""
+        pass
+
+    @abc.abstractmethod
+    def read_calibration(self) -> dict[str, MotorCalibration]:
+        """Read calibration parameters from the motors."""
+        pass
+
+    @abc.abstractmethod
+    def write_calibration(self, calibration_dict: dict[str, MotorCalibration], cache: bool = True) -> None:
+        """Write calibration parameters to the motors."""
+        pass
+
+
 def get_ctrl_table(model_ctrl_table: dict[str, dict], model: str) -> dict[str, tuple[int, int]]:
     ctrl_table = model_ctrl_table.get(model)
     if ctrl_table is None:
@@ -203,15 +291,15 @@ class GroupSyncWrite(Protocol):
     def txPacket(self): ...
 
 
-class MotorsBus(abc.ABC):
+class SerialMotorsBus(MotorsBusBase):
     """
-    A MotorsBus allows to efficiently read and write to the attached motors.
+    A SerialMotorsBus allows to efficiently read and write to motors connected via serial communication.
     It represents several motors daisy-chained together and connected through a serial port.
-    There are currently two implementations of this abstract class:
+    There are currently two implementations of this class:
         - DynamixelMotorsBus
         - FeetechMotorsBus
 
-    Note: This class may evolve in the future should we add support for other types of bus.
+    This class is specifically for serial-based motor protocols (Dynamixel, Feetech, etc.).
 
     A MotorsBus subclass instance requires a port (e.g. `FeetechMotorsBus(port="/dev/tty.usbmodem575E0031751"`)).
     To find the port, you can run our utility script:
@@ -1212,3 +1300,7 @@ class MotorsBus(abc.ABC):
         for id_, value in ids_values.items():
             data = self._serialize_data(value, length)
             self.sync_writer.addParam(id_, data)
+
+
+# Backward compatibility alias
+MotorsBus = SerialMotorsBus

src/lerobot/policies/__init__.py

@@ -21,7 +21,6 @@ from .smolvla.configuration_smolvla import SmolVLAConfig as SmolVLAConfig
 from .smolvla.processor_smolvla import SmolVLANewLineProcessor
 from .tdmpc.configuration_tdmpc import TDMPCConfig as TDMPCConfig
 from .vqbet.configuration_vqbet import VQBeTConfig as VQBeTConfig
-from .xvla.configuration_xvla import XVLAConfig as XVLAConfig
 
 __all__ = [
     "ACTConfig",
@@ -32,5 +31,4 @@ __all__ = [
     "TDMPCConfig",
     "VQBeTConfig",
     "GrootConfig",
-    "XVLAConfig",
 ]

src/lerobot/policies/factory.py

@@ -38,9 +38,7 @@ from lerobot.policies.sac.configuration_sac import SACConfig
 from lerobot.policies.sac.reward_model.configuration_classifier import RewardClassifierConfig
 from lerobot.policies.smolvla.configuration_smolvla import SmolVLAConfig
 from lerobot.policies.tdmpc.configuration_tdmpc import TDMPCConfig
-from lerobot.policies.utils import validate_visual_features_consistency
 from lerobot.policies.vqbet.configuration_vqbet import VQBeTConfig
-from lerobot.policies.xvla.configuration_xvla import XVLAConfig
 from lerobot.processor import PolicyAction, PolicyProcessorPipeline
 from lerobot.processor.converters import (
     batch_to_transition,
@@ -108,10 +106,6 @@ def get_policy_class(name: str) -> type[PreTrainedPolicy]:
         from lerobot.policies.groot.modeling_groot import GrootPolicy
 
         return GrootPolicy
-    elif name == "xvla":
-        from lerobot.policies.xvla.modeling_xvla import XVLAPolicy
-
-        return XVLAPolicy
     else:
         raise NotImplementedError(f"Policy with name {name} is not implemented.")
 
@@ -155,8 +149,6 @@ def make_policy_config(policy_type: str, **kwargs) -> PreTrainedConfig:
         return RewardClassifierConfig(**kwargs)
     elif policy_type == "groot":
         return GrootConfig(**kwargs)
-    elif policy_type == "xvla":
-        return XVLAConfig(**kwargs)
     else:
         raise ValueError(f"Policy type '{policy_type}' is not available.")
 
@@ -336,15 +328,6 @@ def make_pre_post_processors(
             config=policy_cfg,
             dataset_stats=kwargs.get("dataset_stats"),
         )
-    elif isinstance(policy_cfg, XVLAConfig):
-        from lerobot.policies.xvla.processor_xvla import (
-            make_xvla_pre_post_processors,
-        )
-
-        processors = make_xvla_pre_post_processors(
-            config=policy_cfg,
-            dataset_stats=kwargs.get("dataset_stats"),
-        )
 
     else:
         raise NotImplementedError(f"Processor for policy type '{policy_cfg.type}' is not implemented.")
@@ -437,7 +420,20 @@ def make_policy(
     # policy = torch.compile(policy, mode="reduce-overhead")
 
     if not rename_map:
-        validate_visual_features_consistency(cfg, features)
-        # TODO: (jadechoghari) - add a check_state(cfg, features) and check_action(cfg, features)
-
+        expected_features = set(cfg.input_features.keys()) | set(cfg.output_features.keys())
+        provided_features = set(features.keys())
+        if expected_features and provided_features != expected_features:
+            missing = expected_features - provided_features
+            extra = provided_features - expected_features
+            # TODO (jadechoghari): provide a dynamic rename map suggestion to the user.
+            raise ValueError(
+                f"Feature mismatch between dataset/environment and policy config.\n"
+                f"- Missing features: {sorted(missing) if missing else 'None'}\n"
+                f"- Extra features: {sorted(extra) if extra else 'None'}\n\n"
+                f"Please ensure your dataset and policy use consistent feature names.\n"
+                f"If your dataset uses different observation keys (e.g., cameras named differently), "
+                f"use the `--rename_map` argument, for example:\n"
+                f'  --rename_map=\'{{"observation.images.left": "observation.images.camera1", '
+                f'"observation.images.top": "observation.images.camera2"}}\''
+            )
     return policy

src/lerobot/policies/pi0/configuration_pi0.py

@@ -20,7 +20,6 @@ from lerobot.configs.policies import PreTrainedConfig
 from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
 from lerobot.optim.optimizers import AdamWConfig
 from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
 from lerobot.utils.constants import OBS_IMAGES
 
 
@@ -48,9 +47,6 @@ class PI0Config(PreTrainedConfig):
     min_period: float = 4e-3
     max_period: float = 4.0
 
-    # Real-Time Chunking (RTC) configuration
-    rtc_config: RTCConfig | None = None
-
     image_resolution: tuple[int, int] = (224, 224)  # see openpi `preprocessing_pytorch.py`
 
     # Add empty images. Used to add empty cameras when no image features are present.

src/lerobot/policies/pi0/modeling_pi0.py

@@ -19,12 +19,11 @@ import logging
 import math
 from collections import deque
 from pathlib import Path
-from typing import TYPE_CHECKING, Literal, TypedDict
+from typing import TYPE_CHECKING, Literal
 
 import torch
 import torch.nn.functional as F  # noqa: N812
 from torch import Tensor, nn
-from typing_extensions import Unpack
 
 from lerobot.utils.import_utils import _transformers_available
 
@@ -43,7 +42,6 @@ else:
 from lerobot.configs.policies import PreTrainedConfig
 from lerobot.policies.pi0.configuration_pi0 import PI0Config
 from lerobot.policies.pretrained import PreTrainedPolicy, T
-from lerobot.policies.rtc.modeling_rtc import RTCProcessor
 from lerobot.utils.constants import (
     ACTION,
     OBS_LANGUAGE_ATTENTION_MASK,
@@ -53,12 +51,6 @@ from lerobot.utils.constants import (
 )
 
 
-class ActionSelectKwargs(TypedDict, total=False):
-    inference_delay: int | None
-    prev_chunk_left_over: Tensor | None
-    execution_horizon: int | None
-
-
 def get_safe_dtype(target_dtype, device_type):
     """Get a safe dtype for the given device type."""
     if device_type == "mps" and target_dtype == torch.float64:
@@ -511,10 +503,9 @@ class PaliGemmaWithExpertModel(
 class PI0Pytorch(nn.Module):  # see openpi `PI0Pytorch`
     """Core PI0 PyTorch model."""
 
-    def __init__(self, config: PI0Config, rtc_processor: RTCProcessor | None = None):
+    def __init__(self, config: PI0Config):
         super().__init__()
         self.config = config
-        self.rtc_processor = rtc_processor
 
         paligemma_config = get_gemma_config(config.paligemma_variant)
         action_expert_config = get_gemma_config(config.action_expert_variant)
@@ -569,9 +560,6 @@ class PI0Pytorch(nn.Module):  # see openpi `PI0Pytorch`
         self.paligemma_with_expert.gemma_expert.model.gradient_checkpointing = False
         logging.info("Disabled gradient checkpointing for PI0Pytorch model")
 
-    def _rtc_enabled(self):
-        return self.config.rtc_config is not None and self.config.rtc_config.enabled
-
     def _apply_checkpoint(self, func, *args, **kwargs):
         """Helper method to apply gradient checkpointing if enabled."""
         if self.gradient_checkpointing_enabled and self.training:
@@ -768,15 +756,7 @@ class PI0Pytorch(nn.Module):  # see openpi `PI0Pytorch`
 
     @torch.no_grad()  # see openpi `sample_actions` (slightly adapted)
     def sample_actions(
-        self,
-        images,
-        img_masks,
-        lang_tokens,
-        lang_masks,
-        state,
-        noise=None,
-        num_steps=None,
-        **kwargs: Unpack[ActionSelectKwargs],
+        self, images, img_masks, lang_tokens, lang_masks, state, noise=None, num_steps=None
     ) -> Tensor:
         """Do a full inference forward and compute the action."""
         if num_steps is None:
@@ -818,41 +798,14 @@ class PI0Pytorch(nn.Module):  # see openpi `PI0Pytorch`
         time = torch.tensor(1.0, dtype=torch.float32, device=device)
         while time >= -dt / 2:
             expanded_time = time.expand(bsize)
-
-            # Define a closure function to properly capture expanded_time
-            # This avoids the lambda expression (E731) and loop variable binding (B023) issues
-            def denoise_step_partial_call(input_x_t, current_timestep=expanded_time):
-                return self.denoise_step(
-                    state=state,
-                    prefix_pad_masks=prefix_pad_masks,
-                    past_key_values=past_key_values,
-                    x_t=input_x_t,
-                    timestep=current_timestep,
-                )
-
-            if self._rtc_enabled():
-                inference_delay = kwargs.get("inference_delay")
-                prev_chunk_left_over = kwargs.get("prev_chunk_left_over")
-                execution_horizon = kwargs.get("execution_horizon")
-
-                v_t = self.rtc_processor.denoise_step(
-                    x_t=x_t,
-                    prev_chunk_left_over=prev_chunk_left_over,
-                    inference_delay=inference_delay,
-                    time=time,
-                    original_denoise_step_partial=denoise_step_partial_call,
-                    execution_horizon=execution_horizon,
-                )
-            else:
-                v_t = denoise_step_partial_call(x_t)
-
-            # Euler step
-            x_t += dt * v_t
-
-            # Record x_t and v_t after Euler step
-            if self.rtc_processor is not None and self.rtc_processor.is_debug_enabled():
-                self.rtc_processor.track(time=time, x_t=x_t, v_t=v_t)
-
+            v_t = self.denoise_step(
+                state,
+                prefix_pad_masks,
+                past_key_values,
+                x_t,
+                expanded_time,
+            )
+            x_t = x_t + dt * v_t
             time += dt
 
         return x_t
@@ -916,8 +869,7 @@ class PI0Policy(PreTrainedPolicy):
         self.config = config
 
         # Initialize the core PI0 model
-        self.init_rtc_processor()
-        self.model = PI0Pytorch(config, rtc_processor=self.rtc_processor)
+        self.model = PI0Pytorch(config)
 
         # Enable gradient checkpointing if requested
         if config.gradient_checkpointing:
@@ -1107,22 +1059,6 @@ class PI0Policy(PreTrainedPolicy):
             ACTION: deque(maxlen=self.config.n_action_steps),
         }
 
-    def init_rtc_processor(self):
-        """Initialize RTC processor if RTC is enabled in config."""
-        self.rtc_processor = None
-
-        # Create processor if config provided
-        # If RTC is not enabled - we can still track the denoising data
-        if self.config.rtc_config is not None:
-            self.rtc_processor = RTCProcessor(self.config.rtc_config)
-
-            model_value = getattr(self, "model", None)
-            if model_value is not None:
-                model_value.rtc_processor = self.rtc_processor
-
-    def _rtc_enabled(self) -> bool:
-        return self.config.rtc_config is not None and self.config.rtc_config.enabled
-
     def _preprocess_images(self, batch: dict[str, Tensor]) -> tuple[list[Tensor], list[Tensor]]:
         """Preprocess images for the model.
 
@@ -1201,10 +1137,6 @@ class PI0Policy(PreTrainedPolicy):
     @torch.no_grad()
     def select_action(self, batch: dict[str, Tensor]) -> Tensor:
         """Select a single action given environment observations."""
-        assert not self._rtc_enabled(), (
-            "RTC is not supported for select_action, use it with predict_action_chunk"
-        )
-
         self.eval()
 
         # Action queue logic for n_action_steps > 1
@@ -1216,7 +1148,7 @@ class PI0Policy(PreTrainedPolicy):
         return self._action_queue.popleft()
 
     @torch.no_grad()
-    def predict_action_chunk(self, batch: dict[str, Tensor], **kwargs: Unpack[ActionSelectKwargs]) -> Tensor:
+    def predict_action_chunk(self, batch: dict[str, Tensor]) -> Tensor:
         """Predict a chunk of actions given environment observations."""
         self.eval()
 
@@ -1225,8 +1157,8 @@ class PI0Policy(PreTrainedPolicy):
         lang_tokens, lang_masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
         state = self.prepare_state(batch)
 
-        # Sample actions using the model (pass through RTC kwargs)
-        actions = self.model.sample_actions(images, img_masks, lang_tokens, lang_masks, state, **kwargs)
+        # Sample actions using the model
+        actions = self.model.sample_actions(images, img_masks, lang_tokens, lang_masks, state)
 
         # Unpad actions to actual action dimension
         original_action_dim = self.config.output_features[ACTION].shape[0]

src/lerobot/policies/pi05/configuration_pi05.py

@@ -20,7 +20,6 @@ from lerobot.configs.policies import PreTrainedConfig
 from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
 from lerobot.optim.optimizers import AdamWConfig
 from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
 
 
 @PreTrainedConfig.register_subclass("pi05")
@@ -47,9 +46,6 @@ class PI05Config(PreTrainedConfig):
     min_period: float = 4e-3
     max_period: float = 4.0
 
-    # Real-Time Chunking (RTC) configuration
-    rtc_config: RTCConfig | None = None
-
     image_resolution: tuple[int, int] = (224, 224)  # see openpi `preprocessing_pytorch.py`
 
     # Add empty images. Used to add empty cameras when no image features are present.

src/lerobot/policies/pi05/modeling_pi05.py

@@ -19,12 +19,11 @@ import logging
 import math
 from collections import deque
 from pathlib import Path
-from typing import TYPE_CHECKING, Literal, TypedDict
+from typing import TYPE_CHECKING, Literal
 
 import torch
 import torch.nn.functional as F  # noqa: N812
 from torch import Tensor, nn
-from typing_extensions import Unpack
 
 from lerobot.utils.import_utils import _transformers_available
 
@@ -43,7 +42,6 @@ else:
 from lerobot.configs.policies import PreTrainedConfig
 from lerobot.policies.pi05.configuration_pi05 import PI05Config
 from lerobot.policies.pretrained import PreTrainedPolicy, T
-from lerobot.policies.rtc.modeling_rtc import RTCProcessor
 from lerobot.utils.constants import (
     ACTION,
     OBS_LANGUAGE_ATTENTION_MASK,
@@ -52,12 +50,6 @@ from lerobot.utils.constants import (
 )
 
 
-class ActionSelectKwargs(TypedDict, total=False):
-    inference_delay: int | None
-    prev_chunk_left_over: Tensor | None
-    execution_horizon: int | None
-
-
 def get_safe_dtype(target_dtype, device_type):
     """Get a safe dtype for the given device type."""
     if device_type == "mps" and target_dtype == torch.float64:
@@ -510,10 +502,9 @@ class PaliGemmaWithExpertModel(
 class PI05Pytorch(nn.Module):  # see openpi `PI0Pytorch`
     """Core PI05 PyTorch model."""
 
-    def __init__(self, config: PI05Config, rtc_processor: RTCProcessor | None = None):
+    def __init__(self, config: PI05Config):
         super().__init__()
         self.config = config
-        self.rtc_processor = rtc_processor
 
         paligemma_config = get_gemma_config(config.paligemma_variant)
         action_expert_config = get_gemma_config(config.action_expert_variant)
@@ -565,9 +556,6 @@ class PI05Pytorch(nn.Module):  # see openpi `PI0Pytorch`
         self.paligemma_with_expert.gemma_expert.model.gradient_checkpointing = False
         logging.info("Disabled gradient checkpointing for PI05Pytorch model")
 
-    def _rtc_enabled(self):
-        return self.config.rtc_config is not None and self.config.rtc_config.enabled
-
     def _apply_checkpoint(self, func, *args, **kwargs):
         """Helper method to apply gradient checkpointing if enabled."""
         if self.gradient_checkpointing_enabled and self.training:
@@ -743,16 +731,7 @@ class PI05Pytorch(nn.Module):  # see openpi `PI0Pytorch`
         return F.mse_loss(u_t, v_t, reduction="none")
 
     @torch.no_grad()  # see openpi `sample_actions` (slightly adapted)
-    def sample_actions(
-        self,
-        images,
-        img_masks,
-        tokens,
-        masks,
-        noise=None,
-        num_steps=None,
-        **kwargs: Unpack[ActionSelectKwargs],
-    ) -> Tensor:
+    def sample_actions(self, images, img_masks, tokens, masks, noise=None, num_steps=None) -> Tensor:
         """Do a full inference forward and compute the action."""
         if num_steps is None:
             num_steps = self.config.num_inference_steps
@@ -791,40 +770,13 @@ class PI05Pytorch(nn.Module):  # see openpi `PI0Pytorch`
         time = torch.tensor(1.0, dtype=torch.float32, device=device)
         while time >= -dt / 2:
             expanded_time = time.expand(bsize)
-
-            # Define a closure function to properly capture expanded_time
-            # This avoids the lambda expression (E731) and loop variable binding (B023) issues
-            def denoise_step_partial_call(input_x_t, current_timestep=expanded_time):
-                return self.denoise_step(
-                    prefix_pad_masks=prefix_pad_masks,
-                    past_key_values=past_key_values,
-                    x_t=input_x_t,
-                    timestep=current_timestep,
-                )
-
-            if self._rtc_enabled():
-                inference_delay = kwargs.get("inference_delay")
-                prev_chunk_left_over = kwargs.get("prev_chunk_left_over")
-                execution_horizon = kwargs.get("execution_horizon")
-
-                v_t = self.rtc_processor.denoise_step(
-                    x_t=x_t,
-                    prev_chunk_left_over=prev_chunk_left_over,
-                    inference_delay=inference_delay,
-                    time=time,
-                    original_denoise_step_partial=denoise_step_partial_call,
-                    execution_horizon=execution_horizon,
-                )
-            else:
-                v_t = denoise_step_partial_call(x_t)
-
-            # Euler step
-            x_t += dt * v_t
-
-            # Record x_t and v_t after Euler step
-            if self.rtc_processor is not None and self.rtc_processor.is_debug_enabled():
-                self.rtc_processor.track(time=time, x_t=x_t, v_t=v_t)
-
+            v_t = self.denoise_step(
+                prefix_pad_masks,
+                past_key_values,
+                x_t,
+                expanded_time,
+            )
+            x_t = x_t + dt * v_t
             time += dt
 
         return x_t
@@ -887,8 +839,7 @@ class PI05Policy(PreTrainedPolicy):
         self.config = config
 
         # Initialize the core PI05 model
-        self.init_rtc_processor()
-        self.model = PI05Pytorch(config, rtc_processor=self.rtc_processor)
+        self.model = PI05Pytorch(config)
 
         # Enable gradient checkpointing if requested
         if config.gradient_checkpointing:
@@ -1084,22 +1035,6 @@ class PI05Policy(PreTrainedPolicy):
             ACTION: deque(maxlen=self.config.n_action_steps),
         }
 
-    def init_rtc_processor(self):
-        """Initialize RTC processor if RTC is enabled in config."""
-        self.rtc_processor = None
-
-        # Create processor if config provided
-        # If RTC is not enabled - we can still track the denoising data
-        if self.config.rtc_config is not None:
-            self.rtc_processor = RTCProcessor(self.config.rtc_config)
-
-            model_value = getattr(self, "model", None)
-            if model_value is not None:
-                model_value.rtc_processor = self.rtc_processor
-
-    def _rtc_enabled(self) -> bool:
-        return self.config.rtc_config is not None and self.config.rtc_config.enabled
-
     def _preprocess_images(self, batch: dict[str, Tensor]) -> tuple[list[Tensor], list[Tensor]]:
         """Preprocess images for the model.
 
@@ -1174,10 +1109,6 @@ class PI05Policy(PreTrainedPolicy):
     @torch.no_grad()
     def select_action(self, batch: dict[str, Tensor]) -> Tensor:
         """Select a single action given environment observations."""
-        assert not self._rtc_enabled(), (
-            "RTC is not supported for select_action, use it with predict_action_chunk"
-        )
-
         self.eval()
 
         # Action queue logic for n_action_steps > 1
@@ -1189,7 +1120,7 @@ class PI05Policy(PreTrainedPolicy):
         return self._action_queue.popleft()
 
     @torch.no_grad()
-    def predict_action_chunk(self, batch: dict[str, Tensor], **kwargs: Unpack[ActionSelectKwargs]) -> Tensor:
+    def predict_action_chunk(self, batch: dict[str, Tensor]) -> Tensor:
         """Predict a chunk of actions given environment observations."""
         self.eval()
 
@@ -1197,8 +1128,8 @@ class PI05Policy(PreTrainedPolicy):
         images, img_masks = self._preprocess_images(batch)
         tokens, masks = batch[f"{OBS_LANGUAGE_TOKENS}"], batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
 
-        # Sample actions using the model (pass through RTC kwargs, no separate state needed for PI05)
-        actions = self.model.sample_actions(images, img_masks, tokens, masks, **kwargs)
+        # Sample actions using the model (no separate state needed for PI05)
+        actions = self.model.sample_actions(images, img_masks, tokens, masks)
 
         # Unpad actions to actual action dimension
         original_action_dim = self.config.output_features[ACTION].shape[0]

src/lerobot/policies/rtc/README.md

@@ -1,38 +0,0 @@
-# Real-Time Chunking (RTC)
-
-This module contains the LeRobot implementation of **Real-Time Chunking (RTC)**, an inference-time technique for flow-matching based policies.
-
-**Note**: RTC is not a policy itself, but rather an inference enhancement that works with flow-matching based policies including [π₀](../pi0/), [π₀.₅](../pi05/), and [SmolVLA](../smolvla/).
-
----
-
-## Citation
-
-If you use Real-Time Chunking in your work, please cite:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{black2025realtimeexecutionactionchunking,
-      title={Real-Time Execution of Action Chunking Flow Policies},
-      author={Kevin Black and Manuel Y. Galliker and Sergey Levine},
-      year={2025},
-      eprint={2506.07339},
-      archivePrefix={arXiv},
-      primaryClass={cs.RO},
-      url={https://arxiv.org/abs/2506.07339},
-}
-```
-
----
-
-## License
-
-This implementation follows the **Apache 2.0 License**, consistent with the LeRobot project.

src/lerobot/policies/rtc/action_queue.py

@@ -1,219 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Action queue management for Real-Time Chunking (RTC).
-
-This module provides ActionQueue, a thread-safe queue for managing action chunks
-in real-time control scenarios. It supports both RTC-enabled and non-RTC modes,
-handling action merging and leftover tracking.
-"""
-
-import logging
-from threading import Lock
-
-import torch
-from torch import Tensor
-
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
-
-logger = logging.getLogger(__name__)
-
-
-class ActionQueue:
-    """Thread-safe queue for managing action chunks in real-time control.
-
-    This queue handles two types of action sequences:
-    - Original actions: Used for RTC to compute leftovers from previous chunks
-    - Processed actions: Post-processed actions ready for robot execution
-
-    The queue operates in two modes:
-    1. RTC-enabled: Replaces the entire queue with new actions, accounting for inference delay
-    2. RTC-disabled: Appends new actions to the queue, maintaining continuity
-
-    Args:
-        cfg (RTCConfig): Configuration for Real-Time Chunking behavior.
-
-    Attributes:
-        queue (Tensor | None): Processed actions for robot rollout (time_steps, action_dim).
-        original_queue (Tensor | None): Original actions for RTC computation (time_steps, action_dim).
-        last_index (int): Current consumption index in the queue.
-    """
-
-    def __init__(self, cfg: RTCConfig):
-        """Initialize the action queue.
-
-        Args:
-            cfg: RTC configuration controlling queue behavior.
-        """
-        self.queue = None  # Processed actions for robot rollout
-        self.original_queue = None  # Original actions for RTC
-        self.lock = Lock()
-        self.last_index = 0
-        self.cfg = cfg
-
-    def get(self) -> Tensor | None:
-        """Get the next action from the queue.
-
-        Returns:
-            Tensor | None: The next action (action_dim,) or None if queue is empty.
-                          Returns a clone to prevent external modifications.
-        """
-        with self.lock:
-            if self.queue is None or self.last_index >= len(self.queue):
-                return None
-
-            action = self.queue[self.last_index]
-            self.last_index += 1
-            return action.clone()
-
-    def qsize(self) -> int:
-        """Get the number of remaining actions in the queue.
-
-        Returns:
-            int: Number of unconsumed actions.
-        """
-        if self.queue is None:
-            return 0
-        length = len(self.queue)
-        return length - self.last_index
-
-    def empty(self) -> bool:
-        """Check if the queue is empty.
-
-        Returns:
-            bool: True if no actions remain, False otherwise.
-        """
-        if self.queue is None:
-            return True
-
-        length = len(self.queue)
-        return length - self.last_index <= 0
-
-    def get_action_index(self) -> int:
-        """Get the current action consumption index.
-
-        Returns:
-            int: Index of the next action to be consumed.
-        """
-        return self.last_index
-
-    def get_left_over(self) -> Tensor | None:
-        """Get leftover original actions for RTC prev_chunk_left_over.
-
-        These are the unconsumed actions from the current chunk, which will be
-        used by RTC to compute corrections for the next chunk.
-
-        Returns:
-            Tensor | None: Remaining original actions (remaining_steps, action_dim),
-                          or None if no original queue exists.
-        """
-        with self.lock:
-            if self.original_queue is None:
-                return None
-            return self.original_queue[self.last_index :]
-
-    def merge(
-        self,
-        original_actions: Tensor,
-        processed_actions: Tensor,
-        real_delay: int,
-        action_index_before_inference: int | None = 0,
-    ):
-        """Merge new actions into the queue.
-
-        This method operates differently based on RTC mode:
-        - RTC enabled: Replaces the queue, accounting for inference delay
-        - RTC disabled: Appends to the queue, maintaining continuity
-
-        Args:
-            original_actions: Unprocessed actions from policy (time_steps, action_dim).
-            processed_actions: Post-processed actions for robot (time_steps, action_dim).
-            real_delay: Number of time steps of inference delay.
-            action_index_before_inference: Index before inference started, for validation.
-        """
-        with self.lock:
-            self._check_delays(real_delay, action_index_before_inference)
-
-            if self.cfg.enabled:
-                self._replace_actions_queue(original_actions, processed_actions, real_delay)
-                return
-
-            self._append_actions_queue(original_actions, processed_actions)
-
-    def _replace_actions_queue(self, original_actions: Tensor, processed_actions: Tensor, real_delay: int):
-        """Replace the queue with new actions (RTC mode).
-
-        Discards the first `real_delay` actions since they correspond to the time
-        spent during inference, when the robot was executing previous actions.
-
-        Args:
-            original_actions: Unprocessed actions from policy.
-            processed_actions: Post-processed actions for robot.
-            real_delay: Number of time steps to skip due to inference delay.
-        """
-        self.original_queue = original_actions[real_delay:].clone()
-        self.queue = processed_actions[real_delay:].clone()
-
-        logger.debug(f"original_actions shape: {self.original_queue.shape}")
-        logger.debug(f"processed_actions shape: {self.queue.shape}")
-        logger.debug(f"real_delay: {real_delay}")
-
-        self.last_index = 0
-
-    def _append_actions_queue(self, original_actions: Tensor, processed_actions: Tensor):
-        """Append new actions to the queue (non-RTC mode).
-
-        Removes already-consumed actions and appends new ones, maintaining
-        queue continuity without replacement.
-
-        Args:
-            original_actions: Unprocessed actions from policy.
-            processed_actions: Post-processed actions for robot.
-        """
-        if self.queue is None:
-            self.original_queue = original_actions.clone()
-            self.queue = processed_actions.clone()
-            return
-
-        self.original_queue = torch.cat([self.original_queue, original_actions.clone()])
-        self.original_queue = self.original_queue[self.last_index :]
-
-        self.queue = torch.cat([self.queue, processed_actions.clone()])
-        self.queue = self.queue[self.last_index :]
-
-        self.last_index = 0
-
-    def _check_delays(self, real_delay: int, action_index_before_inference: int | None = None):
-        """Validate that computed delays match expectations.
-
-        Compares the delay computed from inference latency with the actual
-        number of actions consumed during inference.
-
-        Args:
-            real_delay: Delay computed from inference latency.
-            action_index_before_inference: Action index when inference started.
-        """
-        if action_index_before_inference is None:
-            return
-
-        indexes_diff = self.last_index - action_index_before_inference
-        if indexes_diff != real_delay:
-            # Let's check that action index difference (real delay calculated based on action queue)
-            # is the same as delay calculated based on inference latency
-            logger.warning(
-                f"[ACTION_QUEUE] Indexes diff is not equal to real delay. "
-                f"Indexes diff: {indexes_diff}, real delay: {real_delay}"
-            )

src/lerobot/policies/rtc/configuration_rtc.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Real Time Chunking (RTC) and Bidirectional Decoding (BID) configuration classes.
-
-Based on:
-- Real Time Chunking: https://www.physicalintelligence.company/research/real_time_chunking
-"""
-
-from dataclasses import dataclass
-
-from lerobot.configs.types import RTCAttentionSchedule
-
-
-@dataclass
-class RTCConfig:
-    """Configuration for Real Time Chunking (RTC) inference.
-
-    RTC improves real-time inference by treating chunk generation as an inpainting problem,
-    strategically handling overlapping timesteps between action chunks using prefix attention.
-    """
-
-    # Infrastructure
-    enabled: bool = False
-
-    # Core RTC settings
-    # Todo change to exp
-    prefix_attention_schedule: RTCAttentionSchedule = RTCAttentionSchedule.LINEAR
-    max_guidance_weight: float = 10.0
-    execution_horizon: int = 10
-
-    # Debug settings
-    debug: bool = False
-    debug_maxlen: int = 100
-
-    def __post_init__(self):
-        """Validate RTC configuration parameters."""
-        if self.max_guidance_weight <= 0:
-            raise ValueError(f"max_guidance_weight must be positive, got {self.max_guidance_weight}")
-        if self.debug_maxlen <= 0:
-            raise ValueError(f"debug_maxlen must be positive, got {self.debug_maxlen}")

src/lerobot/policies/rtc/debug_tracker.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Debug information handler for Real-Time Chunking (RTC)."""
-
-from dataclasses import dataclass, field
-from typing import Any
-
-import torch
-from torch import Tensor
-
-
-@dataclass
-class DebugStep:
-    """Container for debug information from a single denoising step.
-
-    Attributes:
-        step_idx (int): Step index/counter.
-        x_t (Tensor | None): Current latent/state tensor.
-        v_t (Tensor | None): Velocity from denoiser.
-        x1_t (Tensor | None): Denoised prediction (x_t - time * v_t).
-        correction (Tensor | None): Correction gradient tensor.
-        err (Tensor | None): Weighted error term.
-        weights (Tensor | None): Prefix attention weights.
-        guidance_weight (float | Tensor | None): Applied guidance weight.
-        time (float | Tensor | None): Time parameter.
-        inference_delay (int | None): Inference delay parameter.
-        execution_horizon (int | None): Execution horizon parameter.
-        metadata (dict[str, Any]): Additional metadata.
-    """
-
-    step_idx: int = 0
-    x_t: Tensor | None = None
-    v_t: Tensor | None = None
-    x1_t: Tensor | None = None
-    correction: Tensor | None = None
-    err: Tensor | None = None
-    weights: Tensor | None = None
-    guidance_weight: float | Tensor | None = None
-    time: float | Tensor | None = None
-    inference_delay: int | None = None
-    execution_horizon: int | None = None
-    metadata: dict[str, Any] = field(default_factory=dict)
-
-    def to_dict(self, include_tensors: bool = False) -> dict[str, Any]:
-        """Convert debug step to dictionary.
-
-        Args:
-            include_tensors (bool): If True, include tensor values. If False, only include
-                tensor statistics (shape, mean, std, min, max).
-
-        Returns:
-            Dictionary representation of the debug step.
-        """
-        result = {
-            "step_idx": self.step_idx,
-            "guidance_weight": (
-                self.guidance_weight.item()
-                if isinstance(self.guidance_weight, Tensor)
-                else self.guidance_weight
-            ),
-            "time": self.time.item() if isinstance(self.time, Tensor) else self.time,
-            "inference_delay": self.inference_delay,
-            "execution_horizon": self.execution_horizon,
-            "metadata": self.metadata.copy(),
-        }
-
-        # Add tensor information
-        tensor_fields = ["x_t", "v_t", "x1_t", "correction", "err", "weights"]
-        for field_name in tensor_fields:
-            tensor = getattr(self, field_name)
-            if tensor is not None:
-                if include_tensors:
-                    result[field_name] = tensor.detach().cpu()
-                else:
-                    result[f"{field_name}_stats"] = {
-                        "shape": tuple(tensor.shape),
-                        "mean": tensor.mean().item(),
-                        "std": tensor.std().item(),
-                        "min": tensor.min().item(),
-                        "max": tensor.max().item(),
-                    }
-
-        return result
-
-
-class Tracker:
-    """Collects and manages debug information for RTC processing.
-
-    This tracker stores debug information from recent denoising steps in a dictionary,
-    using time as the key for efficient lookups and updates.
-
-    Args:
-        enabled (bool): Whether debug collection is enabled.
-        maxlen (int | None): Optional sliding window size. If provided, only the
-            most recent ``maxlen`` debug steps are kept. If ``None``, keeps all.
-    """
-
-    def __init__(self, enabled: bool = False, maxlen: int = 100):
-        self.enabled = enabled
-        self._steps = {} if enabled else None  # Dictionary with time as key
-        self._maxlen = maxlen
-        self._step_counter = 0
-
-    def reset(self) -> None:
-        """Clear all recorded debug information."""
-        if self.enabled and self._steps is not None:
-            self._steps.clear()
-        self._step_counter = 0
-
-    @torch._dynamo.disable
-    def track(
-        self,
-        time: float | Tensor,
-        x_t: Tensor | None = None,
-        v_t: Tensor | None = None,
-        x1_t: Tensor | None = None,
-        correction: Tensor | None = None,
-        err: Tensor | None = None,
-        weights: Tensor | None = None,
-        guidance_weight: float | Tensor | None = None,
-        inference_delay: int | None = None,
-        execution_horizon: int | None = None,
-        **metadata,
-    ) -> None:
-        """Track debug information for a denoising step at a given time.
-
-        If a step with the given time already exists, it will be updated with the new data.
-        Otherwise, a new step will be created. Only non-None fields are updated/set.
-
-        Note: This method is excluded from torch.compile to avoid graph breaks from
-        operations like .item() which are incompatible with compiled graphs.
-
-        Args:
-            time (float | Tensor): Time parameter - used as the key to identify the step.
-            x_t (Tensor | None): Current latent/state tensor.
-            v_t (Tensor | None): Velocity from denoiser.
-            x1_t (Tensor | None): Denoised prediction.
-            correction (Tensor | None): Correction gradient tensor.
-            err (Tensor | None): Weighted error term.
-            weights (Tensor | None): Prefix attention weights.
-            guidance_weight (float | Tensor | None): Applied guidance weight.
-            inference_delay (int | None): Inference delay parameter.
-            execution_horizon (int | None): Execution horizon parameter.
-            **metadata: Additional metadata to store.
-        """
-        if not self.enabled:
-            return
-
-        # Convert time to float and round to avoid float precision issues
-        time_value = time.item() if isinstance(time, Tensor) else time
-        time_key = round(time_value, 6)  # Use rounded time as dictionary key
-
-        # Check if step with this time already exists
-        if time_key in self._steps:
-            # Update existing step with non-None fields
-            existing_step = self._steps[time_key]
-            if x_t is not None:
-                existing_step.x_t = x_t.detach().clone()
-            if v_t is not None:
-                existing_step.v_t = v_t.detach().clone()
-            if x1_t is not None:
-                existing_step.x1_t = x1_t.detach().clone()
-            if correction is not None:
-                existing_step.correction = correction.detach().clone()
-            if err is not None:
-                existing_step.err = err.detach().clone()
-            if weights is not None:
-                existing_step.weights = weights.detach().clone()
-            if guidance_weight is not None:
-                existing_step.guidance_weight = guidance_weight
-            if inference_delay is not None:
-                existing_step.inference_delay = inference_delay
-            if execution_horizon is not None:
-                existing_step.execution_horizon = execution_horizon
-            if metadata:
-                existing_step.metadata.update(metadata)
-        else:
-            # Create new step
-            step = DebugStep(
-                step_idx=self._step_counter,
-                x_t=x_t.detach().clone() if x_t is not None else None,
-                v_t=v_t.detach().clone() if v_t is not None else None,
-                x1_t=x1_t.detach().clone() if x1_t is not None else None,
-                correction=correction.detach().clone() if correction is not None else None,
-                err=err.detach().clone() if err is not None else None,
-                weights=weights.detach().clone() if weights is not None else None,
-                guidance_weight=guidance_weight,
-                time=time_value,
-                inference_delay=inference_delay,
-                execution_horizon=execution_horizon,
-                metadata=metadata,
-            )
-
-            # Add to dictionary
-            self._steps[time_key] = step
-            self._step_counter += 1
-
-            # Enforce maxlen if set
-            if self._maxlen is not None and len(self._steps) > self._maxlen:
-                # Remove oldest entry (first key in dict - Python 3.7+ preserves insertion order)
-                oldest_key = next(iter(self._steps))
-                del self._steps[oldest_key]
-
-    def get_all_steps(self) -> list[DebugStep]:
-        """Get all recorded debug steps.
-
-        Returns:
-            List of all DebugStep objects (may be empty if disabled).
-        """
-        if not self.enabled or self._steps is None:
-            return []
-
-        return list(self._steps.values())
-
-    def __len__(self) -> int:
-        """Return the number of recorded debug steps."""
-        if not self.enabled or self._steps is None:
-            return 0
-        return len(self._steps)

src/lerobot/policies/rtc/debug_visualizer.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Visualization utilities for RTC debug information."""
-
-import torch
-
-
-class RTCDebugVisualizer:
-    """Visualizer for RTC debug information.
-
-    This class provides methods to visualize debug information collected by the Tracker,
-    including corrections, errors, weights, and guidance weights over denoising steps.
-    """
-
-    @staticmethod
-    def plot_waypoints(
-        axes,
-        tensor,
-        start_from: int = 0,
-        color: str = "blue",
-        label: str = "",
-        alpha: float = 0.7,
-        linewidth: float = 2,
-        marker: str | None = None,
-        markersize: int = 4,
-    ):
-        """Plot trajectories across multiple dimensions.
-
-        This function plots a tensor's values across time for multiple dimensions,
-        with each dimension plotted on a separate axis.
-
-        Args:
-            axes: Array of matplotlib axes (one for each dimension).
-            tensor: The tensor to plot (can be torch.Tensor or numpy array).
-                   Shape should be (time_steps, num_dims) or (batch, time_steps, num_dims).
-            start_from: Starting index for the x-axis.
-            color: Color for the plot lines.
-            label: Label for the plot legend.
-            alpha: Transparency level for the plot.
-            linewidth: Width of the plot lines.
-            marker: Marker style for data points (e.g., 'o', 's', '^').
-            markersize: Size of the markers.
-        """
-        import numpy as np
-
-        # Handle None tensor
-        if tensor is None:
-            return
-
-        # Convert tensor to numpy if needed
-        tensor_np = tensor.detach().cpu().numpy() if isinstance(tensor, torch.Tensor) else tensor
-
-        # Handle different tensor shapes
-        if tensor_np.ndim == 3:
-            # If batch dimension present, take first batch
-            tensor_np = tensor_np[0]
-        elif tensor_np.ndim == 1:
-            # If 1D, reshape to (time_steps, 1)
-            tensor_np = tensor_np.reshape(-1, 1)
-
-        # Get dimensions
-        time_steps, num_dims = tensor_np.shape
-
-        # Create x-axis indices
-        x_indices = np.arange(start_from, start_from + time_steps)
-
-        # Plot each dimension on its corresponding axis
-        num_axes = len(axes) if hasattr(axes, "__len__") else 1
-        for dim_idx in range(min(num_dims, num_axes)):
-            ax = axes[dim_idx] if hasattr(axes, "__len__") else axes
-
-            # Plot the trajectory
-            if marker:
-                ax.plot(
-                    x_indices,
-                    tensor_np[:, dim_idx],
-                    color=color,
-                    label=label if dim_idx == 0 else "",  # Only show label once
-                    alpha=alpha,
-                    linewidth=linewidth,
-                    marker=marker,
-                    markersize=markersize,
-                )
-            else:
-                ax.plot(
-                    x_indices,
-                    tensor_np[:, dim_idx],
-                    color=color,
-                    label=label if dim_idx == 0 else "",  # Only show label once
-                    alpha=alpha,
-                    linewidth=linewidth,
-                )
-
-            # Add grid and labels if not already present
-            if not ax.xaxis.get_label().get_text():
-                ax.set_xlabel("Step", fontsize=10)
-            if not ax.yaxis.get_label().get_text():
-                ax.set_ylabel(f"Dim {dim_idx}", fontsize=10)
-            ax.grid(True, alpha=0.3)

src/lerobot/policies/rtc/latency_tracker.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Latency tracking utilities for Real-Time Chunking (RTC)."""
-
-from collections import deque
-
-import numpy as np
-
-
-class LatencyTracker:
-    """Tracks recent latencies and provides max/percentile queries.
-
-    Args:
-        maxlen (int | None): Optional sliding window size. If provided, only the
-            most recent ``maxlen`` latencies are kept. If ``None``, keeps all.
-    """
-
-    def __init__(self, maxlen: int = 100):
-        self._values = deque(maxlen=maxlen)
-        self.reset()
-
-    def reset(self) -> None:
-        """Clear all recorded latencies."""
-        self._values.clear()
-        self.max_latency = 0.0
-
-    def add(self, latency: float) -> None:
-        """Add a latency sample (seconds)."""
-        # Ensure numeric and non-negative
-        val = float(latency)
-
-        if val < 0:
-            return
-        self._values.append(val)
-        self.max_latency = max(self.max_latency, val)
-
-    def __len__(self) -> int:
-        return len(self._values)
-
-    def max(self) -> float | None:
-        """Return the maximum latency or None if empty."""
-        return self.max_latency
-
-    def percentile(self, q: float) -> float | None:
-        """Return the q-quantile (q in [0,1]) of recorded latencies or None if empty."""
-        if not self._values:
-            return 0.0
-        q = float(q)
-        if q <= 0.0:
-            return min(self._values)
-        if q >= 1.0:
-            return self.max_latency
-        vals = np.array(list(self._values), dtype=np.float32)
-        return float(np.quantile(vals, q))
-
-    def p95(self) -> float | None:
-        """Return the 95th percentile latency or None if empty."""
-        return self.percentile(0.95)

src/lerobot/policies/rtc/modeling_rtc.py

@@ -1,297 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Real-Time Chunking (RTC) implementation for LeRobot.
-
-Based on Physical Intelligence's Kinetix implementation:
-https://github.com/Physical-Intelligence/real-time-chunking-kinetix/blob/main/src/model.py#L214
-"""
-
-import logging
-import math
-
-import torch
-from torch import Tensor
-
-from lerobot.configs.types import RTCAttentionSchedule
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
-from lerobot.policies.rtc.debug_tracker import Tracker
-
-logger = logging.getLogger(__name__)
-
-
-class RTCProcessor:
-    """Real-Time Chunking processor for action chunking policies.
-
-    This class implements RTC techniques including velocity calculation,
-    prefix attention, and adaptive chunk processing.
-    """
-
-    def __init__(self, rtc_config: RTCConfig):
-        self.rtc_config = rtc_config
-
-        self.tracker = None
-
-        if rtc_config.debug:
-            self.tracker = Tracker(
-                enabled=rtc_config.debug,
-                maxlen=rtc_config.debug_maxlen,
-            )
-
-    # ====================== Tracker Proxy Methods ======================
-    def track(
-        self,
-        time: float | Tensor,
-        x_t: Tensor | None = None,
-        v_t: Tensor | None = None,
-        x1_t: Tensor | None = None,
-        correction: Tensor | None = None,
-        err: Tensor | None = None,
-        weights: Tensor | None = None,
-        guidance_weight: float | Tensor | None = None,
-        inference_delay: int | None = None,
-        execution_horizon: int | None = None,
-        **metadata,
-    ) -> None:
-        """Proxy method to track debug information.
-
-        If tracker is None or disabled, this method does nothing.
-        Otherwise, it forwards the call to tracker.track().
-        """
-        if self.tracker is not None:
-            self.tracker.track(
-                time=time,
-                x_t=x_t,
-                v_t=v_t,
-                x1_t=x1_t,
-                correction=correction,
-                err=err,
-                weights=weights,
-                guidance_weight=guidance_weight,
-                inference_delay=inference_delay,
-                execution_horizon=execution_horizon,
-                **metadata,
-            )
-
-    def get_all_debug_steps(self) -> list:
-        """Get all debug steps from tracker.
-
-        Returns empty list if tracker is disabled or None.
-        """
-        if self.tracker is not None:
-            return self.tracker.get_all_steps()
-        return []
-
-    def is_debug_enabled(self) -> bool:
-        """Check if debug tracking is enabled.
-
-        Returns True if tracker exists and is enabled.
-        """
-        return self.tracker is not None and self.tracker.enabled
-
-    def reset_tracker(self) -> None:
-        """Reset the tracker, clearing all recorded steps.
-
-        Does nothing if tracker is None.
-        """
-        if self.tracker is not None:
-            self.tracker.reset()
-
-    # ====================== End Tracker Proxy Methods ======================
-
-    def denoise_step(
-        self,
-        x_t,
-        prev_chunk_left_over,
-        inference_delay,
-        time,
-        original_denoise_step_partial,
-        execution_horizon=None,
-    ) -> Tensor:
-        """RTC guidance wrapper around an existing denoiser.
-
-        This method wraps an original denoising callable that only takes ``x_t`` and
-        returns a base denoised velocity ``v_t``. It then applies Real-Time Chunking
-        (RTC) prefix guidance using the leftover prefix from the previous chunk.
-
-        Args:
-            x_t (Tensor): Current latent/state to denoise. Shape ``(B, T, A)`` or ``(T, A)``.
-            prev_chunk_left_over (Tensor | None): Unexecuted prefix from the previous
-                chunk. Shape ``(B, T_prev, A)`` or ``(T_prev, A)``. If ``None``, no guidance
-                is applied and the method returns ``v_t`` from the original denoiser.
-            inference_delay (int): Number of timesteps from the prefix to use for guidance.
-            time (float | Tensor): Scalar in [0, 1] indicating normalized time. Must be
-                broadcastable with ``x_t``.
-            original_denoise_step_partial (Callable[[Tensor], Tensor]): Callable that
-                computes the base denoised velocity given only ``x_t``.
-            execution_horizon (int | None): Horizon used to build prefix weights. If
-                ``None``, defaults to ``self.rtc_config.execution_horizon``.
-
-        Returns:
-            Tensor: Guided velocity with the same shape as ``v_t``.
-
-        Notes:
-            - If inputs are 2D, a batch dimension is temporarily added and removed at the end.
-            - If ``prev_chunk_left_over`` is shorter than the current chunk length ``T``, it is
-              right-padded with zeros to match ``T``.
-            - Prefix weights are constructed via ``get_prefix_weights(inference_delay, execution_horizon, T)``
-              and broadcast to ``(B, T, A)``.
-            - Guidance correction is computed via autograd using ``x1_t = x_t + time * v_t`` and
-              ``error = (prev_chunk_left_over - x1_t) * weights``.
-            - The final guidance weight is clamped by ``max_guidance_weight`` from the config.
-
-        Reference:
-            https://www.physicalintelligence.company/download/real_time_chunking.pdf
-        """
-
-        # In the original implementation, the time goes from 0 to 1 and
-        # In our implementation, the time goes from 1 to 0
-        # So we need to invert the time
-        tau = 1 - time
-
-        if prev_chunk_left_over is None:
-            # First step, no guidance - return v_t
-            v_t = original_denoise_step_partial(x_t)
-            return v_t
-
-        x_t = x_t.clone().detach()
-
-        squeezed = False
-        if len(x_t.shape) < 3:
-            # Add batch dimension
-            x_t = x_t.unsqueeze(0)
-            squeezed = True
-
-        if len(prev_chunk_left_over.shape) < 3:
-            # Add batch dimension
-            prev_chunk_left_over = prev_chunk_left_over.unsqueeze(0)
-
-        if execution_horizon is None:
-            execution_horizon = self.rtc_config.execution_horizon
-
-        # If the previous action chunk is to short then it doesn't make sense to use long execution horizon
-        # because there is nothing to merge
-        if execution_horizon > prev_chunk_left_over.shape[1]:
-            execution_horizon = prev_chunk_left_over.shape[1]
-
-        batch_size = x_t.shape[0]
-        action_chunk_size = x_t.shape[1]
-        action_dim = x_t.shape[2]
-
-        if prev_chunk_left_over.shape[1] < action_chunk_size or prev_chunk_left_over.shape[2] < action_dim:
-            padded = torch.zeros(batch_size, action_chunk_size, action_dim).to(x_t.device)
-            padded[:, : prev_chunk_left_over.shape[1], : prev_chunk_left_over.shape[2]] = prev_chunk_left_over
-            prev_chunk_left_over = padded
-
-        assert prev_chunk_left_over.shape == x_t.shape, (
-            "The padded previous chunk must be the same size as the input tensor"
-        )
-
-        weights = (
-            self.get_prefix_weights(inference_delay, execution_horizon, action_chunk_size)
-            .to(x_t.device)
-            .unsqueeze(0)
-            .unsqueeze(-1)
-        )
-
-        with torch.enable_grad():
-            v_t = original_denoise_step_partial(x_t)
-            x_t.requires_grad_(True)
-
-            x1_t = x_t - time * v_t  # noqa: N806
-            err = (prev_chunk_left_over - x1_t) * weights
-            grad_outputs = err.clone().detach()
-            correction = torch.autograd.grad(x1_t, x_t, grad_outputs, retain_graph=False)[0]
-
-        max_guidance_weight = torch.as_tensor(self.rtc_config.max_guidance_weight)
-        tau_tensor = torch.as_tensor(tau)
-        squared_one_minus_tau = (1 - tau_tensor) ** 2
-        inv_r2 = (squared_one_minus_tau + tau_tensor**2) / (squared_one_minus_tau)
-        c = torch.nan_to_num((1 - tau_tensor) / tau_tensor, posinf=max_guidance_weight)
-        guidance_weight = torch.nan_to_num(c * inv_r2, posinf=max_guidance_weight)
-        guidance_weight = torch.minimum(guidance_weight, max_guidance_weight)
-
-        result = v_t - guidance_weight * correction
-
-        # Remove the batch dimension if it was added
-        if squeezed:
-            result = result.squeeze(0)
-            correction = correction.squeeze(0)
-            x1_t = x1_t.squeeze(0)
-            err = err.squeeze(0)
-
-        self.track(
-            time=time,
-            x1_t=x1_t,
-            correction=correction,
-            err=err,
-            weights=weights,
-            guidance_weight=guidance_weight,
-            inference_delay=inference_delay,
-            execution_horizon=execution_horizon,
-        )
-
-        return result
-
-    def get_prefix_weights(self, start, end, total):
-        start = min(start, end)
-
-        if self.rtc_config.prefix_attention_schedule == RTCAttentionSchedule.ZEROS:
-            weights = torch.zeros(total)
-            weights[:start] = 1.0
-        elif self.rtc_config.prefix_attention_schedule == RTCAttentionSchedule.ONES:
-            weights = torch.ones(total)
-            weights[end:] = 0.0
-        elif self.rtc_config.prefix_attention_schedule == RTCAttentionSchedule.LINEAR:
-            lin_weights = self._linweights(start, end, total)
-            weights = self._add_trailing_zeros(lin_weights, total, end)
-            weights = self._add_leading_ones(weights, start, total)
-        elif self.rtc_config.prefix_attention_schedule == RTCAttentionSchedule.EXP:
-            lin_weights = self._linweights(start, end, total)
-            lin_weights = lin_weights * torch.expm1(lin_weights).div(math.e - 1)
-            weights = self._add_trailing_zeros(lin_weights, total, end)
-            weights = self._add_leading_ones(weights, start, total)
-
-        return weights
-
-    def _linweights(self, start, end, total):
-        skip_steps_at_end = max(total - end, 0)
-
-        linspace_steps = total - skip_steps_at_end - start
-
-        if end <= start or linspace_steps <= 0:
-            return torch.tensor([])
-
-        return torch.linspace(1, 0, linspace_steps + 2)[1:-1]
-
-    def _add_trailing_zeros(self, weights, total, end):
-        zeros_len = total - end
-
-        if zeros_len <= 0:
-            return weights
-
-        zeros = torch.zeros(zeros_len)
-        return torch.cat([weights, zeros])
-
-    def _add_leading_ones(self, weights, start, total):
-        ones_len = min(start, total)
-
-        if ones_len <= 0:
-            return weights
-
-        ones = torch.ones(ones_len)
-        return torch.cat([ones, weights])

src/lerobot/policies/smolvla/configuration_smolvla.py

@@ -20,7 +20,6 @@ from lerobot.optim.optimizers import AdamWConfig
 from lerobot.optim.schedulers import (
     CosineDecayWithWarmupSchedulerConfig,
 )
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
 from lerobot.utils.constants import OBS_IMAGES
 
 
@@ -103,9 +102,6 @@ class SmolVLAConfig(PreTrainedConfig):
     min_period: float = 4e-3  # sensitivity range for the timestep used in sine-cosine positional encoding
     max_period: float = 4.0
 
-    # Real-Time Chunking (RTC) configuration
-    rtc_config: RTCConfig | None = None
-
     def __post_init__(self):
         super().__post_init__()
 

src/lerobot/policies/smolvla/modeling_smolvla.py

@@ -54,15 +54,12 @@ policy = SmolVLAPolicy.from_pretrained("lerobot/smolvla_base")
 
 import math
 from collections import deque
-from typing import TypedDict
 
 import torch
 import torch.nn.functional as F  # noqa: N812
 from torch import Tensor, nn
-from typing_extensions import Unpack
 
 from lerobot.policies.pretrained import PreTrainedPolicy
-from lerobot.policies.rtc.modeling_rtc import RTCProcessor
 from lerobot.policies.smolvla.configuration_smolvla import SmolVLAConfig
 from lerobot.policies.smolvla.smolvlm_with_expert import SmolVLMWithExpertModel
 from lerobot.policies.utils import (
@@ -72,12 +69,6 @@ from lerobot.utils.constants import ACTION, OBS_LANGUAGE_ATTENTION_MASK, OBS_LAN
 from lerobot.utils.utils import get_safe_dtype
 
 
-class ActionSelectKwargs(TypedDict, total=False):
-    inference_delay: int | None
-    prev_chunk_left_over: Tensor | None
-    execution_horizon: int | None
-
-
 def create_sinusoidal_pos_embedding(
     time: torch.tensor, dimension: int, min_period: float, max_period: float, device="cpu"
 ) -> Tensor:
@@ -241,8 +232,8 @@ class SmolVLAPolicy(PreTrainedPolicy):
         super().__init__(config)
         config.validate_features()
         self.config = config
-        self.init_rtc_processor()
-        self.model = VLAFlowMatching(config, rtc_processor=self.rtc_processor)
+
+        self.model = VLAFlowMatching(config)
         self.reset()
 
     def reset(self):
@@ -251,28 +242,10 @@ class SmolVLAPolicy(PreTrainedPolicy):
             ACTION: deque(maxlen=self.config.n_action_steps),
         }
 
-    def init_rtc_processor(self):
-        """Initialize RTC processor if RTC is enabled in config."""
-        self.rtc_processor = None
-
-        # Lets create processor if the config provided
-        # If RTC is not enabled - we still can track the denoising data
-        if self.config.rtc_config is not None:
-            self.rtc_processor = RTCProcessor(self.config.rtc_config)
-
-            # In case of calling init_rtc_processor after the model is created
-            # We need to set the rtc_processor to the model
-            # During the normal initialization process the model is not created yet
-            model_value = getattr(self, "model", None)
-            if model_value is not None:
-                model_value.rtc_processor = self.rtc_processor
-
     def get_optim_params(self) -> dict:
         return self.parameters()
 
-    def _get_action_chunk(
-        self, batch: dict[str, Tensor], noise: Tensor | None = None, **kwargs: Unpack[ActionSelectKwargs]
-    ) -> Tensor:
+    def _get_action_chunk(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor:
         # TODO: Check if this for loop is needed.
         # Context: In fact, self.queues contains only ACTION field, and in inference, we don't have action in the batch
         # In the case of offline inference, we have the action in the batch
@@ -287,9 +260,7 @@ class SmolVLAPolicy(PreTrainedPolicy):
         lang_tokens = batch[f"{OBS_LANGUAGE_TOKENS}"]
         lang_masks = batch[f"{OBS_LANGUAGE_ATTENTION_MASK}"]
 
-        actions = self.model.sample_actions(
-            images, img_masks, lang_tokens, lang_masks, state, noise=noise, **kwargs
-        )
+        actions = self.model.sample_actions(images, img_masks, lang_tokens, lang_masks, state, noise=noise)
 
         # Unpad actions
         original_action_dim = self.config.action_feature.shape[0]
@@ -307,37 +278,30 @@ class SmolVLAPolicy(PreTrainedPolicy):
         return batch
 
     @torch.no_grad()
-    def predict_action_chunk(
-        self, batch: dict[str, Tensor], noise: Tensor | None = None, **kwargs: Unpack[ActionSelectKwargs]
-    ) -> Tensor:
+    def predict_action_chunk(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor:
         self.eval()
 
         batch = self._prepare_batch(batch)
         self._queues = populate_queues(self._queues, batch, exclude_keys=[ACTION])
 
-        actions = self._get_action_chunk(batch, noise, **kwargs)
+        actions = self._get_action_chunk(batch, noise)
         return actions
 
     @torch.no_grad()
-    def select_action(
-        self, batch: dict[str, Tensor], noise: Tensor | None = None, **kwargs: Unpack[ActionSelectKwargs]
-    ) -> Tensor:
+    def select_action(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor:
         """Select a single action given environment observations.
 
         This method wraps `select_actions` in order to return one action at a time for execution in the
         environment. It works by managing the actions in a queue and only calling `select_actions` when the
         queue is empty.
         """
-
-        assert not self._rtc_enabled(), (
-            "RTC is not supported for select_action, use it with predict_action_chunk"
-        )
-
         self.eval()
         batch = self._prepare_batch(batch)
         self._queues = populate_queues(self._queues, batch, exclude_keys=[ACTION])
 
-        if self._check_get_actions_condition():
+        # Action queue logic for n_action_steps > 1. When the action_queue is depleted, populate it by
+        # querying the policy.
+        if len(self._queues[ACTION]) == 0:
             actions = self._get_action_chunk(batch, noise)
 
             # `self.predict_action_chunk` returns a (batch_size, n_action_steps, action_dim) tensor, but the queue
@@ -346,12 +310,6 @@ class SmolVLAPolicy(PreTrainedPolicy):
 
         return self._queues[ACTION].popleft()
 
-    def _check_get_actions_condition(self) -> bool:
-        return len(self._queues[ACTION]) == 0
-
-    def _rtc_enabled(self) -> bool:
-        return self.config.rtc_config is not None and self.config.rtc_config.enabled
-
     def forward(self, batch: dict[str, Tensor], noise=None, time=None) -> dict[str, Tensor]:
         """Do a full training forward pass to compute the loss"""
         if self.config.adapt_to_pi_aloha:
@@ -513,7 +471,7 @@ class VLAFlowMatching(nn.Module):
     └──────────────────────────────┘
     """
 
-    def __init__(self, config: SmolVLAConfig, rtc_processor: RTCProcessor | None = None):
+    def __init__(self, config: SmolVLAConfig):
         super().__init__()
         self.config = config
 
@@ -527,6 +485,7 @@ class VLAFlowMatching(nn.Module):
             num_vlm_layers=self.config.num_vlm_layers,
             self_attn_every_n_layers=self.config.self_attn_every_n_layers,
             expert_width_multiplier=self.config.expert_width_multiplier,
+            device=self.config.device,
         )
         self.state_proj = nn.Linear(
             self.config.max_state_dim, self.vlm_with_expert.config.text_config.hidden_size
@@ -551,10 +510,6 @@ class VLAFlowMatching(nn.Module):
         self.add_image_special_tokens = self.config.add_image_special_tokens
         self.image_end_token = torch.tensor([self.fake_image_token], dtype=torch.long)
         self.prefix_length = self.config.prefix_length
-        self.rtc_processor = rtc_processor
-
-    def _rtc_enabled(self):
-        return self.config.rtc_config is not None and self.config.rtc_config.enabled
 
     def set_requires_grad(self):
         for params in self.state_proj.parameters():
@@ -751,16 +706,7 @@ class VLAFlowMatching(nn.Module):
         losses = F.mse_loss(u_t, v_t, reduction="none")
         return losses
 
-    def sample_actions(
-        self,
-        images,
-        img_masks,
-        lang_tokens,
-        lang_masks,
-        state,
-        noise=None,
-        **kwargs: Unpack[ActionSelectKwargs],
-    ) -> Tensor:
+    def sample_actions(self, images, img_masks, lang_tokens, lang_masks, state, noise=None) -> Tensor:
         """Do a full inference forward and compute the action (batch_size x num_steps x num_motors)"""
         bsize = state.shape[0]
         device = state.device
@@ -788,45 +734,17 @@ class VLAFlowMatching(nn.Module):
 
         x_t = noise
         time = torch.tensor(1.0, dtype=torch.float32, device=device)
-
         while time >= -dt / 2:
             expanded_time = time.expand(bsize)
-
-            # Define a closure function to properly capture expanded_time
-            # This avoids the lambda expression (E731) and loop variable binding (B023) issues
-            def denoise_step_partial_call(input_x_t, current_timestep=expanded_time):
-                return self.denoise_step(
-                    x_t=input_x_t,
-                    prefix_pad_masks=prefix_pad_masks,
-                    past_key_values=past_key_values,
-                    timestep=current_timestep,
-                )
-
-            if self._rtc_enabled():
-                inference_delay = kwargs.get("inference_delay")
-                prev_chunk_left_over = kwargs.get("prev_chunk_left_over")
-                execution_horizon = kwargs.get("execution_horizon")
-
-                v_t = self.rtc_processor.denoise_step(
-                    x_t=x_t,
-                    prev_chunk_left_over=prev_chunk_left_over,
-                    inference_delay=inference_delay,
-                    time=time,
-                    original_denoise_step_partial=denoise_step_partial_call,
-                    execution_horizon=execution_horizon,
-                )
-            else:
-                v_t = denoise_step_partial_call(x_t)
-
+            v_t = self.denoise_step(
+                prefix_pad_masks,
+                past_key_values,
+                x_t,
+                expanded_time,
+            )
             # Euler step
             x_t += dt * v_t
-
-            # Record x_t and v_t after Euler step (other params are recorded in rtc_processor.denoise_step)
-            if self.rtc_processor is not None and self.rtc_processor.is_debug_enabled():
-                self.rtc_processor.track(time=time, x_t=x_t, v_t=v_t)
-
             time += dt
-
         return x_t
 
     def denoise_step(

src/lerobot/policies/utils.py

@@ -22,8 +22,6 @@ import numpy as np
 import torch
 from torch import nn
 
-from lerobot.configs.policies import PreTrainedConfig
-from lerobot.configs.types import FeatureType, PolicyFeature
 from lerobot.datasets.utils import build_dataset_frame
 from lerobot.processor import PolicyAction, RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION, OBS_STR
@@ -200,42 +198,3 @@ def make_robot_action(action_tensor: PolicyAction, ds_features: dict[str, dict])
         f"{name}": float(action_tensor[i]) for i, name in enumerate(action_names)
     }
     return act_processed_policy
-
-
-def raise_feature_mismatch_error(
-    provided_features: set[str],
-    expected_features: set[str],
-) -> None:
-    """
-    Raises a standardized ValueError for feature mismatches between dataset/environment and policy config.
-    """
-    missing = expected_features - provided_features
-    extra = provided_features - expected_features
-    # TODO (jadechoghari): provide a dynamic rename map suggestion to the user.
-    raise ValueError(
-        f"Feature mismatch between dataset/environment and policy config.\n"
-        f"- Missing features: {sorted(missing) if missing else 'None'}\n"
-        f"- Extra features: {sorted(extra) if extra else 'None'}\n\n"
-        f"Please ensure your dataset and policy use consistent feature names.\n"
-        f"If your dataset uses different observation keys (e.g., cameras named differently), "
-        f"use the `--rename_map` argument, for example:\n"
-        f'  --rename_map=\'{{"observation.images.left": "observation.images.camera1", '
-        f'"observation.images.top": "observation.images.camera2"}}\''
-    )
-
-
-def validate_visual_features_consistency(
-    cfg: PreTrainedConfig,
-    features: dict[str, PolicyFeature],
-) -> None:
-    """
-    Validates visual feature consistency between a policy config and provided dataset/environment features.
-
-    Args:
-        cfg (PreTrainedConfig): The model or policy configuration containing input_features and type.
-        features (Dict[str, PolicyFeature]): A mapping of feature names to PolicyFeature objects.
-    """
-    expected_visuals = {k for k, v in cfg.input_features.items() if v.type == FeatureType.VISUAL}
-    provided_visuals = {k for k, v in features.items() if v.type == FeatureType.VISUAL}
-    if not provided_visuals.issubset(expected_visuals):
-        raise_feature_mismatch_error(provided_visuals, expected_visuals)

src/lerobot/policies/xvla/__init__.py

@@ -1,6 +0,0 @@
-# register the processor steps
-from lerobot.policies.xvla.processor_xvla import (
-    XVLAAddDomainIdProcessorStep,
-    XVLAImageNetNormalizeProcessorStep,
-    XVLAImageToFloatProcessorStep,
-)

src/lerobot/policies/xvla/action_hub.py

@@ -1,454 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright 2025 2toINF and HuggingFace Inc. (https://github.com/2toINF)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------------
-
-from __future__ import annotations
-
-from collections.abc import Iterable
-
-import torch
-import torch.nn as nn
-
-# =============================================================================
-# Registry
-# =============================================================================
-ACTION_REGISTRY: dict[str, type[BaseActionSpace]] = {}
-
-
-def register_action(name: str):
-    """Decorator for registering a new action space."""
-
-    def _wrap(cls):
-        key = name.lower()
-        if key in ACTION_REGISTRY:
-            raise KeyError(f"ActionSpace '{key}' already registered -> {ACTION_REGISTRY[key]}")
-        ACTION_REGISTRY[key] = cls
-        cls.name = key
-        return cls
-
-    return _wrap
-
-
-def build_action_space(name: str, **kwargs) -> BaseActionSpace:
-    """Instantiate a registered action space by name."""
-    key = name.lower()
-    if key not in ACTION_REGISTRY:
-        raise KeyError(f"Unknown action space '{name}'. Available: {list(ACTION_REGISTRY.keys())}")
-    return ACTION_REGISTRY[key](**kwargs)
-
-
-# =============================================================================
-# Base class
-# =============================================================================
-class BaseActionSpace(nn.Module):
-    """
-    Abstract base class for all action-space definitions.
-
-    Each subclass defines:
-      - `dim_action`: dimension of the action vector.
-      - `gripper_idx`: indices of gripper channels.
-      - `compute_loss(pred, target)`: supervised loss for this space.
-      - `preprocess(proprio, action, mode)`: pre-step modifications.
-      - `postprocess(action)`: post-step corrections (e.g. apply sigmoid).
-    """
-
-    name: str = "base"
-    dim_action: int = 0
-    gripper_idx: tuple[int, ...] = ()
-
-    def __init__(self):
-        super().__init__()
-
-    # ---------------------------------------------------------------------
-    # Core supervised loss
-    # ---------------------------------------------------------------------
-    def compute_loss(self, pred: torch.Tensor, target: torch.Tensor) -> dict[str, torch.Tensor]:
-        raise NotImplementedError
-
-    def forward(self, pred: torch.Tensor, target: torch.Tensor) -> dict[str, torch.Tensor]:
-        """Alias for compute_loss."""
-        return self.compute_loss(pred, target)
-
-    # ---------------------------------------------------------------------
-    # Space-level hooks
-    # ---------------------------------------------------------------------
-    def preprocess(
-        self,
-        proprio: torch.Tensor,
-        action: torch.Tensor,
-        mode: str = "train",
-    ) -> tuple[torch.Tensor, torch.Tensor]:
-        """Default: return unchanged."""
-        return proprio, action
-
-    def postprocess(self, action: torch.Tensor) -> torch.Tensor:
-        """Default: return unchanged."""
-        return action
-
-
-# =============================================================================
-# Utilities
-# =============================================================================
-def _ensure_indices_valid(dim_action: int, idx: Iterable[int], name: str) -> None:
-    bad = [i for i in idx if i < 0 or i >= dim_action]
-    if bad:
-        raise IndexError(f"{name} contains out-of-range indices {bad} for action dim dim_action={dim_action}")
-
-
-# =============================================================================
-# Implementations
-# =============================================================================
-@register_action("ee6d")
-class EE6DActionSpace(BaseActionSpace):
-    """End-effector layout with xyz, 6D rotation, and gripper channels."""
-
-    dim_action = 20
-    gripper_idx = (9, 19)
-    GRIPPER_SCALE = 1.0
-    XYZ_SCALE = 500.0
-    ROT_SCALE = 10.0
-
-    POS_IDX_1 = (0, 1, 2)
-    POS_IDX_2 = (10, 11, 12)
-    ROT_IDX_1 = (3, 4, 5, 6, 7, 8)
-    ROT_IDX_2 = (13, 14, 15, 16, 17, 18)
-
-    def __init__(self):
-        super().__init__()
-        self.mse = nn.MSELoss()
-        self.bce = nn.BCEWithLogitsLoss()
-
-    def compute_loss(self, pred, target):
-        assert pred.shape == target.shape, "pred/target shapes must match"
-        batch_size, seq_len, action_dim = pred.shape
-        _ensure_indices_valid(action_dim, self.gripper_idx, "gripper_idx")
-
-        # Gripper BCE
-        g_losses = [self.bce(pred[:, :, gi], target[:, :, gi]) for gi in self.gripper_idx]
-        gripper_loss = sum(g_losses) / len(self.gripper_idx) * self.GRIPPER_SCALE
-
-        # XYZ position
-        pos_loss = (
-            self.mse(pred[:, :, self.POS_IDX_1], target[:, :, self.POS_IDX_1])
-            + self.mse(pred[:, :, self.POS_IDX_2], target[:, :, self.POS_IDX_2])
-        ) * self.XYZ_SCALE
-
-        # Rotation 6D
-        rot_loss = (
-            self.mse(pred[:, :, self.ROT_IDX_1], target[:, :, self.ROT_IDX_1])
-            + self.mse(pred[:, :, self.ROT_IDX_2], target[:, :, self.ROT_IDX_2])
-        ) * self.ROT_SCALE
-
-        return {
-            "position_loss": pos_loss,
-            "rotate6D_loss": rot_loss,
-            "gripper_loss": gripper_loss,
-        }
-
-    def preprocess(self, proprio, action, mode="train"):
-        """Zero-out gripper channels in proprio/action."""
-        proprio_m = proprio.clone()
-        action_m = action.clone()
-        proprio_m[..., self.gripper_idx] = 0.0
-        action_m[..., self.gripper_idx] = 0.0
-        return proprio_m, action_m
-
-    def postprocess(self, action: torch.Tensor) -> torch.Tensor:
-        """Apply sigmoid to gripper logits."""
-        if action.size(-1) > max(self.gripper_idx):
-            action[..., self.gripper_idx] = torch.sigmoid(action[..., self.gripper_idx])
-        return action
-
-
-@register_action("joint")
-class JointActionSpace(BaseActionSpace):
-    """Joint-space layout with joints + gripper only."""
-
-    dim_action = 14
-    gripper_idx = (6, 13)
-    GRIPPER_SCALE = 0.1
-    JOINTS_SCALE = 1.0
-
-    def __init__(self):
-        super().__init__()
-        self.mse = nn.MSELoss()
-        self.bce = nn.BCEWithLogitsLoss()
-
-    def compute_loss(self, pred, target):
-        assert pred.shape == target.shape
-        batch_size, seq_len, action_dim = pred.shape
-        _ensure_indices_valid(action_dim, self.gripper_idx, "gripper_idx")
-
-        g_losses = [self.bce(pred[:, :, gi], target[:, :, gi]) for gi in self.gripper_idx]
-        gripper_loss = sum(g_losses) / len(self.gripper_idx) * self.GRIPPER_SCALE
-
-        joints_idx = tuple(i for i in range(action_dim) if i not in set(self.gripper_idx))
-        joints_loss = self.mse(pred[:, :, joints_idx], target[:, :, joints_idx]) * self.JOINTS_SCALE
-
-        return {
-            "joints_loss": joints_loss,
-            "gripper_loss": gripper_loss,
-        }
-
-    def preprocess(self, proprio, action, mode="train"):
-        """Zero-out gripper channels in proprio/action."""
-        proprio_m = proprio.clone()
-        action_m = action.clone()
-        proprio_m[..., self.gripper_idx] = 0.0
-        action_m[..., self.gripper_idx] = 0.0
-        return proprio_m, action_m
-
-    def postprocess(self, action: torch.Tensor) -> torch.Tensor:
-        """Apply sigmoid to gripper logits."""
-        if action.size(-1) > max(self.gripper_idx):
-            action[..., self.gripper_idx] = torch.sigmoid(action[..., self.gripper_idx])
-        return action
-
-
-@register_action("agibot_ee6d")
-class AGIBOTEE6DActionSpace(BaseActionSpace):
-    """AGI-bot variant of EE6DActionSpace using MSE for all components."""
-
-    dim_action = 20
-    gripper_idx = (9, 19)
-    GRIPPER_SCALE = 10.0
-    XYZ_SCALE = 500.0
-    ROT_SCALE = 10.0
-    POS_IDX_1 = (0, 1, 2)
-    POS_IDX_2 = (10, 11, 12)
-    ROT_IDX_1 = (3, 4, 5, 6, 7, 8)
-    ROT_IDX_2 = (13, 14, 15, 16, 17, 18)
-
-    def __init__(self):
-        super().__init__()
-        self.mse = nn.MSELoss()
-
-    def compute_loss(self, pred, target):
-        assert pred.shape == target.shape
-        batch_size, seq_len, action_dim = pred.shape
-        _ensure_indices_valid(action_dim, self.gripper_idx, "gripper_idx")
-
-        gripper_loss = (
-            self.mse(pred[:, :, self.gripper_idx], target[:, :, self.gripper_idx]) * self.GRIPPER_SCALE
-        )
-        pos_loss = (
-            self.mse(pred[:, :, self.POS_IDX_1], target[:, :, self.POS_IDX_1])
-            + self.mse(pred[:, :, self.POS_IDX_2], target[:, :, self.POS_IDX_2])
-        ) * self.XYZ_SCALE
-        rot_loss = (
-            self.mse(pred[:, :, self.ROT_IDX_1], target[:, :, self.ROT_IDX_1])
-            + self.mse(pred[:, :, self.ROT_IDX_2], target[:, :, self.ROT_IDX_2])
-        ) * self.ROT_SCALE
-
-        return {
-            "position_loss": pos_loss,
-            "rotate6D_loss": rot_loss,
-            "gripper_loss": gripper_loss,
-        }
-
-    def preprocess(self, proprio, action, mode="train"):
-        """No preprocessing applied in AGIBOT variant."""
-        return proprio, action
-
-    def postprocess(self, action: torch.Tensor) -> torch.Tensor:
-        """AGIBOT does not postprocess."""
-        return action
-
-
-@register_action("franka_joint7")
-class FrankaJoint7ActionSpace(BaseActionSpace):
-    """Franka Panda joint-space: 7 joints, no gripper."""
-
-    dim_action = 7
-    JOINTS_SCALE = 1.0
-
-    def __init__(self):
-        super().__init__()
-        self.mse = nn.MSELoss()
-
-    def compute_loss(self, pred, target):
-        assert pred.shape == target.shape, "pred/target shapes must match"
-        joints_loss = self.mse(pred, target) * self.JOINTS_SCALE
-        return {"joints_loss": joints_loss}
-
-    def preprocess(self, proprio, action, mode="train"):
-        """No preprocessing needed for 7 joint actions."""
-        return proprio, action
-
-    def postprocess(self, action: torch.Tensor) -> torch.Tensor:
-        """Return directly (no sigmoid since no gripper)."""
-        return action
-
-
-@register_action("so101_bimanual")
-class BimanualSO101ActionSpace(BaseActionSpace):
-    """
-    Bimanual SO101 robot: 2 arms with 5 joints each + gripper.
-
-    Layout (real robot):
-    [left_arm (5 joints + gripper), right_arm (5 joints + gripper)]
-    - Left arm:  shoulder_pan, shoulder_lift, elbow_flex, wrist_flex, wrist_roll, gripper
-    - Right arm: shoulder_pan, shoulder_lift, elbow_flex, wrist_flex, wrist_roll, gripper
-
-    Real action dim: 12
-    Model-facing dim: 20 (extra 8 dummy dims at the end)
-    """
-
-    # Model output / training dimension (to match pretrained policy)
-    dim_action = 20
-
-    # Real robot action dimension
-    REAL_DIM = 12
-
-    # Indices of real vs dummy channels
-    REAL_IDXS = tuple(range(REAL_DIM))  # 0..11
-    DUMMY_IDXS = tuple(range(REAL_DIM, dim_action))  # 12..19
-
-    # Grippers live in the real part
-    gripper_idx = (5, 11)  # left_gripper at idx 5, right_gripper at idx 11
-    GRIPPER_SCALE = 1.0
-    JOINTS_SCALE = 1.0
-
-    # Indices for left and right arm joints (excluding grippers)
-    LEFT_ARM_JOINTS = (0, 1, 2, 3, 4)
-    RIGHT_ARM_JOINTS = (6, 7, 8, 9, 10)
-
-    def __init__(self):
-        super().__init__()
-        self.mse = nn.MSELoss()
-        self.bce = nn.BCEWithLogitsLoss()
-
-    # ---------- helpers ----------
-
-    def _pad_to_model_dim(self, x: torch.Tensor) -> torch.Tensor:
-        """If last dim is REAL_DIM (12), pad zeros to reach dim_action (20)."""
-        if x is None:
-            return None
-        if x.size(-1) == self.dim_action:
-            return x
-        if x.size(-1) != self.REAL_DIM:
-            raise ValueError(
-                f"Expected last dim to be {self.REAL_DIM} or {self.dim_action}, got {x.size(-1)}"
-            )
-        pad_shape = list(x.shape[:-1]) + [self.dim_action - self.REAL_DIM]
-        pad = x.new_zeros(pad_shape)
-        return torch.cat([x, pad], dim=-1)
-
-    def _trim_to_real_dim(self, x: torch.Tensor) -> torch.Tensor:
-        """Keep only the first REAL_DIM (12) dims for the real robot."""
-        return x[..., : self.REAL_DIM]
-
-    # ---------- loss ----------
-
-    def compute_loss(self, pred, target):
-        """
-        pred:  [B, T, 20] from the model
-        target: [B, T, 12] or [B, T, 20]
-        We pad target → 20 and compute loss only on the real dims.
-        """
-        # Ensure both are [B, T, 20]
-        pred = self._pad_to_model_dim(pred)
-        target = self._pad_to_model_dim(target)
-        assert pred.shape == target.shape
-
-        # ---- MSE for all real dims (0–11) ----
-        real_dims = 12
-
-        joints_loss = (
-            self.mse(
-                pred[:, :, :real_dims],
-                target[:, :, :real_dims],
-            )
-            * self.JOINTS_SCALE
-        )
-
-        left_arm_loss = self.mse(pred[:, :, :6], target[:, :, :6])
-        right_arm_loss = self.mse(pred[:, :, 6:12], target[:, :, 6:12])
-
-        gripper_loss = (
-            self.mse(
-                pred[:, :, [5, 11]],
-                target[:, :, [5, 11]],
-            )
-            * self.GRIPPER_SCALE
-        )
-
-        return {
-            "joints_loss": joints_loss,
-            "gripper_loss": gripper_loss,
-            "left_arm_loss": left_arm_loss,
-            "right_arm_loss": right_arm_loss,
-        }
-
-    # ---------- preprocess / postprocess ----------
-
-    def preprocess(self, proprio, action, mode="train"):
-        """
-        - If proprio/action are 12-dim, pad them to 20 for the model.
-        - Zero-out gripper channels in proprio/action to focus learning on joints.
-        """
-        proprio_m = self._pad_to_model_dim(proprio.clone())
-        action_m = self._pad_to_model_dim(action.clone()) if action is not None else None
-
-        proprio_m[..., self.gripper_idx] = 0.0
-        if action_m is not None:
-            action_m[..., self.gripper_idx] = 0.0
-
-        return proprio_m, action_m
-
-    def postprocess(self, action: torch.Tensor) -> torch.Tensor:
-        """
-        - Model outputs [*, 20]
-        - Apply sigmoid to gripper logits
-        - Return only the first 12 dims for the real robot:
-          ["left_shoulder_pan.pos",
-           "left_shoulder_lift.pos",
-           "left_elbow_flex.pos",
-           "left_wrist_flex.pos",
-           "left_wrist_roll.pos",
-           "left_gripper.pos",
-           "right_shoulder_pan.pos",
-           "right_shoulder_lift.pos",
-           "right_elbow_flex.pos",
-           "right_wrist_flex.pos",
-           "right_wrist_roll.pos",
-           "right_gripper.pos"]
-        """
-        # Ensure we at least have the real dims + grippers
-        if action.size(-1) < self.REAL_DIM:
-            raise ValueError(f"Expected at least {self.REAL_DIM} dims in action, got {action.size(-1)}")
-
-        # Apply sigmoid on gripper channels in model space (indices 5 and 11)
-        if action.size(-1) > max(self.gripper_idx):
-            action[..., self.gripper_idx] = torch.sigmoid(action[..., self.gripper_idx])
-
-        # Return only the real 12-dim control vector for the env
-        return self._trim_to_real_dim(action)
-
-
-# =============================================================================
-# Exports
-# =============================================================================
-__all__ = [
-    "BaseActionSpace",
-    "build_action_space",
-    "register_action",
-    "EE6DActionSpace",
-    "JointActionSpace",
-    "AGIBOTEE6DActionSpace",
-    "FrankaJoint7ActionSpace",
-    "BimanualSO101ActionSpace",
-    "ACTION_REGISTRY",
-]

src/lerobot/policies/xvla/configuration_florence2.py

@@ -1,353 +0,0 @@
-# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import warnings
-
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-""" Florence-2 configuration"""
-
-logger = logging.get_logger(__name__)
-
-
-class Florence2VisionConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`Florence2VisionModel`]. It is used to instantiate a Florence2VisionModel
-    according to the specified arguments, defining the model architecture. Instantiating a configuration with the
-    defaults will yield a similar configuration to that of the Florence2VisionModel architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-    Args:
-        drop_path_rate (`float`, *optional*, defaults to 0.1):
-            The dropout rate of the drop path layer.
-        patch_size (`List[int]`, *optional*, defaults to [7, 3, 3, 3]):
-            The patch size of the image.
-        patch_stride (`List[int]`, *optional*, defaults to [4, 2, 2, 2]):
-            The patch stride of the image.
-        patch_padding (`List[int]`, *optional*, defaults to [3, 1, 1, 1]):
-            The patch padding of the image.
-        patch_prenorm (`List[bool]`, *optional*, defaults to [false, true, true, true]):
-            Whether to apply layer normalization before the patch embedding layer.
-        enable_checkpoint (`bool`, *optional*, defaults to False):
-            Whether to enable checkpointing.
-        dim_embed (`List[int]`, *optional*, defaults to [256, 512, 1024, 2048]):
-            The dimension of the embedding layer.
-        num_heads (`List[int]`, *optional*, defaults to [8, 16, 32, 64]):
-            The number of attention heads.
-        num_groups (`List[int]`, *optional*, defaults to [8, 16, 32, 64]):
-            The number of groups.
-        depths (`List[int]`, *optional*, defaults to [1, 1, 9, 1]):
-            The depth of the model.
-        window_size (`int`, *optional*, defaults to 12):
-            The window size of the model.
-        projection_dim (`int`, *optional*, defaults to 1024):
-            The dimension of the projection layer.
-        visual_temporal_embedding (`dict`, *optional*):
-            The configuration of the visual temporal embedding.
-        image_pos_embed (`dict`, *optional*):
-            The configuration of the image position embedding.
-        image_feature_source (`List[str]`, *optional*, defaults to ["spatial_avg_pool", "temporal_avg_pool"]):
-            The source of the image feature.
-    Example:
-
-    ```python
-    >>> from transformers import Florence2VisionConfig, Florence2VisionModel
-
-    >>> # Initializing a Florence2 Vision style configuration
-    >>> configuration = Florence2VisionConfig()
-
-    >>> # Initializing a model (with random weights)
-    >>> model = Florence2VisionModel(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```"""
-
-    model_type = "davit"
-    keys_to_ignore_at_inference = ["past_key_values"]
-
-    def __init__(
-        self,
-        drop_path_rate=0.1,
-        patch_size=None,
-        patch_stride=None,
-        patch_padding=None,
-        patch_prenorm=None,
-        enable_checkpoint=False,
-        dim_embed=None,
-        num_heads=None,
-        num_groups=None,
-        depths=None,
-        window_size=12,
-        projection_dim=1024,
-        visual_temporal_embedding=None,
-        image_pos_embed=None,
-        image_feature_source=None,
-        **kwargs,
-    ):
-        self.drop_path_rate = drop_path_rate
-        self.patch_size = patch_size if patch_size is not None else [7, 3, 3, 3]
-        self.patch_stride = patch_stride if patch_stride is not None else [4, 2, 2, 2]
-        self.patch_padding = patch_padding if patch_padding is not None else [3, 1, 1, 1]
-        self.patch_prenorm = patch_prenorm if patch_prenorm is not None else [False, True, True, True]
-        self.enable_checkpoint = enable_checkpoint
-        self.dim_embed = dim_embed if dim_embed is not None else [256, 512, 1024, 2048]
-        self.num_heads = num_heads if num_heads is not None else [8, 16, 32, 64]
-        self.num_groups = num_groups if num_groups is not None else [8, 16, 32, 64]
-        self.depths = depths if depths is not None else [1, 1, 9, 1]
-        self.window_size = window_size
-        self.projection_dim = projection_dim
-
-        if visual_temporal_embedding is None:
-            visual_temporal_embedding = {
-                "type": "COSINE",
-                "max_temporal_embeddings": 100,
-            }
-        self.visual_temporal_embedding = visual_temporal_embedding
-
-        if image_pos_embed is None:
-            image_pos_embed = {
-                "type": "learned_abs_2d",
-                "max_pos_embeddings": 1000,
-            }
-        self.image_pos_embed = image_pos_embed
-
-        self.image_feature_source = (
-            image_feature_source
-            if image_feature_source is not None
-            else ["spatial_avg_pool", "temporal_avg_pool"]
-        )
-
-        super().__init__(**kwargs)
-
-
-class Florence2LanguageConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`Florence2LanguagePreTrainedModel`]. It is used to instantiate a BART
-    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
-    defaults will yield a similar configuration to that of the BART
-    [facebook/bart-large](https://huggingface.co/facebook/bart-large) architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-
-    Args:
-        vocab_size (`int`, *optional*, defaults to 51289):
-            Vocabulary size of the Florence2Language model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`Florence2LanguageModel`].
-        d_model (`int`, *optional*, defaults to 1024):
-            Dimensionality of the layers and the pooler layer.
-        encoder_layers (`int`, *optional*, defaults to 12):
-            Number of encoder layers.
-        decoder_layers (`int`, *optional*, defaults to 12):
-            Number of decoder layers.
-        encoder_attention_heads (`int`, *optional*, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        decoder_attention_heads (`int`, *optional*, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer decoder.
-        decoder_ffn_dim (`int`, *optional*, defaults to 4096):
-            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
-        encoder_ffn_dim (`int`, *optional*, defaults to 4096):
-            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
-        activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
-            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
-            `"relu"`, `"silu"` and `"gelu_new"` are supported.
-        dropout (`float`, *optional*, defaults to 0.1):
-            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout ratio for the attention probabilities.
-        activation_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout ratio for activations inside the fully connected layer.
-        classifier_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout ratio for classifier.
-        max_position_embeddings (`int`, *optional*, defaults to 1024):
-            The maximum sequence length that this model might ever be used with. Typically set this to something large
-            just in case (e.g., 512 or 1024 or 2048).
-        init_std (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        encoder_layerdrop (`float`, *optional*, defaults to 0.0):
-            The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
-            for more details.
-        decoder_layerdrop (`float`, *optional*, defaults to 0.0):
-            The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
-            for more details.
-        scale_embedding (`bool`, *optional*, defaults to `False`):
-            Scale embeddings by diving by sqrt(d_model).
-        use_cache (`bool`, *optional*, defaults to `True`):
-            Whether or not the model should return the last key/values attentions (not used by all models).
-        num_labels (`int`, *optional*, defaults to 3):
-            The number of labels to use in [`Florence2LanguageForSequenceClassification`].
-        forced_eos_token_id (`int`, *optional*, defaults to 2):
-            The id of the token to force as the last generated token when `max_length` is reached. Usually set to
-            `eos_token_id`.
-
-    Example:
-
-    ```python
-    >>> from transformers import Florence2LanguageConfig, Florence2LanguageModel
-
-    >>> # Initializing a Florence2 Language style configuration
-    >>> configuration = Florence2LanguageConfig()
-
-    >>> # Initializing a model (with random weights)
-    >>> model = Florence2LanguageModel(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```"""
-
-    model_type = "florence2_language"
-    keys_to_ignore_at_inference = ["past_key_values"]
-    attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}
-
-    def __init__(
-        self,
-        vocab_size=51289,
-        max_position_embeddings=1024,
-        encoder_layers=12,
-        encoder_ffn_dim=4096,
-        encoder_attention_heads=16,
-        decoder_layers=12,
-        decoder_ffn_dim=4096,
-        decoder_attention_heads=16,
-        encoder_layerdrop=0.0,
-        decoder_layerdrop=0.0,
-        activation_function="gelu",
-        d_model=1024,
-        dropout=0.1,
-        attention_dropout=0.0,
-        activation_dropout=0.0,
-        init_std=0.02,
-        classifier_dropout=0.0,
-        scale_embedding=False,
-        use_cache=True,
-        num_labels=3,
-        pad_token_id=1,
-        bos_token_id=0,
-        eos_token_id=2,
-        is_encoder_decoder=True,
-        decoder_start_token_id=2,
-        forced_eos_token_id=2,
-        **kwargs,
-    ):
-        self.vocab_size = vocab_size
-        self.max_position_embeddings = max_position_embeddings
-        self.d_model = d_model
-        self.encoder_ffn_dim = encoder_ffn_dim
-        self.encoder_layers = encoder_layers
-        self.encoder_attention_heads = encoder_attention_heads
-        self.decoder_ffn_dim = decoder_ffn_dim
-        self.decoder_layers = decoder_layers
-        self.decoder_attention_heads = decoder_attention_heads
-        self.dropout = dropout
-        self.attention_dropout = attention_dropout
-        self.activation_dropout = activation_dropout
-        self.activation_function = activation_function
-        self.init_std = init_std
-        self.encoder_layerdrop = encoder_layerdrop
-        self.decoder_layerdrop = decoder_layerdrop
-        self.classifier_dropout = classifier_dropout
-        self.use_cache = use_cache
-        self.num_hidden_layers = encoder_layers
-        self.scale_embedding = scale_embedding  # scale factor will be sqrt(d_model) if True
-
-        super().__init__(
-            num_labels=num_labels,
-            pad_token_id=pad_token_id,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            is_encoder_decoder=is_encoder_decoder,
-            decoder_start_token_id=decoder_start_token_id,
-            forced_eos_token_id=forced_eos_token_id,
-            **kwargs,
-        )
-
-        # ensure backward compatibility for BART CNN models
-        if self.forced_bos_token_id is None and kwargs.get("force_bos_token_to_be_generated", False):
-            self.forced_bos_token_id = self.bos_token_id
-            warnings.warn(
-                f"Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. "
-                "The config can simply be saved and uploaded again to be fixed.",
-                stacklevel=2,
-            )
-
-
-class Florence2Config(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`Florence2ForConditionalGeneration`]. It is used to instantiate an
-    Florence-2 model according to the specified arguments, defining the model architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-    Args:
-        vision_config (`Florence2VisionConfig`,  *optional*):
-            Custom vision config or dict
-        text_config (`Union[AutoConfig, dict]`, *optional*):
-            The config object of the text backbone.
-        ignore_index (`int`, *optional*, defaults to -100):
-            The ignore index for the loss function.
-        vocab_size (`int`, *optional*, defaults to 51289):
-            Vocabulary size of the Florence2model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`~Florence2ForConditionalGeneration`]
-        projection_dim (`int`, *optional*, defaults to 1024):
-            Dimension of the multimodal projection space.
-
-    Example:
-
-    ```python
-    >>> from transformers import Florence2ForConditionalGeneration, Florence2Config, CLIPVisionConfig, BartConfig
-
-    >>> # Initializing a clip-like vision config
-    >>> vision_config = CLIPVisionConfig()
-
-    >>> # Initializing a Bart config
-    >>> text_config = BartConfig()
-
-    >>> # Initializing a Florence-2 configuration
-    >>> configuration = Florence2Config(vision_config, text_config)
-
-    >>> # Initializing a model from the florence-2 configuration
-    >>> model = Florence2ForConditionalGeneration(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```"""
-
-    model_type = "florence2"
-    is_composition = False
-
-    def __init__(
-        self,
-        vision_config=None,
-        text_config=None,
-        ignore_index=-100,
-        vocab_size=51289,
-        projection_dim=1024,
-        **kwargs,
-    ):
-        self.ignore_index = ignore_index
-        self.vocab_size = vocab_size
-        self.projection_dim = projection_dim
-        if vision_config is not None:
-            vision_config = Florence2VisionConfig(**vision_config)
-        self.vision_config = vision_config
-
-        self.text_config = text_config
-        if text_config is not None:
-            self.text_config = Florence2LanguageConfig(**text_config)
-
-        super().__init__(**kwargs)

src/lerobot/policies/xvla/configuration_xvla.py

@@ -1,190 +0,0 @@
-#!/usr/bin/env python
-
-# ------------------------------------------------------------------------------
-# Copyright 2025 The HuggingFace Inc. team and 2toINF (https://github.com/2toINF)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------------
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any
-
-from lerobot.configs.policies import PreTrainedConfig
-from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
-from lerobot.optim.optimizers import AdamWConfig
-from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
-from lerobot.utils.constants import OBS_IMAGES
-
-# Conditional import for type checking and lazy loading
-from lerobot.utils.import_utils import _transformers_available
-
-if TYPE_CHECKING or _transformers_available:
-    from .configuration_florence2 import Florence2Config
-else:
-    Florence2Config = None
-
-
-@PreTrainedConfig.register_subclass("xvla")
-@dataclass
-class XVLAConfig(PreTrainedConfig):
-    """
-    Configuration class for the XVLA (Extended Vision-Language-Action) policy so it can
-    plug into the LeRobot training stack.
-
-    The config mirrors the knobs exposed in the original XVLA repository but also
-    declares the input/output feature contract required by LeRobot.
-    """
-
-    # Input / output structure
-    n_obs_steps: int = 1
-    chunk_size: int = 32
-    n_action_steps: int = 32
-    dtype: str = "float32"  # Options: "bfloat16", "float32"
-
-    normalization_mapping: dict[str, NormalizationMode] = field(
-        default_factory=lambda: {
-            "VISUAL": NormalizationMode.IDENTITY,
-            "STATE": NormalizationMode.IDENTITY,
-            "ACTION": NormalizationMode.MEAN_STD,
-        }
-    )
-
-    # Florence2 backbone and tokenizer configuration
-    florence_config: dict[str, Any] = field(default_factory=dict)
-    tokenizer_name: str = "facebook/bart-large"
-    tokenizer_max_length: int = 64
-    tokenizer_padding_side: str = "right"
-    pad_language_to: str = "max_length"
-
-    # Transformer head
-    hidden_size: int = 1024
-    depth: int = 24
-    num_heads: int = 16
-    mlp_ratio: float = 4.0
-    num_domains: int = 30
-    len_soft_prompts: int = 32
-    dim_time: int = 32
-    max_len_seq: int = 512
-    use_hetero_proj: bool = False
-
-    # Action & proprioception
-    action_mode: str = "ee6d"
-    num_denoising_steps: int = 10
-    use_proprio: bool = True
-    max_state_dim: int = 32
-    domain_feature_key: str | None = None
-
-    # Vision preprocessing
-    resize_imgs_with_padding: tuple[int, int] | None = None
-    num_image_views: int | None = None
-    empty_cameras: int = 0
-
-    # Freezing options for VLM components
-    # By default, VLM encoders are frozen and only policy transformer + soft prompts train
-    freeze_vision_encoder: bool = True  # Freeze VLM vision encoder weights
-    freeze_language_encoder: bool = True  # Freeze VLM language encoder weights
-    train_policy_transformer: bool = True  # Allow policy transformer to train
-    train_soft_prompts: bool = True  # Allow soft prompts to train
-
-    # Training presets
-    optimizer_lr: float = 1e-4
-    optimizer_betas: tuple[float, float] = (0.9, 0.95)
-    optimizer_eps: float = 1e-8
-    optimizer_weight_decay: float = 1e-4
-    optimizer_grad_clip_norm: float = 10.0
-
-    scheduler_warmup_steps: int = 1_000
-    scheduler_decay_steps: int = 30_000
-    scheduler_decay_lr: float = 2.5e-6
-
-    def __post_init__(self) -> None:
-        super().__post_init__()
-
-        if self.chunk_size <= 0:
-            raise ValueError("`chunk_size` must be strictly positive.")
-        if self.n_action_steps > self.chunk_size:
-            raise ValueError(
-                f"`n_action_steps` ({self.n_action_steps}) must be <= `chunk_size` ({self.chunk_size})."
-            )
-        if self.num_image_views is not None and self.num_image_views <= 0:
-            raise ValueError("`num_image_views` must be > 0 when specified.")
-        if self.dtype not in ["bfloat16", "float32"]:
-            raise ValueError(f"Invalid dtype: {self.dtype}")
-        self._florence_config_obj: Florence2Config | None = None
-
-    def get_florence_config(self) -> Florence2Config:
-        """
-        Build (and cache) the Florence2 transformer config that should back the VLM.
-        """
-        if self._florence_config_obj is None:
-            config_dict = dict(self.florence_config)
-            if "vision_config" not in config_dict or config_dict["vision_config"] is None:
-                raise ValueError("vision_config is required")
-
-            if "text_config" not in config_dict or config_dict["text_config"] is None:
-                raise ValueError("text_config is required")
-            self._florence_config_obj = Florence2Config(**config_dict)
-        return self._florence_config_obj
-
-    def validate_features(self) -> None:
-        if not self.image_features:
-            raise ValueError("XVLA requires at least one visual feature in the inputs.")
-        if self.use_proprio and self.robot_state_feature is None:
-            raise ValueError("`use_proprio=True` requires a proprioceptive state feature.")
-        if self.num_image_views is None:
-            self.num_image_views = len(self.image_features) + self.empty_cameras
-        else:
-            self.num_image_views = max(self.num_image_views, len(self.image_features) + self.empty_cameras)
-
-        if self.empty_cameras > 0:
-            height, width = (480, 640)
-            if self.resize_imgs_with_padding is not None:
-                height, width = self.resize_imgs_with_padding
-            for idx in range(self.empty_cameras):
-                key = f"{OBS_IMAGES}.empty_camera_{idx}"
-                if key not in self.input_features:
-                    self.input_features[key] = PolicyFeature(
-                        type=FeatureType.VISUAL,
-                        shape=(3, height, width),
-                    )
-
-    def get_optimizer_preset(self) -> AdamWConfig:
-        return AdamWConfig(
-            lr=self.optimizer_lr,
-            betas=self.optimizer_betas,
-            eps=self.optimizer_eps,
-            weight_decay=self.optimizer_weight_decay,
-            grad_clip_norm=self.optimizer_grad_clip_norm,
-        )
-
-    def get_scheduler_preset(self) -> CosineDecayWithWarmupSchedulerConfig:
-        return CosineDecayWithWarmupSchedulerConfig(
-            peak_lr=self.optimizer_lr,
-            decay_lr=self.scheduler_decay_lr,
-            num_warmup_steps=self.scheduler_warmup_steps,
-            num_decay_steps=self.scheduler_decay_steps,
-        )
-
-    @property
-    def observation_delta_indices(self) -> list[int] | None:
-        return None
-
-    @property
-    def action_delta_indices(self) -> list[int]:
-        return list(range(self.chunk_size))
-
-    @property
-    def reward_delta_indices(self) -> list[int] | None:
-        return None

src/lerobot/policies/xvla/modeling_xvla.py

@@ -1,526 +0,0 @@
-#!/usr/bin/env python
-
-# ------------------------------------------------------------------------------
-# Copyright 2025 The HuggingFace Inc. team and 2toINF (https://github.com/2toINF)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------------
-
-from __future__ import annotations
-
-import builtins
-import logging
-import os
-from collections import deque
-from pathlib import Path
-
-import torch
-import torch.nn.functional as F  # noqa: N812
-from torch import Tensor, nn
-
-from lerobot.configs.policies import PreTrainedConfig
-from lerobot.policies.pretrained import PreTrainedPolicy, T
-from lerobot.policies.utils import populate_queues
-from lerobot.utils.constants import ACTION, OBS_LANGUAGE_TOKENS, OBS_STATE
-
-from .action_hub import build_action_space
-from .configuration_florence2 import Florence2Config
-from .configuration_xvla import XVLAConfig
-from .modeling_florence2 import Florence2ForConditionalGeneration
-from .soft_transformer import SoftPromptedTransformer
-
-
-class XVLAModel(nn.Module):
-    """
-    XVLA backbone that stitches Florence-2 embeddings with the temporal/action transformer head.
-    """
-
-    def __init__(
-        self,
-        config: XVLAConfig,
-        florence_config: Florence2Config,
-        proprio_dim: int,
-    ) -> None:
-        super().__init__()
-        self.config = config
-        self.chunk_size: int = config.chunk_size
-        self.use_proprio: bool = config.use_proprio
-        self.action_space = build_action_space(config.action_mode.lower())
-        self.dim_action = self.action_space.dim_action
-        self.dim_proprio = proprio_dim
-
-        self.vlm = Florence2ForConditionalGeneration(florence_config)
-        if hasattr(self.vlm, "language_model"):
-            lm = self.vlm.language_model
-            if hasattr(lm, "model") and hasattr(lm.model, "decoder"):
-                del lm.model.decoder
-            if hasattr(lm, "lm_head"):
-                del lm.lm_head
-
-        projection_dim = getattr(self.vlm.config, "projection_dim", None)
-        if projection_dim is None:
-            raise ValueError("Florence2 config must provide `projection_dim` for multimodal fusion.")
-
-        self.transformer = SoftPromptedTransformer(
-            hidden_size=config.hidden_size,
-            multi_modal_input_size=projection_dim,
-            depth=config.depth,
-            num_heads=config.num_heads,
-            mlp_ratio=config.mlp_ratio,
-            num_domains=config.num_domains,
-            dim_action=self.dim_action,
-            dim_propio=self.dim_proprio,
-            len_soft_prompts=config.len_soft_prompts,
-            dim_time=config.dim_time,
-            max_len_seq=config.max_len_seq,
-            use_hetero_proj=config.use_hetero_proj,
-        )
-
-        # Apply freezing based on config
-        self._apply_freezing()
-
-        # Apply dtype casting based on config
-        self._apply_dtype()
-
-    def _get_target_dtype(self) -> torch.dtype:
-        """Get the target dtype based on config."""
-        if self.config.dtype == "bfloat16":
-            return torch.bfloat16
-        return torch.float32
-
-    def _apply_dtype(self) -> None:
-        """
-        Apply dtype casting to model components based on config.
-        """
-        target_dtype = self._get_target_dtype()
-        self.to(dtype=target_dtype)
-
-    def _apply_freezing(self) -> None:
-        """
-        Freeze VLM vision and language encoders based on config options.
-        Keep only policy transformer and soft prompts trainable.
-        """
-        # Freeze vision encoder
-        if self.config.freeze_vision_encoder and hasattr(self.vlm, "vision_tower"):
-            for param in self.vlm.vision_tower.parameters():
-                param.requires_grad = False
-
-        # Freeze language encoder
-        if self.config.freeze_language_encoder and hasattr(self.vlm, "language_model"):
-            lm = self.vlm.language_model
-            # Freeze encoder
-            if hasattr(lm, "model") and hasattr(lm.model, "encoder"):
-                for param in lm.model.encoder.parameters():
-                    param.requires_grad = False
-            # Freeze shared embeddings
-            if hasattr(lm, "model") and hasattr(lm.model, "shared"):
-                for param in lm.model.shared.parameters():
-                    param.requires_grad = False
-
-        # Freeze or unfreeze policy transformer
-        if not self.config.train_policy_transformer:
-            for name, param in self.transformer.named_parameters():
-                if "soft_prompts" not in name:
-                    param.requires_grad = False
-
-        # Freeze or unfreeze soft prompts
-        if not self.config.train_soft_prompts and hasattr(self.transformer, "soft_prompt_hub"):
-            for param in self.transformer.soft_prompt_hub.parameters():
-                param.requires_grad = False
-
-    def forward_vlm(
-        self,
-        input_ids: torch.LongTensor,
-        pixel_values: torch.FloatTensor,
-        image_mask: torch.Tensor,
-    ) -> dict[str, torch.Tensor]:
-        """
-        Encode text and multi-view images via Florence2 encoder.
-        """
-        batch_size, num_views = pixel_values.shape[:2]
-        flat_mask = image_mask.view(-1).to(dtype=torch.bool)
-        flat_images = pixel_values.flatten(0, 1)
-        num_valid = int(flat_mask.sum().item())
-        if num_valid == 0:
-            raise ValueError("At least one image view must be valid per batch.")
-
-        valid_images = flat_images[flat_mask]
-        valid_feats = self.vlm._encode_image(valid_images)
-        tokens_per_view, hidden_dim = valid_feats.shape[1:]
-
-        image_features = valid_feats.new_zeros((batch_size * num_views, tokens_per_view, hidden_dim))
-        image_features[flat_mask] = valid_feats
-        image_features = image_features.view(batch_size, num_views, tokens_per_view, hidden_dim)
-        inputs_embeds = self.vlm.get_input_embeddings()(input_ids)
-        merged_embeds, attention_mask = self.vlm._merge_input_ids_with_image_features(
-            image_features[:, 0],
-            inputs_embeds,
-        )
-
-        enc_out = self.vlm.language_model.model.encoder(
-            attention_mask=attention_mask,
-            inputs_embeds=merged_embeds,
-        )[0]
-
-        aux_visual_inputs = image_features[:, 1:].reshape(batch_size, -1, hidden_dim)
-        return {"vlm_features": enc_out, "aux_visual_inputs": aux_visual_inputs}
-
-    def forward(
-        self,
-        input_ids: torch.LongTensor,
-        image_input: torch.FloatTensor,
-        image_mask: torch.Tensor,
-        domain_id: torch.LongTensor,
-        proprio: torch.Tensor,
-        action: torch.Tensor,
-    ) -> dict[str, torch.Tensor]:
-        enc = self.forward_vlm(input_ids, image_input, image_mask)
-
-        batch_size = input_ids.shape[0]
-        t = (
-            torch.rand(1, device=input_ids.device)
-            + torch.arange(batch_size, device=input_ids.device) / batch_size
-        ) % (1 - 1e-5)
-
-        action_noisy = torch.randn_like(action) * t.view(-1, 1, 1) + action * (1 - t).view(-1, 1, 1)
-        proprio_m, action_noisy_m = self.action_space.preprocess(proprio, action_noisy)
-
-        pred_action = self.transformer(
-            domain_id=domain_id,
-            action_with_noise=action_noisy_m,
-            t=t,
-            proprio=proprio_m,
-            **enc,
-        )
-        return self.action_space.compute_loss(pred_action, action)
-
-    @torch.no_grad()
-    def generate_actions(
-        self,
-        input_ids: torch.LongTensor,
-        image_input: torch.FloatTensor,
-        image_mask: torch.Tensor,
-        domain_id: torch.LongTensor,
-        proprio: torch.Tensor,
-        steps: int,
-    ) -> torch.Tensor:
-        self.eval()
-        enc = self.forward_vlm(input_ids, image_input, image_mask)
-
-        batch_size = input_ids.shape[0]
-        action_dim = self.dim_action
-
-        x1 = torch.randn(batch_size, self.chunk_size, action_dim, device=proprio.device, dtype=proprio.dtype)
-        action = torch.zeros_like(x1)
-
-        steps = max(1, int(steps))
-        for i in range(steps, 0, -1):
-            t = torch.full((batch_size,), i / steps, device=proprio.device, dtype=proprio.dtype)
-            x_t = x1 * t.view(-1, 1, 1) + action * (1 - t).view(-1, 1, 1)
-            proprio_m, x_t_m = self.action_space.preprocess(proprio, x_t)
-            action = self.transformer(
-                domain_id=domain_id,
-                action_with_noise=x_t_m,
-                proprio=proprio_m,
-                t=t,
-                **enc,
-            )
-        return self.action_space.postprocess(action)
-
-
-class XVLAPolicy(PreTrainedPolicy):
-    """LeRobot-compliant wrapper built around the XVLA model."""
-
-    config_class = XVLAConfig
-    name = "xvla"
-
-    def __init__(self, config: XVLAConfig):
-        super().__init__(config)
-        config.validate_features()
-        florence_config = config.get_florence_config()
-        proprio_dim = config.max_state_dim if config.use_proprio else 0
-        self.model = XVLAModel(config=config, florence_config=florence_config, proprio_dim=proprio_dim)
-        self.reset()
-
-    def reset(self) -> None:
-        self._queues = {
-            ACTION: deque(maxlen=self.config.n_action_steps),
-        }
-
-    def get_optim_params(self) -> dict:
-        """Return only trainable parameters for optimization."""
-        return filter(lambda p: p.requires_grad, self.parameters())
-
-    def _prepare_state(self, batch: dict[str, Tensor], batch_size: int, device: torch.device) -> Tensor:
-        if not self.config.use_proprio or OBS_STATE not in batch:
-            return torch.zeros(batch_size, 0, device=device)
-        state = batch[OBS_STATE]
-        if state.ndim > 2:
-            state = state[:, -1, :]
-        return pad_vector(state, self.model.dim_proprio)
-
-    def _prepare_images(self, batch: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
-        present_img_keys = [key for key in self.config.image_features if key in batch]
-        if len(present_img_keys) == 0:
-            raise ValueError(
-                "All image features are missing from the batch. "
-                f"Batch keys: {list(batch.keys())}, expected at least one of {list(self.config.image_features)}."
-            )
-
-        images = []
-        masks = []
-        for key in present_img_keys:
-            img = batch[key][:, -1] if batch[key].ndim == 5 else batch[key]
-            if self.config.resize_imgs_with_padding is not None:
-                img = resize_with_pad(img, *self.config.resize_imgs_with_padding)
-            images.append(img)
-            masks.append(torch.ones(img.size(0), dtype=torch.bool, device=img.device))
-
-        stacked_imgs = torch.stack(images, dim=1)
-        stacked_masks = torch.stack(masks, dim=1)
-
-        total_views = self.config.num_image_views or stacked_imgs.size(1)
-        total_views = max(total_views, stacked_imgs.size(1))
-        num_pad = total_views - stacked_imgs.size(1)
-        if num_pad > 0:
-            pad_shape = (stacked_imgs.size(0), num_pad, *stacked_imgs.shape[2:])
-            pad_imgs = stacked_imgs.new_zeros(pad_shape)
-            pad_masks = stacked_masks.new_zeros((stacked_masks.size(0), num_pad))
-            stacked_imgs = torch.cat([stacked_imgs, pad_imgs], dim=1)
-            stacked_masks = torch.cat([stacked_masks, pad_masks], dim=1)
-
-        return stacked_imgs, stacked_masks
-
-    def _get_domain_id(self, batch: dict[str, Tensor], batch_size: int, device: torch.device) -> Tensor:
-        candidate = None
-        if self.config.domain_feature_key and self.config.domain_feature_key in batch:
-            candidate = batch[self.config.domain_feature_key]
-        elif "domain_id" in batch:
-            candidate = batch["domain_id"]
-
-        if candidate is None:
-            return torch.zeros(batch_size, dtype=torch.long, device=device)
-
-        if not isinstance(candidate, torch.Tensor):
-            candidate = torch.as_tensor(candidate, device=device)
-        else:
-            candidate = candidate.to(device=device)
-
-        if candidate.ndim == 0:
-            candidate = candidate.expand(batch_size)
-        if candidate.ndim > 1:
-            candidate = candidate.view(candidate.shape[0], -1)[:, 0]
-        if candidate.shape[0] != batch_size:
-            candidate = candidate.expand(batch_size)
-        return candidate.to(dtype=torch.long)
-
-    def _prepare_action_targets(self, batch: dict[str, Tensor]) -> Tensor:
-        if ACTION not in batch:
-            raise ValueError("Batch is missing action targets required for training.")
-        actions = batch[ACTION]
-        if actions.ndim == 2:
-            actions = actions.unsqueeze(1)
-        actions = pad_tensor_along_dim(actions, self.config.chunk_size, dim=1)
-        if actions.shape[-1] != self.model.dim_action:
-            actions = pad_vector(actions, self.model.dim_action)
-        return actions
-
-    def _build_model_inputs(self, batch: dict[str, Tensor]) -> dict[str, Tensor]:
-        input_ids = batch[OBS_LANGUAGE_TOKENS]
-        batch_size = input_ids.shape[0]
-        images, image_mask = self._prepare_images(batch)
-        domain_id = self._get_domain_id(batch, batch_size, images.device)
-        proprio = self._prepare_state(batch, batch_size, images.device)
-        return {
-            "input_ids": input_ids,
-            "image_input": images,
-            "image_mask": image_mask,
-            "domain_id": domain_id,
-            "proprio": proprio,
-        }
-
-    def _trim_action_dim(self, actions: Tensor) -> Tensor:
-        feature = self.config.action_feature
-        if feature is None:
-            return actions
-        desired_dim = self.model.dim_action
-        if desired_dim == actions.shape[-1]:
-            return actions
-        if desired_dim < actions.shape[-1]:
-            return actions[..., :desired_dim]
-        return pad_vector(actions, desired_dim)
-
-    def forward(self, batch: dict[str, Tensor]) -> tuple[Tensor, dict]:
-        inputs = self._build_model_inputs(batch)
-        targets = self._prepare_action_targets(batch)
-        losses = self.model(action=targets, **inputs)
-        total_loss = sum(losses.values())
-
-        log_dict = {k: v.detach().item() for k, v in losses.items()}
-        log_dict["loss"] = total_loss.detach().item()
-        return total_loss, log_dict
-
-    def _get_action_chunk(self, batch: dict[str, Tensor]) -> Tensor:
-        inputs = self._build_model_inputs(batch)
-        actions = self.model.generate_actions(**inputs, steps=self.config.num_denoising_steps)
-        actions = self._trim_action_dim(actions)
-        return actions
-
-    @torch.no_grad()
-    def predict_action_chunk(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor:  # noqa: ARG002
-        self.eval()
-        self._queues = populate_queues(self._queues, batch, exclude_keys=[ACTION])
-        return self._get_action_chunk(batch)
-
-    @torch.no_grad()
-    def select_action(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor:  # noqa: ARG002
-        self.eval()
-        self._queues = populate_queues(self._queues, batch, exclude_keys=[ACTION])
-
-        if len(self._queues[ACTION]) == 0:
-            actions = self._get_action_chunk(batch)
-            self._queues[ACTION].extend(actions.transpose(0, 1)[: self.config.n_action_steps])
-
-        return self._queues[ACTION].popleft()
-
-    @classmethod
-    def from_pretrained(
-        cls: builtins.type[T],
-        pretrained_name_or_path: str | Path,
-        *,
-        config: PreTrainedConfig | None = None,
-        force_download: bool = False,
-        resume_download: bool | None = None,
-        proxies: dict | None = None,
-        token: str | bool | None = None,
-        cache_dir: str | Path | None = None,
-        local_files_only: bool = False,
-        revision: str | None = None,
-        strict: bool = False,
-        **kwargs,
-    ):
-        """
-        Loads XVLA model weights with:
-        - automatic prefix 'model.' added to all keys
-        - skip list for layers that should remain randomly initialized
-        """
-        import safetensors.torch
-
-        # step 1: load config
-        # TODO: jadechoghari, fix this
-        if config is None:
-            config = PreTrainedConfig.from_pretrained(
-                pretrained_name_or_path=pretrained_name_or_path,
-                force_download=force_download,
-                resume_download=resume_download,
-                proxies=proxies,
-                token=token,
-                cache_dir=cache_dir,
-                local_files_only=local_files_only,
-                revision=revision,
-                **kwargs,
-            )
-
-        model_id = str(pretrained_name_or_path)
-        instance = cls(config, **kwargs)
-        # step 2: locate model.safetensors
-        if os.path.isdir(model_id):
-            logging.info("Loading weights from local directory")
-            model_file = os.path.join(model_id, "model.safetensors")
-        else:
-            try:
-                from huggingface_hub import hf_hub_download
-                from huggingface_hub.utils import HfHubHTTPError
-
-                model_file = hf_hub_download(
-                    repo_id=model_id,
-                    filename="model.safetensors",
-                    revision=revision,
-                    cache_dir=cache_dir,
-                    force_download=force_download,
-                    proxies=proxies,
-                    resume_download=resume_download,
-                    token=token,
-                    local_files_only=local_files_only,
-                )
-            except HfHubHTTPError as e:
-                raise FileNotFoundError(f"model.safetensors not found on the Hub at {model_id}") from e
-
-        logging.info(f"Loading checkpoint from {model_file}")
-        # step 3: load state dict
-        state_dict = safetensors.torch.load_file(model_file)
-        encoder_key = "model.vlm.language_model.model.encoder.embed_tokens.weight"
-        shared_key = "model.vlm.language_model.model.shared.weight"
-        if encoder_key in state_dict:
-            state_dict[shared_key] = state_dict[encoder_key]
-            # or deepcopy
-        # step 4: load into instance
-        instance.load_state_dict(state_dict, strict=True)
-        logging.info("Loaded XVLA checkpoint")
-        # step 5: finalize
-        # Reapply dtype after loading state dict
-        instance.model._apply_dtype()
-        instance.to(config.device)
-        instance.eval()
-        return instance
-
-
-def resize_with_pad(img: torch.Tensor, height: int, width: int, pad_value: float = 0.0) -> torch.Tensor:
-    if img.ndim != 4:
-        raise ValueError(f"(b,c,h,w) expected, but got {img.shape}")
-
-    current_height, current_width = img.shape[2:]
-    if current_height == height and current_width == width:
-        return img
-
-    ratio = max(current_width / width, current_height / height)
-    resized_height = int(current_height / ratio)
-    resized_width = int(current_width / ratio)
-    resized_img = F.interpolate(
-        img, size=(resized_height, resized_width), mode="bilinear", align_corners=False
-    )
-
-    pad_height = max(0, height - resized_height)
-    pad_width = max(0, width - resized_width)
-    padded_img = F.pad(resized_img, (pad_width, 0, pad_height, 0), value=pad_value)
-    return padded_img
-
-
-def pad_vector(vector: Tensor, new_dim: int) -> Tensor:
-    if vector.shape[-1] == new_dim:
-        return vector
-    if new_dim == 0:
-        shape = list(vector.shape)
-        shape[-1] = 0
-        return vector.new_zeros(*shape)
-    shape = list(vector.shape)
-    current_dim = shape[-1]
-    shape[-1] = new_dim
-    new_vector = vector.new_zeros(*shape)
-    length = min(current_dim, new_dim)
-    new_vector[..., :length] = vector[..., :length]
-    return new_vector
-
-
-def pad_tensor_along_dim(tensor: Tensor, target_len: int, dim: int = 1) -> Tensor:
-    current_len = tensor.size(dim)
-    if current_len == target_len:
-        return tensor
-    if current_len > target_len:
-        slices = [slice(None)] * tensor.dim()
-        slices[dim] = slice(0, target_len)
-        return tensor[tuple(slices)]
-    pad_shape = list(tensor.shape)
-    pad_shape[dim] = target_len - current_len
-    pad_tensor = tensor.new_zeros(pad_shape)
-    return torch.cat([tensor, pad_tensor], dim=dim)

src/lerobot/policies/xvla/processor_xvla.py

@@ -1,551 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright 2025 The HuggingFace Inc. team and 2toINF (https://github.com/2toINF)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------------
-
-from dataclasses import dataclass
-from typing import Any
-
-import numpy as np
-import torch
-
-from lerobot.configs.types import PipelineFeatureType, PolicyFeature
-from lerobot.datasets.factory import IMAGENET_STATS
-from lerobot.policies.xvla.configuration_xvla import XVLAConfig
-from lerobot.policies.xvla.utils import rotate6d_to_axis_angle
-from lerobot.processor import (
-    AddBatchDimensionProcessorStep,
-    DeviceProcessorStep,
-    NormalizerProcessorStep,
-    ObservationProcessorStep,
-    PolicyAction,
-    PolicyProcessorPipeline,
-    ProcessorStep,
-    ProcessorStepRegistry,
-    RenameObservationsProcessorStep,
-    TokenizerProcessorStep,
-    UnnormalizerProcessorStep,
-)
-from lerobot.processor.converters import policy_action_to_transition, transition_to_policy_action
-from lerobot.processor.core import EnvTransition, TransitionKey
-from lerobot.utils.constants import (
-    OBS_IMAGES,
-    OBS_STATE,
-    POLICY_POSTPROCESSOR_DEFAULT_NAME,
-    POLICY_PREPROCESSOR_DEFAULT_NAME,
-)
-
-
-def make_xvla_pre_post_processors(
-    config: XVLAConfig,
-    dataset_stats: dict[str, dict[str, torch.Tensor]] | None = None,
-) -> tuple[
-    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
-    PolicyProcessorPipeline[PolicyAction, PolicyAction],
-]:
-    """
-    Build the LeRobot processor pipelines for XVLA.
-    """
-
-    features = {**config.input_features, **config.output_features}
-    input_steps = [
-        RenameObservationsProcessorStep(rename_map={}),
-        AddBatchDimensionProcessorStep(),
-        TokenizerProcessorStep(
-            tokenizer_name=config.tokenizer_name,
-            max_length=config.tokenizer_max_length,
-            padding=config.pad_language_to,
-            padding_side=config.tokenizer_padding_side,
-        ),
-        XVLAImageToFloatProcessorStep(),
-        XVLAImageNetNormalizeProcessorStep(),
-        XVLAAddDomainIdProcessorStep(),
-        DeviceProcessorStep(device=config.device),
-        NormalizerProcessorStep(
-            features=features, norm_map=config.normalization_mapping, stats=dataset_stats
-        ),
-    ]
-    output_steps = [
-        UnnormalizerProcessorStep(
-            features=config.output_features,
-            norm_map=config.normalization_mapping,
-            stats=dataset_stats,
-        ),
-        DeviceProcessorStep(device="cpu"),
-    ]
-
-    return (
-        PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
-            steps=input_steps,
-            name=POLICY_PREPROCESSOR_DEFAULT_NAME,
-        ),
-        PolicyProcessorPipeline[PolicyAction, PolicyAction](
-            steps=output_steps,
-            name=POLICY_POSTPROCESSOR_DEFAULT_NAME,
-            to_transition=policy_action_to_transition,
-            to_output=transition_to_policy_action,
-        ),
-    )
-
-
-# Custom XVLA processor steps
-@dataclass
-class LiberoProcessorStep(ObservationProcessorStep):
-    """
-    Processes LIBERO observations into the LeRobot format.
-
-    This step handles the specific observation structure from LIBERO environments,
-    which includes nested robot_state dictionaries and image observations.
-
-    **State Processing:**
-    -   Processes the `robot_state` dictionary which contains nested end-effector,
-        gripper, and joint information.
-    -   Extracts and concatenates:
-        - End-effector position (3D)
-        - End-effector quaternion converted to axis-angle (3D)
-        - Gripper joint positions (2D)
-    -   Maps the concatenated state to `"observation.state"`.
-
-    **Image Processing:**
-    -   Rotates images by 180 degrees by flipping both height and width dimensions.
-    -   This accounts for the HuggingFaceVLA/libero camera orientation convention.
-    """
-
-    def _process_observation(self, observation):
-        """
-        Processes both image and robot_state observations from LIBERO.
-        """
-        processed_obs = observation.copy()
-        for key in list(processed_obs.keys()):
-            if key.startswith(f"{OBS_IMAGES}."):
-                img = processed_obs[key]
-
-                if key == f"{OBS_IMAGES}.image":
-                    # Flip both H and W
-                    img = torch.flip(img, dims=[2, 3])
-
-                processed_obs[key] = img
-        # Process robot_state into a flat state vector
-        if "observation.robot_state" in processed_obs:
-            robot_state = processed_obs.pop("observation.robot_state")
-
-            # Extract components
-            eef_pos = robot_state["eef"]["pos"]  # (B, 3,)
-            eef_mat = robot_state["eef"]["mat"]  # (B, 3, 3)
-            eef_rot6d = self._mat_to_rotate6d(eef_mat)  # (B, 6)
-
-            extra = torch.zeros((eef_pos.shape[0], 1), dtype=torch.float32, device=eef_pos.device)
-
-            proprio_state = torch.cat((eef_pos, eef_rot6d, extra), dim=-1)  # (B, 10)
-            state = torch.cat((proprio_state, torch.zeros_like(proprio_state)), dim=-1)  # (B, 20)
-            # ensure float32
-            state = state.float()
-            if state.dim() == 1:
-                state = state.unsqueeze(0)
-
-            processed_obs[OBS_STATE] = state
-        return processed_obs
-
-    def transform_features(
-        self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
-    ) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
-        """
-        Transforms feature keys from the LIBERO format to the LeRobot standard.
-        """
-        new_features: dict[PipelineFeatureType, dict[str, PolicyFeature]] = {}
-
-        # copy over non-STATE features
-        for ft, feats in features.items():
-            if ft != PipelineFeatureType.STATE:
-                new_features[ft] = feats.copy()
-
-        # rebuild STATE features
-        state_feats = {}
-
-        # add our new flattened state
-        state_feats["observation.state"] = PolicyFeature(
-            key="observation.state",
-            shape=(20,),
-            dtype="float32",
-        )
-
-        new_features[PipelineFeatureType.STATE] = state_feats
-
-        return new_features
-
-    def _mat_to_rotate6d(self, rot_mats: torch.Tensor) -> torch.Tensor:
-        """
-        Convert batched rotation matrices (B, 3, 3) into 6D rotation representation (B, 6).
-
-        Args:
-            rot_mats (Tensor): Rotation matrices of shape (B, 3, 3)
-
-        Returns:
-            Tensor: 6D rotation representation, shape (B, 6)
-
-        Raises:
-            TypeError: if input is not a torch tensor
-            ValueError: if shape is not (B, 3, 3)
-        """
-
-        if not isinstance(rot_mats, torch.Tensor):
-            raise TypeError(f"mat_to_rot6d expects a torch.Tensor, got {type(rot_mats)}")
-
-        if rot_mats.ndim != 3 or rot_mats.shape[1:] != (3, 3):
-            raise ValueError(f"mat_to_rot6d expects shape (B, 3, 3), got {tuple(rot_mats.shape)}")
-
-        rot_mats = rot_mats.to(torch.float32)
-
-        col1 = rot_mats[:, :3, 0]  # (B, 3)
-        col2 = rot_mats[:, :3, 1]  # (B, 3)
-
-        rot6d = torch.cat([col1, col2], dim=-1)  # (B, 6)
-
-        return rot6d
-
-    def observation(self, observation):
-        return self._process_observation(observation)
-
-
-@dataclass
-@ProcessorStepRegistry.register(name="xvla_image_scale")
-class XVLAImageScaleProcessorStep(ProcessorStep):
-    """Scale image observations by 255 to convert from [0, 1] to [0, 255] range.
-
-    This processor step multiplies all image observations by 255, which is required
-    for XVLA models that expect images in uint8-like range.
-
-    Args:
-        image_keys: List of observation keys that contain images to scale.
-                   If None, will automatically detect keys starting with "observation.images."
-    """
-
-    image_keys: list[str] | None = None
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        """Scale image observations by 255."""
-        new_transition = transition.copy()
-        obs = new_transition.get(TransitionKey.OBSERVATION, {})
-        if obs is None:
-            return new_transition
-
-        # Make a copy of observations to avoid modifying the original
-        obs = obs.copy()
-
-        # Determine which keys to scale
-        keys_to_scale = self.image_keys
-        if keys_to_scale is None:
-            # Auto-detect image keys
-            keys_to_scale = [k for k in obs if k.startswith("observation.images.")]
-
-        # Scale each image
-        for key in keys_to_scale:
-            if key in obs and isinstance(obs[key], torch.Tensor):
-                obs[key] = obs[key] * 255
-
-        new_transition[TransitionKey.OBSERVATION] = obs
-        return new_transition
-
-    def transform_features(self, features):
-        """Image scaling doesn't change feature structure."""
-        return features
-
-    def get_config(self) -> dict[str, Any]:
-        """Return serializable configuration."""
-        return {
-            "image_keys": self.image_keys,
-        }
-
-
-@dataclass
-@ProcessorStepRegistry.register(name="xvla_image_to_float")
-class XVLAImageToFloatProcessorStep(ProcessorStep):
-    """Convert image observations from [0, 255] to [0, 1] range.
-
-    This processor step divides image observations by 255 to convert from uint8-like
-    range [0, 255] to float range [0, 1]. This is typically used when loading images
-    that are stored as uint8 values.
-
-    Args:
-        image_keys: List of observation keys that contain images to convert.
-                   If None, will automatically detect keys starting with "observation.images."
-        validate_range: If True, validates that input values are in [0, 255] range (default: True)
-
-    Raises:
-        ValueError: If validate_range is True and image values are not in [0, 255] range.
-    """
-
-    image_keys: list[str] | None = None
-    validate_range: bool = True
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        """Convert image observations from [0, 255] to [0, 1]."""
-        new_transition = transition.copy()
-        obs = new_transition.get(TransitionKey.OBSERVATION, {})
-        if obs is None:
-            return new_transition
-
-        # Make a copy of observations to avoid modifying the original
-        obs = obs.copy()
-
-        # Determine which keys to convert
-        keys_to_convert = self.image_keys
-        if keys_to_convert is None:
-            # Auto-detect image keys
-            keys_to_convert = [k for k in obs if k.startswith("observation.images.")]
-
-        # Convert each image
-        for key in keys_to_convert:
-            if key in obs and isinstance(obs[key], torch.Tensor):
-                tensor = obs[key]
-
-                # Validate that values are in [0, 255] range if requested
-                if self.validate_range:
-                    min_val = tensor.min().item()
-                    max_val = tensor.max().item()
-                    if min_val < 0.0 or max_val > 255.0:
-                        raise ValueError(
-                            f"Image '{key}' has values outside [0, 255] range: "
-                            f"min={min_val:.4f}, max={max_val:.4f}. "
-                            f"Cannot convert to [0, 1] range."
-                        )
-
-                # Convert to float and divide by 255
-                obs[key] = tensor.float() / 255.0
-
-        new_transition[TransitionKey.OBSERVATION] = obs
-        return new_transition
-
-    def transform_features(self, features):
-        """Image conversion doesn't change feature structure."""
-        return features
-
-    def get_config(self) -> dict[str, Any]:
-        """Return serializable configuration."""
-        return {
-            "image_keys": self.image_keys,
-            "validate_range": self.validate_range,
-        }
-
-
-@dataclass
-@ProcessorStepRegistry.register(name="xvla_imagenet_normalize")
-class XVLAImageNetNormalizeProcessorStep(ProcessorStep):
-    """Normalize image observations using ImageNet statistics.
-
-    This processor step applies ImageNet normalization (mean and std) to image observations.
-    It validates that input values are in the [0, 1] range before normalizing.
-
-    The normalization formula is: (image - mean) / std
-
-    Args:
-        image_keys: List of observation keys that contain images to normalize.
-                   If None, will automatically detect keys starting with "observation.images."
-
-    Raises:
-        ValueError: If image values are not in the [0, 1] range.
-    """
-
-    image_keys: list[str] | None = None
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        """Normalize image observations using ImageNet statistics."""
-        new_transition = transition.copy()
-        obs = new_transition.get(TransitionKey.OBSERVATION, {})
-        if obs is None:
-            return new_transition
-
-        # Make a copy of observations to avoid modifying the original
-        obs = obs.copy()
-
-        # Determine which keys to normalize
-        keys_to_normalize = self.image_keys
-        if keys_to_normalize is None:
-            # Auto-detect image keys
-            keys_to_normalize = [k for k in obs if k.startswith("observation.images.")]
-
-        # Normalize each image
-        for key in keys_to_normalize:
-            if key in obs and isinstance(obs[key], torch.Tensor):
-                tensor = obs[key]
-
-                # Validate that values are in [0, 1] range
-                min_val = tensor.min().item()
-                max_val = tensor.max().item()
-                if min_val < 0.0 or max_val > 1.0:
-                    raise ValueError(
-                        f"Image '{key}' has values outside [0, 1] range: "
-                        f"min={min_val:.4f}, max={max_val:.4f}. "
-                        f"ImageNet normalization requires input values in [0, 1]."
-                    )
-
-                # Apply ImageNet normalization
-                mean = torch.tensor(IMAGENET_STATS["mean"], device=tensor.device, dtype=tensor.dtype)
-                std = torch.tensor(IMAGENET_STATS["std"], device=tensor.device, dtype=tensor.dtype)
-
-                # Expand mean/std to match tensor dims (e.g., BCHW or BNCHW)
-                while mean.dim() < tensor.dim():
-                    mean = mean.unsqueeze(0)
-                    std = std.unsqueeze(0)
-
-                # Normalize: (image - mean) / std
-                obs[key] = (tensor - mean) / std
-
-        new_transition[TransitionKey.OBSERVATION] = obs
-        return new_transition
-
-    def transform_features(self, features):
-        """ImageNet normalization doesn't change feature structure."""
-        return features
-
-    def get_config(self) -> dict[str, Any]:
-        """Return serializable configuration."""
-        return {
-            "image_keys": self.image_keys,
-        }
-
-
-@dataclass
-@ProcessorStepRegistry.register(name="xvla_add_domain_id")
-class XVLAAddDomainIdProcessorStep(ProcessorStep):
-    """Add domain_id to complementary data.
-
-    This processor step adds a domain_id tensor to the complementary data,
-    which is used by XVLA to identify different robot embodiments or task domains.
-
-    Args:
-        domain_id: The domain ID to add (default: 3)
-    """
-
-    domain_id: int = 0
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        """Add domain_id to complementary data."""
-        new_transition = transition.copy()
-        comp = new_transition.get(TransitionKey.COMPLEMENTARY_DATA, {})
-        comp = {} if comp is None else comp.copy()
-
-        # Infer batch size from observation tensors
-        obs = new_transition.get(TransitionKey.OBSERVATION, {})
-        batch_size = 1
-        if obs:
-            for v in obs.values():
-                if isinstance(v, torch.Tensor):
-                    batch_size = v.shape[0]
-                    break
-
-        # Add domain_id tensor
-        comp["domain_id"] = torch.tensor([int(self.domain_id)] * batch_size, dtype=torch.long)
-
-        new_transition[TransitionKey.COMPLEMENTARY_DATA] = comp
-        return new_transition
-
-    def transform_features(self, features):
-        """Domain ID addition doesn't change feature structure."""
-        return features
-
-    def get_config(self) -> dict[str, Any]:
-        """Return serializable configuration."""
-        return {
-            "domain_id": self.domain_id,
-        }
-
-
-@dataclass
-@ProcessorStepRegistry.register(name="xvla_rotation_6d_to_axis_angle")
-class XVLARotation6DToAxisAngleProcessorStep(ProcessorStep):
-    """Convert 6D rotation representation to axis-angle and reorganize action dimensions.
-
-    This processor step takes actions with 6D rotation representation and converts them to
-    axis-angle representation, reorganizing the action dimensions as:
-    - action[:, :3] -> target_eef (end-effector position)
-    - action[:, 3:9] -> 6D rotation (converted to axis-angle, 3D)
-    - action[:, 9:10] -> gripper action
-
-    Final output: [target_eef (3), axis_angle (3), gripper (1)] = 7D action
-
-    Args:
-        expected_action_dim: Expected input action dimension (default: 10, supports 6D rotation + extras)
-    """
-
-    expected_action_dim: int = 10
-
-    def __call__(self, transition: EnvTransition) -> EnvTransition:
-        """Convert 6D rotation to axis-angle in action."""
-        new_transition = transition.copy()
-        action = new_transition.get(TransitionKey.ACTION)
-
-        if action is None or not isinstance(action, torch.Tensor):
-            return new_transition
-
-        # Convert to numpy for processing
-        device = action.device
-        dtype = action.dtype
-        action_np = action.cpu().numpy()
-
-        # Extract components
-        # action shape: (B, D) where D >= 10
-        target_eef = action_np[:, :3]  # (B, 3)
-        rotation_6d = action_np[:, 3:9]  # (B, 6)
-        target_act = action_np[:, 9:10]  # (B, 1)
-
-        # Convert 6D rotation to axis-angle
-        target_axis = rotate6d_to_axis_angle(rotation_6d)  # (B, 3)
-
-        # Concatenate: [eef (3), axis_angle (3), gripper (1)] = 7D
-        action_np = np.concatenate([target_eef, target_axis, target_act], axis=-1)
-
-        # Convert gripper action to -1 or 1
-        action_np[:, -1] = np.where(action_np[:, -1] > 0.5, 1.0, -1.0)
-
-        # Convert back to tensor
-        action = torch.from_numpy(action_np).to(device=device, dtype=dtype)
-
-        new_transition[TransitionKey.ACTION] = action
-        return new_transition
-
-    def transform_features(self, features):
-        """Rotation conversion changes action dimension from 10 to 7."""
-        # Note: This is a simplified version. In practice, you might want to
-        # update the action feature shape in the features dict.
-        return features
-
-    def get_config(self) -> dict[str, Any]:
-        """Return serializable configuration."""
-        return {
-            "expected_action_dim": self.expected_action_dim,
-        }
-
-
-def make_xvla_libero_pre_post_processors() -> tuple[
-    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
-    PolicyProcessorPipeline[PolicyAction, PolicyAction],
-]:
-    """
-    Build the LeRobot processor pipelines for XVLA with LIBERO environment.
-    """
-    pre_processor_steps: list[ProcessorStep] = []
-    post_processor_steps: list[ProcessorStep] = []
-    pre_processor_steps.extend(
-        [LiberoProcessorStep(), XVLAImageNetNormalizeProcessorStep(), XVLAAddDomainIdProcessorStep()]
-    )
-    post_processor_steps.extend([XVLARotation6DToAxisAngleProcessorStep()])
-    return (
-        PolicyProcessorPipeline[dict[str, Any], dict[str, Any]](
-            steps=pre_processor_steps,
-        ),
-        PolicyProcessorPipeline[PolicyAction, PolicyAction](
-            steps=post_processor_steps,
-        ),
-    )

src/lerobot/policies/xvla/soft_transformer.py

@@ -1,415 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright 2025 2toINF (https://github.com/2toINF)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------------
-
-from __future__ import annotations
-
-import math
-from collections.abc import Iterable
-from functools import partial
-from typing import Final
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as functional
-
-# ------------------------------- Small utils ----------------------------------
-
-
-def _to_2tuple(x) -> tuple:
-    """Minimal replacement for timm.layers.to_2tuple."""
-    if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
-        t = tuple(x)
-        return (t[0], t[1]) if len(t) >= 2 else (t[0], t[0])
-    return (x, x)
-
-
-def _has_sdp_attention() -> bool:
-    """Check if we can use PyTorch fused scaled_dot_product_attention."""
-    return hasattr(functional, "scaled_dot_product_attention")
-
-
-# ---------------------------------- MLP --------------------------------------
-
-
-class Mlp(nn.Module):
-    """
-    MLP used in ViT-style blocks.
-
-    Supports Linear or 1x1 Conv 'linear_layer' for token/channel mixing.
-    """
-
-    def __init__(
-        self,
-        in_features: int,
-        hidden_features: int | None = None,
-        out_features: int | None = None,
-        norm_layer: type[nn.Module] | None = None,
-        bias: bool | tuple[bool, bool] = True,
-        drop: float | tuple[float, float] = 0.0,
-        use_conv: bool = False,
-    ) -> None:
-        super().__init__()
-        out_features = out_features or in_features
-        hidden_features = hidden_features or in_features
-        bias = _to_2tuple(bias)
-        drop_probs = _to_2tuple(drop)
-        linear_layer = partial(nn.Conv2d, kernel_size=1) if use_conv else nn.Linear
-
-        self.fc1 = linear_layer(in_features, hidden_features, bias=bias[0])
-        self.act = nn.GELU(approximate="tanh")
-        self.drop1 = nn.Dropout(drop_probs[0])
-        self.norm = norm_layer(hidden_features) if norm_layer is not None else nn.Identity()
-        self.fc2 = linear_layer(hidden_features, out_features, bias=bias[1])
-        self.drop2 = nn.Dropout(drop_probs[1])
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        # Expect [B, T, C] for Linear variant; caller is responsible for shapes.
-        x = self.fc1(x)
-        x = self.act(x)
-        x = self.drop1(x)
-        x = self.norm(x)
-        x = self.fc2(x)
-        x = self.drop2(x)
-        return x
-
-
-# -------------------------------- Attention ----------------------------------
-
-
-class Attention(nn.Module):
-    """
-    Multi-Head Self-Attention with optional fused SDPA fallback.
-
-    If PyTorch provides `scaled_dot_product_attention`, it will be used
-    (usually faster and more stable); otherwise we use a manual implementation.
-    """
-
-    fused_attn: Final[bool]
-
-    def __init__(
-        self,
-        dim: int,
-        num_heads: int = 8,
-        qkv_bias: bool = False,
-        qk_norm: bool = False,
-        attn_drop: float = 0.0,
-        proj_drop: float = 0.0,
-        norm_layer: type[nn.Module] = nn.LayerNorm,
-    ) -> None:
-        super().__init__()
-        assert dim % num_heads == 0, "dim should be divisible by num_heads"
-        self.num_heads = num_heads
-        self.head_dim = dim // num_heads
-        self.scale = self.head_dim**-0.5
-        self.fused_attn = _has_sdp_attention()
-
-        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
-        self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
-        self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
-        self.attn_drop = nn.Dropout(attn_drop)
-        self.proj = nn.Linear(dim, dim)
-        self.proj_drop = nn.Dropout(proj_drop)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        """
-        Parameters
-        ----------
-        x : Tensor, shape [batch_size, seq_len, channels]
-            Input sequence.
-
-        Returns
-        -------
-        Tensor, shape [batch_size, seq_len, channels]
-            Output sequence after MHSA + projection.
-        """
-        batch_size, seq_len, channels = x.shape
-        qkv = (
-            self.qkv(x)
-            .reshape(batch_size, seq_len, 3, self.num_heads, self.head_dim)
-            .permute(2, 0, 3, 1, 4)  # 3 x [batch_size, num_heads, seq_len, head_dim]
-        )
-        q, k, v = qkv.unbind(0)  # each: [batch_size, num_heads, seq_len, head_dim]
-        q, k = self.q_norm(q), self.k_norm(k)
-
-        if self.fused_attn:
-            x = functional.scaled_dot_product_attention(
-                q,
-                k,
-                v,
-                dropout_p=self.attn_drop.p if self.training else 0.0,
-            )  # [batch_size, num_heads, seq_len, head_dim]
-        else:
-            q = q * self.scale
-            attn = q @ k.transpose(-2, -1)  # [batch_size, num_heads, seq_len, seq_len]
-            attn = attn.softmax(dim=-1)
-            attn = self.attn_drop(attn)
-            x = attn @ v  # [batch_size, num_heads, seq_len, head_dim]
-
-        x = x.transpose(1, 2).reshape(batch_size, seq_len, channels)  # [batch_size, seq_len, channels]
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        return x
-
-
-# ------------------------------- Utilities -----------------------------------
-
-
-def basic_init(module: nn.Module) -> None:
-    """
-    Apply a basic initialization scheme to Linear layers.
-
-    - Weight: Xavier uniform initialization.
-    - Bias: Set to zero.
-    """
-    if isinstance(module, nn.Linear):
-        nn.init.xavier_uniform_(module.weight)
-        if module.bias is not None:
-            nn.init.constant_(module.bias, 0.0)
-
-
-def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 100) -> torch.Tensor:
-    """
-    Create sinusoidal timestep embeddings.
-
-    Parameters
-    ----------
-    t : torch.Tensor
-        Shape [B]. Each element is a timestep index, may be fractional.
-    dim : int
-        Dimensionality of the output embedding.
-    max_period : int, default=100
-        Controls the minimum frequency of the sinusoids.
-
-    Returns
-    -------
-    torch.Tensor
-        Shape [B, dim]. Sinusoidal embeddings.
-    """
-    half = dim // 2
-    freqs = torch.exp(
-        -math.log(max_period) * torch.arange(start=0, end=half, dtype=t.dtype, device=t.device) / half
-    )
-    args = t[:, None] * freqs[None]
-    embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
-    if dim % 2 == 1:
-        embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
-    return embedding
-
-
-# ------------------------------- Core Layers ----------------------------------
-
-
-class DomainAwareLinear(nn.Module):
-    """
-    Linear layer with domain-conditioned parameters (per-sample).
-
-    Each domain has its own weight and bias vectors, stored in embeddings.
-    """
-
-    def __init__(self, input_size: int, output_size: int, num_domains: int = 20) -> None:
-        super().__init__()
-        self.input_size = input_size
-        self.output_size = output_size
-        self.fc = nn.Embedding(num_domains, output_size * input_size)
-        self.bias = nn.Embedding(num_domains, output_size)
-        nn.init.xavier_uniform_(self.fc.weight)
-        nn.init.zeros_(self.bias.weight)
-
-    def forward(self, x: torch.Tensor, domain_id: torch.LongTensor) -> torch.Tensor:
-        """
-        Parameters
-        ----------
-        x : Tensor
-            [B, I] or [B, T, I]
-        domain_id : LongTensor
-            [B], domain indices.
-
-        Returns
-        -------
-        Tensor
-            [batch_size, output_size] or [batch_size, seq_len, output_size]
-        """
-        batch_size = domain_id.shape[0]
-        squeeze_seq = False
-        if x.dim() == 2:
-            x = x.unsqueeze(1)
-            squeeze_seq = True
-        weight = self.fc(domain_id).view(batch_size, self.input_size, self.output_size)
-        bias = self.bias(domain_id).view(batch_size, self.output_size)
-        y = torch.matmul(x, weight) + bias.view(batch_size, 1, self.output_size)
-        if squeeze_seq:
-            y = y.squeeze(1)
-        return y
-
-
-class TransformerBlock(nn.Module):
-    """
-    Standard Transformer block (pre-LN): LN → MHSA → residual, LN → MLP → residual.
-    """
-
-    def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float = 4.0) -> None:
-        super().__init__()
-        self.norm1 = nn.LayerNorm(hidden_size)
-        self.norm2 = nn.LayerNorm(hidden_size)
-        self.attn = Attention(hidden_size, num_heads=num_heads, qkv_bias=True, attn_drop=0.1)
-        self.mlp = Mlp(
-            in_features=hidden_size,
-            hidden_features=int(hidden_size * mlp_ratio),
-            drop=0.1,
-        )
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        """
-        Parameters
-        ----------
-        x : Tensor, [B, T, H]
-
-        Returns
-        -------
-        Tensor, [B, T, H]
-        """
-        x = x + self.attn(self.norm1(x))
-        x = x + self.mlp(self.norm2(x))
-        return x
-
-
-# --------------------------- Main Model ---------------------------------------
-
-
-class SoftPromptedTransformer(nn.Module):
-    """
-    Multi-modal, domain-aware Transformer with optional soft prompts.
-
-    See parameter and forward I/O descriptions inside the docstrings.
-    """
-
-    def __init__(
-        self,
-        hidden_size: int = 768,
-        multi_modal_input_size: int = 768,
-        depth: int = 24,
-        num_heads: int = 16,
-        mlp_ratio: float = 4.0,
-        num_domains: int = 20,
-        dim_action: int = 20,
-        dim_propio: int = 20,
-        dim_time: int = 32,
-        len_soft_prompts: int = 32,
-        max_len_seq: int = 512,
-        use_hetero_proj: bool = False,
-    ) -> None:
-        super().__init__()
-        self.hidden_size = hidden_size
-        self.dim_action = dim_action
-        self.dim_time = dim_time
-        self.len_soft_prompts = len_soft_prompts
-        self.use_hetero_proj = use_hetero_proj
-
-        self.blocks = nn.ModuleList(
-            [TransformerBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio) for _ in range(depth)]
-        )
-
-        if use_hetero_proj:
-            self.vlm_proj = DomainAwareLinear(multi_modal_input_size, hidden_size, num_domains=num_domains)
-            self.aux_visual_proj = DomainAwareLinear(
-                multi_modal_input_size, hidden_size, num_domains=num_domains
-            )
-        else:
-            self.vlm_proj = nn.Linear(multi_modal_input_size, hidden_size)
-            self.aux_visual_proj = nn.Linear(multi_modal_input_size, hidden_size)
-
-        self.pos_emb = nn.Parameter(torch.zeros(1, max_len_seq, hidden_size), requires_grad=True)
-        nn.init.normal_(self.pos_emb, std=0.02)
-
-        self.norm = nn.LayerNorm(hidden_size)
-        self.action_encoder = DomainAwareLinear(
-            dim_action + dim_time + dim_propio, hidden_size, num_domains=num_domains
-        )
-        self.action_decoder = DomainAwareLinear(hidden_size, dim_action, num_domains=num_domains)
-
-        if len_soft_prompts > 0:
-            self.soft_prompt_hub = nn.Embedding(num_domains, len_soft_prompts * hidden_size)
-            nn.init.normal_(self.soft_prompt_hub.weight, std=0.02)
-
-        self.apply(basic_init)
-
-    def forward(
-        self,
-        domain_id: torch.LongTensor,
-        vlm_features: torch.Tensor,
-        aux_visual_inputs: torch.Tensor,
-        action_with_noise: torch.Tensor,
-        proprio: torch.Tensor,
-        t: torch.Tensor,
-    ) -> torch.Tensor:
-        """
-        Forward pass.
-
-        Inputs
-        ------
-        domain_id : [B]
-        vlm_features : [B, T_vlm, D]
-        aux_visual_inputs : [B, T_aux, D]
-        action_with_noise : [B, T_action, dim_action]
-        proprio : [B, dim_propio]
-        t : [B]
-
-        Returns
-        -------
-        Tensor
-            Predicted actions, [batch_size, num_actions, dim_action]
-        """
-        batch_size, num_actions = action_with_noise.shape[:2]
-
-        # Encode (action + proprio + time) → tokens
-        time_emb = timestep_embedding(t, self.dim_time)  # [batch_size, dim_time]
-        time_tokens = time_emb.unsqueeze(1).expand(batch_size, num_actions, self.dim_time)
-        proprio_tokens = proprio.unsqueeze(1).expand(batch_size, num_actions, proprio.shape[-1])
-        action_tokens = torch.cat([action_with_noise, proprio_tokens, time_tokens], dim=-1)
-        x = self.action_encoder(action_tokens, domain_id)  # [batch_size, num_actions, hidden_size]
-
-        # Project visual streams and concatenate
-        if self.use_hetero_proj:
-            x = torch.cat(
-                [
-                    x,
-                    self.vlm_proj(vlm_features, domain_id),
-                    self.aux_visual_proj(aux_visual_inputs, domain_id),
-                ],
-                dim=1,
-            )
-        else:
-            x = torch.cat([x, self.vlm_proj(vlm_features), self.aux_visual_proj(aux_visual_inputs)], dim=1)
-
-        # Add positional embeddings (truncate if needed)
-        seq_len = x.shape[1]
-        if seq_len > self.pos_emb.shape[1]:
-            raise ValueError(f"Sequence length {seq_len} exceeds max_len_seq={self.pos_emb.shape[1]}.")
-        x = x + self.pos_emb[:, :seq_len, :]
-
-        # Append soft prompts
-        if self.len_soft_prompts > 0:
-            soft_prompts = self.soft_prompt_hub(domain_id).view(
-                batch_size, self.len_soft_prompts, self.hidden_size
-            )
-            x = torch.cat([x, soft_prompts], dim=1)
-
-        # Transformer backbone
-        for block in self.blocks:
-            x = block(x)
-
-        # Decode only the action segment
-        return self.action_decoder(self.norm(x[:, :num_actions]), domain_id)

src/lerobot/policies/xvla/utils.py

@@ -1,138 +0,0 @@
-import math
-
-import numpy as np
-
-
-def mat2quat(rmat):
-    """
-    Converts given rotation matrix to quaternion.
-
-    Args:
-        rmat (np.array): 3x3 rotation matrix
-
-    Returns:
-        np.array: (x,y,z,w) float quaternion angles
-    """
-    mat = np.asarray(rmat).astype(np.float32)[:3, :3]
-
-    m00 = mat[0, 0]
-    m01 = mat[0, 1]
-    m02 = mat[0, 2]
-    m10 = mat[1, 0]
-    m11 = mat[1, 1]
-    m12 = mat[1, 2]
-    m20 = mat[2, 0]
-    m21 = mat[2, 1]
-    m22 = mat[2, 2]
-    # symmetric matrix k
-    k = np.array(
-        [
-            [m00 - m11 - m22, np.float32(0.0), np.float32(0.0), np.float32(0.0)],
-            [m01 + m10, m11 - m00 - m22, np.float32(0.0), np.float32(0.0)],
-            [m02 + m20, m12 + m21, m22 - m00 - m11, np.float32(0.0)],
-            [m21 - m12, m02 - m20, m10 - m01, m00 + m11 + m22],
-        ]
-    )
-    k /= 3.0
-    # quaternion is Eigen vector of k that corresponds to largest eigenvalue
-    w, v = np.linalg.eigh(k)
-    inds = np.array([3, 0, 1, 2])
-    q1 = v[inds, np.argmax(w)]
-    if q1[0] < 0.0:
-        np.negative(q1, q1)
-    inds = np.array([1, 2, 3, 0])
-    return q1[inds]
-
-
-def quat2axisangle(quat):
-    """
-    Converts quaternion to axis-angle format.
-    Returns a unit vector direction scaled by its angle in radians.
-
-    Args:
-        quat (np.array): (x,y,z,w) vec4 float angles
-
-    Returns:
-        np.array: (ax,ay,az) axis-angle exponential coordinates
-    """
-    # clip quaternion
-    if quat[3] > 1.0:
-        quat[3] = 1.0
-    elif quat[3] < -1.0:
-        quat[3] = -1.0
-
-    den = np.sqrt(1.0 - quat[3] * quat[3])
-    if math.isclose(den, 0.0):
-        # This is (close to) a zero degree rotation, immediately return
-        return np.zeros(3)
-
-    return (quat[:3] * 2.0 * math.acos(quat[3])) / den
-
-
-def rotate6d_to_axis_angle(r6d):
-    """
-    r6d: np.ndarray, shape (N, 6)
-    return: np.ndarray, shape (N, 3), axis-angle vectors
-    """
-    flag = 0
-    if len(r6d.shape) == 1:
-        r6d = r6d[None, ...]
-        flag = 1
-
-    a1 = r6d[:, 0:3]
-    a2 = r6d[:, 3:6]
-
-    # b1
-    b1 = a1 / (np.linalg.norm(a1, axis=-1, keepdims=True) + 1e-6)
-
-    # b2
-    dot_prod = np.sum(b1 * a2, axis=-1, keepdims=True)
-    b2_orth = a2 - dot_prod * b1
-    b2 = b2_orth / (np.linalg.norm(b2_orth, axis=-1, keepdims=True) + 1e-6)
-
-    # b3
-    b3 = np.cross(b1, b2, axis=-1)
-
-    rotation_matrix = np.stack([b1, b2, b3], axis=-1)  # shape: (N, 3, 3)
-
-    axis_angle_list = []
-    for i in range(rotation_matrix.shape[0]):
-        quat = mat2quat(rotation_matrix[i])
-        axis_angle = quat2axisangle(quat)
-        axis_angle_list.append(axis_angle)
-
-    axis_angle_array = np.stack(axis_angle_list, axis=0)  # shape: (N, 3)
-
-    if flag == 1:
-        axis_angle_array = axis_angle_array[0]
-
-    return axis_angle_array
-
-
-def mat_to_rotate6d(abs_action):
-    if len(abs_action.shape) == 2:
-        return np.concatenate([abs_action[:3, 0], abs_action[:3, 1]], axis=-1)
-    elif len(abs_action.shape) == 3:
-        return np.concatenate([abs_action[:, :3, 0], abs_action[:, :3, 1]], axis=-1)
-    else:
-        raise NotImplementedError
-
-
-def drop_path(x, drop_prob: float = 0.0, training: bool = False, scale_by_keep: bool = True):
-    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
-
-    This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,
-    the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
-    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for
-    changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use
-    'survival rate' as the argument.
-
-    """
-    if drop_prob == 0.0 or not training:
-        return x
-    keep_prob = 1 - drop_prob
-    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
-    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
-    if keep_prob > 0.0 and scale_by_keep:
-        random_tensor.div_(keep_prob)
-    return x * random_tensor