fix(ci): use GITHUB_TOKEN for automated PR

fix(ci): latest deps tests permissions (#3296 )
* fix(ci): latest deps tests permissions * fix(ci): force push dep update branch * fix(ci): change secret for permissions & Ci trigger
2026-05-11 22:59:50 +00:00 · 2026-04-06 21:09:21 +02:00 · 2026-04-06 14:56:05 +02:00 · 2026-04-06 13:22:45 +02:00 · 2026-04-06 12:23:37 +02:00 · 2026-04-05 21:23:13 +02:00
40 changed files with 10484 additions and 596 deletions
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# This workflow handles nightly testing & docker images publishing.
-name: Nightly
+# This workflow handles Docker image publishing & testing.
+name: Docker Publish & Test
 permissions:
  contents: read

@@ -39,8 +39,8 @@ concurrency:

 jobs:
  # This job builds a CPU image for testing & distribution
-  build-docker-cpu-nightly:
-    name: Build CPU Docker for Nightly
+  build-docker-cpu:
+    name: Build CPU Docker
    runs-on:
      group: aws-general-8-plus
    if: github.repository == 'huggingface/lerobot'
@@ -74,8 +74,8 @@ jobs:
          tags: ${{ env.DOCKER_IMAGE_NAME_CPU }}

  # This job builds a GPU image for testing & distribution
-  build-docker-gpu-nightly:
-    name: Build GPU Docker for Nightly
+  build-docker-gpu:
+    name: Build GPU Docker
    runs-on:
      group: aws-general-8-plus
    if: github.repository == 'huggingface/lerobot'
@@ -109,9 +109,9 @@ jobs:
          tags: ${{ env.DOCKER_IMAGE_NAME_GPU }}

  # This job runs the E2E tests + pytest with all extras in the CPU image
-  nightly-cpu-tests:
-    name: Nightly CPU Tests
-    needs: [build-docker-cpu-nightly]
+  cpu-tests:
+    name: CPU Tests
+    needs: [build-docker-cpu]
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
@@ -121,7 +121,7 @@ jobs:
      TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    container:
-      image: ${{ needs.build-docker-cpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images]
+      image: ${{ needs.build-docker-cpu.outputs.image_tag }} # zizmor: ignore[unpinned-images]
      options: --shm-size "16gb"
      credentials:
        username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
@@ -142,9 +142,9 @@ jobs:
        run: make test-end-to-end

  # This job runs the E2E tests + pytest with all extras in the GPU image
-  nightly-gpu-tests:
-    name: Nightly GPU Tests
-    needs: [build-docker-gpu-nightly]
+  gpu-tests:
+    name: GPU Tests
+    needs: [build-docker-gpu]
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
@@ -154,7 +154,7 @@ jobs:
      TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    container:
-      image: ${{ needs.build-docker-gpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images]
+      image: ${{ needs.build-docker-gpu.outputs.image_tag }} # zizmor: ignore[unpinned-images]
      options: --gpus all --shm-size "16gb"
      credentials:
        username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
@@ -175,9 +175,9 @@ jobs:
        run: make test-end-to-end

  # This job runs multi-GPU training tests with 4 GPUs
-  nightly-multi-gpu-tests:
-    name: Nightly Multi-GPU Tests
-    needs: [build-docker-gpu-nightly]
+  multi-gpu-tests:
+    name: Multi-GPU Tests
+    needs: [build-docker-gpu]
    runs-on:
      group: aws-g4dn-12xlarge  # Instance with 4 GPUs
    env:
@@ -188,7 +188,7 @@ jobs:
      CUDA_VISIBLE_DEVICES: "0,1,2,3"
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    container:
-      image: ${{ needs.build-docker-gpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images]
+      image: ${{ needs.build-docker-gpu.outputs.image_tag }} # zizmor: ignore[unpinned-images]
      options: --gpus all --shm-size "16gb"
      credentials:
        username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
@@ -27,6 +27,7 @@ on:
      - "tests/**"
      - ".github/workflows/**"
      - "pyproject.toml"
+      - "uv.lock"
      - "Makefile"
  push:
    branches:
@@ -36,6 +37,7 @@ on:
      - "tests/**"
      - ".github/workflows/**"
      - "pyproject.toml"
+      - "uv.lock"
      - "Makefile"

 permissions:
@@ -88,7 +90,7 @@ jobs:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install lerobot with test extras
-        run: uv sync --extra "test"
+        run: uv sync --locked --extra "test"

      - name: Login to Hugging Face
        if: env.HF_USER_TOKEN != ''
@@ -29,6 +29,7 @@ on:
      - "tests/**"
      - ".github/workflows/**"
      - "pyproject.toml"
+      - "uv.lock"
      - "Makefile"

 permissions:
@@ -86,7 +87,7 @@ jobs:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install lerobot with all extras
-        run: uv sync --extra all # TODO(Steven): Make flash-attn optional
+        run: uv sync --locked --extra all # TODO(Steven): Make flash-attn optional

      - name: Login to Hugging Face
        if: env.HF_USER_TOKEN != ''
@@ -12,38 +12,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# This workflow handles full testing with unboud dependencies versions.
-name: Unbound Dependency Tests
+# This workflow tests the project against the latest upstream dependencies
+# (within pyproject.toml constraints) and opens a PR to update uv.lock
+# if the tests pass and the lockfile has changed.
+name: Latest Dependency Tests

 on:
  # Allows running this workflow manually from the Actions tab
  workflow_dispatch:

-  # Run on the 1st and 15th of every month at 09:00 UTC
-  # schedule:
-  #  - cron: '0 2 1,15 * *'
-
-permissions:
-  contents: read
+  # Runs at 03:00 UTC
+  schedule:
+    - cron: "0 3 * * *"

 # Sets up the environment variables
 env:
  UV_VERSION: "0.8.0"
  PYTHON_VERSION: "3.12"
-  DOCKER_IMAGE_NAME: huggingface/lerobot-gpu:unbound
+  DOCKER_IMAGE_NAME: huggingface/lerobot-gpu:latest-deps

-# Ensures that only the latest action is built, canceling older runs.
+# Ensures that only the latest run is active, canceling older runs.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}
  cancel-in-progress: true

 jobs:

-  # This job runs the E2E tests + pytest with all unbound extras
-  full-tests:
-    name: Full Unbound Tests
+  # This job upgrades the lockfile and checks if dependencies have changed
+  upgrade-lock:
+    name: Upgrade Lockfile
    runs-on: ubuntu-latest
    if: github.repository == 'huggingface/lerobot'
+    permissions:
+      contents: read
+    outputs:
+      changed: ${{ steps.diff.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup uv and Python
+        uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses]
+        with:
+          version: ${{ env.UV_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Upgrade uv.lock
+        run: uv lock --upgrade
+
+      - name: Check for changes
+        id: diff
+        run: |
+          if git diff --quiet uv.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "uv.lock is up to date — no dependency changes."
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "uv.lock has changed — running tests."
+          fi
+
+      - name: Upload updated lockfile
+        if: steps.diff.outputs.changed == 'true'
+        uses: actions/upload-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+          path: uv.lock
+
+  # This job runs the full test suite with the upgraded dependencies
+  cpu-tests:
+    name: CPU Tests (Latest Deps)
+    needs: [upgrade-lock]
+    if: needs.upgrade-lock.outputs.changed == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
    env:
      MUJOCO_GL: egl
      HF_HOME: /mnt/cache/.cache/huggingface
@@ -55,6 +98,11 @@ jobs:
          lfs: true
          persist-credentials: false

+      - name: Download updated lockfile
+        uses: actions/download-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+
      # NOTE(Steven): Mount to `/mnt` to avoid the limited storage on `/home`. Consider cleaning default SDKs or using self-hosted runners for more space.
      # (As of 2024-06-10, the runner's `/home` has only 6.2 GB free—8% of its 72 GB total.)
      - name: Setup /mnt storage
@@ -73,34 +121,32 @@ jobs:
          version: ${{ env.UV_VERSION }}
          python-version: ${{ env.PYTHON_VERSION }}

-      - name: Unbound dependencies
-        run: |
-          sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml
-          echo "Dependencies unbound:" && cat pyproject.toml
-
      - name: Install lerobot with all extras
-        run: uv sync --extra all # TODO(Steven): Make flash-attn optional
+        run: uv sync --locked --extra all # TODO(Steven): Make flash-attn optional
+
      - name: Login to Hugging Face
        if: env.HF_USER_TOKEN != ''
        run: |
          uv run hf auth login --token "$HF_USER_TOKEN" --add-to-git-credential
          uv run hf auth whoami
+
      - name: Run pytest (all extras)
-        run: uv run pytest tests -vv
+        run: uv run pytest tests -vv --maxfail=10

      - name: Run end-to-end tests
        run: uv run make test-end-to-end

-  # This job builds a GPU enabled image for testing
+  # This job builds a GPU-enabled Docker image with the upgraded dependencies
  build-and-push-docker:
    name: Build and Push Docker
+    needs: [upgrade-lock]
+    if: needs.upgrade-lock.outputs.changed == 'true'
+    permissions:
+      contents: read
    runs-on:
      group: aws-general-8-plus
-    if: github.repository == 'huggingface/lerobot'
    outputs:
      image_tag: ${{ env.DOCKER_IMAGE_NAME }}
-    env:
-      GITHUB_REF: ${{ github.ref }}
    steps:
      - name: Install Git LFS
        run: |
@@ -111,6 +157,12 @@ jobs:
        with:
          lfs: true
          persist-credentials: false
+
+      - name: Download updated lockfile
+        uses: actions/download-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses]
        with:
@@ -127,14 +179,13 @@ jobs:
          file: ./docker/Dockerfile.internal
          push: true
          tags: ${{ env.DOCKER_IMAGE_NAME }}
-          build-args: |
-            UNBOUND_DEPS=true

-  # This job runs pytest with all unbound extras in a GPU enabled host
-  # It runs everytime a test image is created
+  # This job runs pytest with all extras on a GPU-enabled host
  gpu-tests:
-    name: GPU Unbound Tests
+    name: GPU Tests (Latest Deps)
    needs: [build-and-push-docker]
+    permissions:
+      contents: read
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
@@ -159,17 +210,69 @@ jobs:
        run: |
          hf auth login --token "$HF_USER_TOKEN" --add-to-git-credential
          hf auth whoami
+      - name: Fix ptxas permissions
+        run: chmod +x /lerobot/.venv/lib/python3.12/site-packages/triton/backends/nvidia/bin/ptxas
      - name: Run pytest on GPU
-        run: pytest tests -vv
+        run: pytest tests -vv --maxfail=10
      - name: Run end-to-end tests
        run: make test-end-to-end

-  # This job deletes the test image recently created
-  # It runs everytime after the gpu-tests have finished
-  delete-unbound-image:
-    name: Delete Unbound Image
+  # This job creates or updates a PR with the upgraded lockfile
+  open-pr:
+    name: Open PR
+    needs: [cpu-tests, gpu-tests, upgrade-lock]
+    if: success() && needs.upgrade-lock.outputs.changed == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Download updated lockfile
+        uses: actions/download-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+
+      - name: Create or update PR
+        run: |
+          set -euo pipefail
+          BRANCH="auto/update-uv-lock"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git"
+
+          git checkout -B "$BRANCH"
+          git add uv.lock
+          git commit -m "chore(dependencies): update uv.lock"
+          git push --force origin "$BRANCH"
+
+          # Create PR only if one doesn't already exist for this branch
+          EXISTING_PR=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number')
+          if [ -z "$EXISTING_PR" ]; then
+            gh pr create \
+              --title "chore(dependencies): update uv.lock" \
+              --body "Automated update of \`uv.lock\` after successful latest dependency tests (CPU + GPU).
+
+          This PR upgrades all dependencies to their latest versions within the ranges specified in \`pyproject.toml\`." \
+              --head "$BRANCH" \
+              --base main
+          else
+            echo "PR #$EXISTING_PR already exists, branch has been updated."
+          fi
+
+  # This job deletes the temporary Docker image after tests complete
+  cleanup-docker:
+    name: Cleanup Docker Image
    needs: [gpu-tests, build-and-push-docker]
    if: always() && needs.build-and-push-docker.result == 'success'
+    permissions:
+      contents: read
    runs-on: ubuntu-latest
    steps:
      - name: Get Docker Hub Token and Delete Image
@@ -180,8 +283,7 @@ jobs:
          IMAGE_FULL: ${{ needs.build-and-push-docker.outputs.image_tag }}
        run: |
          IMAGE_NAME=$(echo "$IMAGE_FULL" | cut -d':' -f1)
-          IMAGE_TAG=$(echo "$IMAGE_FULL" | cut -d':' -f2)
-
+          IMAGE_TAG=$(echo "$IMAGE_FULL" | cut -d':' -f2-)
          echo "Attempting to delete image: $IMAGE_NAME:$IMAGE_TAG"

          TOKEN=$(curl -s -H "Content-Type: application/json" \
@@ -25,7 +25,6 @@ node_modules/

 # Lock files
 poetry.lock
-uv.lock
 Pipfile.lock

 ### Build & Distribution ###
@@ -4,7 +4,7 @@

 <div align="center">

-[![Tests](https://github.com/huggingface/lerobot/actions/workflows/nightly.yml/badge.svg?branch=main)](https://github.com/huggingface/lerobot/actions/workflows/nightly.yml?query=branch%3Amain)
+[![Tests](https://github.com/huggingface/lerobot/actions/workflows/docker_publish.yml/badge.svg?branch=main)](https://github.com/huggingface/lerobot/actions/workflows/docker_publish.yml?query=branch%3Amain)
 [![Python versions](https://img.shields.io/pypi/pyversions/lerobot)](https://www.python.org/downloads/)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/huggingface/lerobot/blob/main/LICENSE)
 [![Status](https://img.shields.io/pypi/status/lerobot)](https://pypi.org/project/lerobot/)
@@ -73,17 +73,10 @@ ENV HOME=/home/user_lerobot \
 RUN uv venv --python python${PYTHON_VERSION}

 # Install Python dependencies for caching
-COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml README.md MANIFEST.in ./
+COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
 COPY --chown=user_lerobot:user_lerobot src/ src/

-ARG UNBOUND_DEPS=false
-
-RUN if [ "$UNBOUND_DEPS" = "true" ]; then \
-    sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml; \
-    echo "Dependencies unbound:" && cat pyproject.toml; \
-    fi
-
-RUN uv pip install --no-cache ".[all]"
+RUN uv sync --locked --extra all --no-cache

 RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas

@@ -61,17 +61,10 @@ ENV HOME=/home/user_lerobot \
 RUN uv venv

 # Install Python dependencies for caching
-COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml README.md MANIFEST.in ./
+COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
 COPY --chown=user_lerobot:user_lerobot src/ src/

-ARG UNBOUND_DEPS=false
-
-RUN if [ "$UNBOUND_DEPS" = "true" ]; then \
-    sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml; \
-    echo "Dependencies unbound:" && cat pyproject.toml; \
-    fi
-
-RUN uv pip install --no-cache ".[all]"
+RUN uv sync --locked --extra all --no-cache

 # Copy the rest of the application code
 # Make sure to have the git-LFS files for testing
@@ -0,0 +1,77 @@
+# Docker
+
+This directory contains Dockerfiles for running LeRobot in containerized environments. Both images are **built nightly from `main`** and published to Docker Hub with the full environment pre-baked — no dependency setup required.
+
+## Pre-built Images
+
+```bash
+# CPU-only image (based on Dockerfile.user)
+docker pull huggingface/lerobot-cpu:latest
+
+# GPU image with CUDA support (based on Dockerfile.internal)
+docker pull huggingface/lerobot-gpu:latest
+```
+
+## Quick Start
+
+The fastest way to start training is to pull the GPU image and run `lerobot-train` directly. This is the same environment used for all of our CI, so it is a well-tested, batteries-included setup.
+
+```bash
+docker run -it --rm --gpus all --shm-size 16gb huggingface/lerobot-gpu:latest
+
+# inside the container:
+lerobot-train --policy.type=act --dataset.repo_id=lerobot/aloha_sim_transfer_cube_human
+```
+
+## Dockerfiles
+
+### `Dockerfile.user` (CPU)
+
+A lightweight image based on `python:3.12-slim`. Includes all Python dependencies and system libraries but does not include CUDA — there is no GPU support. Useful for exploring the codebase, running scripts, or working with robots, but not practical for training.
+
+### `Dockerfile.internal` (GPU)
+
+A CUDA-enabled image based on `nvidia/cuda`. This is the image for training — mostly used for internal interactions with the GPU cluster.
+
+## Usage
+
+### Running a pre-built image
+
+```bash
+# CPU
+docker run -it --rm huggingface/lerobot-cpu:latest
+
+# GPU
+docker run -it --rm --gpus all --shm-size 16gb huggingface/lerobot-gpu:latest
+```
+
+### Building locally
+
+From the repo root:
+
+```bash
+# CPU
+docker build -f docker/Dockerfile.user -t lerobot-user .
+docker run -it --rm lerobot-user
+
+# GPU
+docker build -f docker/Dockerfile.internal -t lerobot-internal .
+docker run -it --rm --gpus all --shm-size 16gb lerobot-internal
+```
+
+### Multi-GPU training
+
+To select specific GPUs, set `CUDA_VISIBLE_DEVICES` when launching the container:
+
+```bash
+# Use 4 GPUs
+docker run -it --rm --gpus all --shm-size 16gb \
+  -e CUDA_VISIBLE_DEVICES=0,1,2,3 \
+  huggingface/lerobot-gpu:latest
+```
+
+### USB device access (e.g. robots, cameras)
+
+```bash
+docker run -it --device=/dev/ -v /dev/:/dev/ --rm huggingface/lerobot-cpu:latest
+```
@@ -17,6 +17,8 @@
    title: Train RL in Simulation
  - local: multi_gpu_training
    title: Multi GPU training
+  - local: hil_data_collection
+    title: Human In the Loop Data Collection
  - local: peft_training
    title: Training with PEFT (e.g., LoRA)
  - local: rename_map
@@ -69,13 +71,17 @@
    title: Environments from the Hub
  - local: envhub_leisaac
    title: Control & Train Robots in Sim (LeIsaac)
+  title: "Simulation"
+- sections:
+  - local: adding_benchmarks
+    title: Adding a New Benchmark
+  - local: libero
+    title: LIBERO
+  - local: metaworld
+    title: Meta-World
  - local: envhub_isaaclab_arena
    title: NVIDIA IsaacLab Arena Environments
-  - local: libero
-    title: Using Libero
-  - local: metaworld
-    title: Using MetaWorld
-  title: "Simulation"
+  title: "Benchmarks"
 - sections:
  - local: introduction_processors
    title: Introduction to Robot Processors
@@ -0,0 +1,329 @@
+# Adding a New Benchmark
+
+This guide walks you through adding a new simulation benchmark to LeRobot. Follow the steps in order and use the existing benchmarks as templates.
+
+A benchmark in LeRobot is a set of [Gymnasium](https://gymnasium.farama.org/) environments that wrap a third-party simulator (like LIBERO or Meta-World) behind a standard `gym.Env` interface. The `lerobot-eval` CLI then runs evaluation uniformly across all benchmarks.
+
+## Existing benchmarks at a glance
+
+Before diving in, here is what is already integrated:
+
+| Benchmark      | Env file            | Config class       | Tasks               | Action dim   | Processor                    |
+| -------------- | ------------------- | ------------------ | ------------------- | ------------ | ---------------------------- |
+| LIBERO         | `envs/libero.py`    | `LiberoEnv`        | 130 across 5 suites | 7            | `LiberoProcessorStep`        |
+| Meta-World     | `envs/metaworld.py` | `MetaworldEnv`     | 50 (MT50)           | 4            | None                         |
+| IsaacLab Arena | Hub-hosted          | `IsaaclabArenaEnv` | Configurable        | Configurable | `IsaaclabArenaProcessorStep` |
+
+Use `src/lerobot/envs/libero.py` and `src/lerobot/envs/metaworld.py` as reference implementations.
+
+## How it all fits together
+
+### Data flow
+
+During evaluation, data moves through four stages:
+
+```
+1. gym.Env  ──→  raw observations (numpy dicts)
+
+2. Preprocessing  ──→  standard LeRobot keys + task description
+   (preprocess_observation, add_envs_task in envs/utils.py)
+
+3. Processors  ──→  env-specific then policy-specific transforms
+   (env_preprocessor, policy_preprocessor)
+
+4. Policy  ──→  select_action()  ──→  action tensor
+   then reverse: policy_postprocessor → env_postprocessor → numpy action → env.step()
+```
+
+Most benchmarks only need to care about stage 1 (producing observations in the right format) and optionally stage 3 (if env-specific transforms are needed).
+
+### Environment structure
+
+`make_env()` returns a nested dict of vectorized environments:
+
+```python
+dict[str, dict[int, gym.vector.VectorEnv]]
+#    ^suite       ^task_id
+```
+
+A single-task env (e.g. PushT) looks like `{"pusht": {0: vec_env}}`.
+A multi-task benchmark (e.g. LIBERO) looks like `{"libero_spatial": {0: vec0, 1: vec1, ...}, ...}`.
+
+### How evaluation runs
+
+All benchmarks are evaluated the same way by `lerobot-eval`:
+
+1. `make_env()` builds the nested `{suite: {task_id: VectorEnv}}` dict.
+2. `eval_policy_all()` iterates over every suite and task.
+3. For each task, it runs `n_episodes` rollouts via `rollout()`.
+4. Results are aggregated hierarchically: episode, task, suite, overall.
+5. Metrics include `pc_success` (success rate), `avg_sum_reward`, and `avg_max_reward`.
+
+The critical piece: your env must return `info["is_success"]` on every `step()` call. This is how the eval loop knows whether a task was completed.
+
+## What your environment must provide
+
+LeRobot does not enforce a strict observation schema. Instead it relies on a set of conventions that all benchmarks follow.
+
+### Env attributes
+
+Your `gym.Env` must set these attributes:
+
+| Attribute            | Type  | Why                                                  |
+| -------------------- | ----- | ---------------------------------------------------- |
+| `_max_episode_steps` | `int` | `rollout()` uses this to cap episode length          |
+| `task_description`   | `str` | Passed to VLA policies as a language instruction     |
+| `task`               | `str` | Fallback identifier if `task_description` is not set |
+
+### Success reporting
+
+Your `step()` and `reset()` must include `"is_success"` in the `info` dict:
+
+```python
+info = {"is_success": True}   # or False
+return observation, reward, terminated, truncated, info
+```
+
+### Observations
+
+The simplest approach is to map your simulator's outputs to the standard keys that `preprocess_observation()` already understands. Do this inside your `gym.Env` (e.g. in a `_format_raw_obs()` helper):
+
+| Your env should output    | LeRobot maps it to         | What it is                            |
+| ------------------------- | -------------------------- | ------------------------------------- |
+| `"pixels"` (single array) | `observation.image`        | Single camera image, HWC uint8        |
+| `"pixels"` (dict)         | `observation.images.<cam>` | Multiple cameras, each HWC uint8      |
+| `"agent_pos"`             | `observation.state`        | Proprioceptive state vector           |
+| `"environment_state"`     | `observation.env_state`    | Full environment state (e.g. PushT)   |
+| `"robot_state"`           | `observation.robot_state`  | Nested robot state dict (e.g. LIBERO) |
+
+If your simulator uses different key names, you have two options:
+
+1. **Recommended:** Rename them to the standard keys inside your `gym.Env` wrapper.
+2. **Alternative:** Write an env processor to transform observations after `preprocess_observation()` runs (see step 4 below).
+
+### Actions
+
+Actions are continuous numpy arrays in a `gym.spaces.Box`. The dimensionality depends on your benchmark (7 for LIBERO, 4 for Meta-World, etc.). Policies adapt to different action dimensions through their `input_features` / `output_features` config.
+
+### Feature declaration
+
+Each `EnvConfig` subclass declares two dicts that tell the policy what to expect:
+
+- `features` — maps feature names to `PolicyFeature(type, shape)` (e.g. action dim, image shape).
+- `features_map` — maps raw observation keys to LeRobot convention keys (e.g. `"agent_pos"` to `"observation.state"`).
+
+## Step by step
+
+<Tip>
+  At minimum, you need three files: a **gym.Env wrapper**, an **EnvConfig
+  subclass**, and a **factory dispatch branch**. Everything else is optional or
+  documentation.
+</Tip>
+
+### Checklist
+
+| File                                     | Required | Why                                       |
+| ---------------------------------------- | -------- | ----------------------------------------- |
+| `src/lerobot/envs/<benchmark>.py`        | Yes      | Wraps the simulator as a standard gym.Env |
+| `src/lerobot/envs/configs.py`            | Yes      | Registers your benchmark for the CLI      |
+| `src/lerobot/envs/factory.py`            | Yes      | Tells `make_env()` how to build your envs |
+| `src/lerobot/processor/env_processor.py` | Optional | Custom observation/action transforms      |
+| `src/lerobot/envs/utils.py`              | Optional | Only if you need new raw observation keys |
+| `pyproject.toml`                         | Yes      | Declares benchmark-specific dependencies  |
+| `docs/source/<benchmark>.mdx`            | Yes      | User-facing documentation page            |
+| `docs/source/_toctree.yml`               | Yes      | Adds your page to the docs sidebar        |
+
+### 1. The gym.Env wrapper (`src/lerobot/envs/<benchmark>.py`)
+
+Create a `gym.Env` subclass that wraps the third-party simulator:
+
+```python
+class MyBenchmarkEnv(gym.Env):
+    metadata = {"render_modes": ["rgb_array"], "render_fps": <fps>}
+
+    def __init__(self, task_suite, task_id, ...):
+        super().__init__()
+        self.task = <task_name_string>
+        self.task_description = <natural_language_instruction>
+        self._max_episode_steps = <max_steps>
+        self.observation_space = spaces.Dict({...})
+        self.action_space = spaces.Box(low=..., high=..., shape=(...,), dtype=np.float32)
+
+    def reset(self, seed=None, **kwargs):
+        ...  # return (observation, info) — info must contain {"is_success": False}
+
+    def step(self, action: np.ndarray):
+        ...  # return (obs, reward, terminated, truncated, info) — info must contain {"is_success": <bool>}
+
+    def render(self):
+        ...  # return RGB image as numpy array
+
+    def close(self):
+        ...
+```
+
+Also provide a factory function that returns the nested dict structure:
+
+```python
+def create_mybenchmark_envs(
+    task: str,
+    n_envs: int,
+    gym_kwargs: dict | None = None,
+    env_cls: type | None = None,
+) -> dict[str, dict[int, Any]]:
+    """Create {suite_name: {task_id: VectorEnv}} for MyBenchmark."""
+    ...
+```
+
+See `create_libero_envs()` (multi-suite, multi-task) and `create_metaworld_envs()` (difficulty-grouped tasks) for reference.
+
+### 2. The config (`src/lerobot/envs/configs.py`)
+
+Register a config dataclass so users can select your benchmark with `--env.type=<name>`:
+
+```python
+@EnvConfig.register_subclass("<benchmark_name>")
+@dataclass
+class MyBenchmarkEnvConfig(EnvConfig):
+    task: str = "<default_task>"
+    fps: int = <fps>
+    obs_type: str = "pixels_agent_pos"
+
+    features: dict[str, PolicyFeature] = field(default_factory=lambda: {
+        ACTION: PolicyFeature(type=FeatureType.ACTION, shape=(<action_dim>,)),
+    })
+    features_map: dict[str, str] = field(default_factory=lambda: {
+        ACTION: ACTION,
+        "agent_pos": OBS_STATE,
+        "pixels": OBS_IMAGE,
+    })
+
+    def __post_init__(self):
+        ...  # populate features based on obs_type
+
+    @property
+    def gym_kwargs(self) -> dict:
+        return {"obs_type": self.obs_type, "render_mode": self.render_mode}
+```
+
+Key points:
+
+- The `register_subclass` name is what users pass on the CLI (`--env.type=<name>`).
+- `features` tells the policy what the environment produces.
+- `features_map` maps raw observation keys to LeRobot convention keys.
+
+### 3. The factory dispatch (`src/lerobot/envs/factory.py`)
+
+Add a branch in `make_env()` to call your factory function:
+
+```python
+elif "<benchmark_name>" in cfg.type:
+    from lerobot.envs.<benchmark> import create_<benchmark>_envs
+
+    if cfg.task is None:
+        raise ValueError("<BenchmarkName> requires a task to be specified")
+
+    return create_<benchmark>_envs(
+        task=cfg.task,
+        n_envs=n_envs,
+        gym_kwargs=cfg.gym_kwargs,
+        env_cls=env_cls,
+    )
+```
+
+If your benchmark needs an env processor, add it in `make_env_pre_post_processors()`:
+
+```python
+if isinstance(env_cfg, MyBenchmarkEnvConfig) or "<benchmark_name>" in env_cfg.type:
+    preprocessor_steps.append(MyBenchmarkProcessorStep())
+```
+
+### 4. Env processor (optional — `src/lerobot/processor/env_processor.py`)
+
+Only needed if your benchmark requires observation transforms beyond what `preprocess_observation()` handles (e.g. image flipping, coordinate conversion):
+
+```python
+@dataclass
+@ProcessorStepRegistry.register(name="<benchmark>_processor")
+class MyBenchmarkProcessorStep(ObservationProcessorStep):
+    def _process_observation(self, observation):
+        processed = observation.copy()
+        # your transforms here
+        return processed
+
+    def transform_features(self, features):
+        return features  # update if shapes change
+
+    def observation(self, observation):
+        return self._process_observation(observation)
+```
+
+See `LiberoProcessorStep` for a full example (image rotation, quaternion-to-axis-angle conversion).
+
+### 5. Dependencies (`pyproject.toml`)
+
+Add a new optional-dependency group:
+
+```toml
+mybenchmark = ["my-benchmark-pkg==1.2.3", "lerobot[scipy-dep]"]
+```
+
+Pinning rules:
+
+- **Always pin** benchmark packages to exact versions for reproducibility (e.g. `metaworld==3.0.0`).
+- **Add platform markers** when needed (e.g. `; sys_platform == 'linux'`).
+- **Pin fragile transitive deps** if known (e.g. `gymnasium==1.1.0` for Meta-World).
+- **Document constraints** in your benchmark doc page.
+
+Users install with:
+
+```bash
+pip install -e ".[mybenchmark]"
+```
+
+### 6. Documentation (`docs/source/<benchmark>.mdx`)
+
+Write a user-facing page following the template in the next section. See `docs/source/libero.mdx` and `docs/source/metaworld.mdx` for full examples.
+
+### 7. Table of contents (`docs/source/_toctree.yml`)
+
+Add your benchmark to the "Benchmarks" section:
+
+```yaml
+- sections:
+    - local: libero
+      title: LIBERO
+    - local: metaworld
+      title: Meta-World
+    - local: envhub_isaaclab_arena
+      title: NVIDIA IsaacLab Arena Environments
+    - local: <your_benchmark>
+      title: <Your Benchmark Name>
+  title: "Benchmarks"
+```
+
+## Verifying your integration
+
+After completing the steps above, confirm that everything works:
+
+1. **Install** — `pip install -e ".[mybenchmark]"` and verify the dependency group installs cleanly.
+2. **Smoke test env creation** — call `make_env()` with your config in Python, check that the returned dict has the expected `{suite: {task_id: VectorEnv}}` shape, and that `reset()` returns observations with the right keys.
+3. **Run a full eval** — `lerobot-eval --env.type=<name> --env.task=<task> --eval.n_episodes=1 --eval.batch_size=1 --policy.path=<any_compatible_policy>` to exercise the full pipeline end-to-end.
+4. **Check success detection** — verify that `info["is_success"]` flips to `True` when the task is actually completed. This is what the eval loop uses to compute success rates.
+
+## Writing a benchmark doc page
+
+Each benchmark `.mdx` page should include:
+
+- **Title and description** — 1-2 paragraphs on what the benchmark tests and why it matters.
+- **Links** — paper, GitHub repo, project website (if available).
+- **Overview image or GIF.**
+- **Available tasks** — table of task suites with counts and brief descriptions.
+- **Installation** — `pip install -e ".[<benchmark>]"` plus any extra steps (env vars, system packages).
+- **Evaluation** — recommended `lerobot-eval` command with `n_episodes` and `batch_size` for reproducible results. Include single-task and multi-task examples if applicable.
+- **Policy inputs and outputs** — observation keys with shapes, action space description.
+- **Recommended evaluation episodes** — how many episodes per task is standard.
+- **Training** — example `lerobot-train` command.
+- **Reproducing published results** — link to pretrained model, eval command, results table (if available).
+
+See `docs/source/libero.mdx` and `docs/source/metaworld.mdx` for complete examples.
@@ -131,4 +131,4 @@ lerobot-record \

 ## License

-This model follows the **Apache 2.0 License**, consistent with the original [GR00T repository](https://github.com/NVIDIA/Isaac-GR00T).
+This model follows NVIDIA's proprietary license, consistent with the original [GR00T repository](https://github.com/NVIDIA/Isaac-GR00T). Future versions (starting from N1.7) will follow **Apache 2.0 License**.
@@ -0,0 +1,269 @@
+# Human-In-the-Loop Data Collection
+
+Human-In-the-Loop (HIL) data collection lets you improve a trained policy by deploying it on a real robot while a human operator monitors and intervenes when needed. The intervention data (recovery movements and corrections) is recorded alongside autonomous segments, producing a richer training dataset that teaches the policy how to handle failures.
+
+---
+
+## Why Human-In-the-Loop?
+
+Standard behavioral cloning trains policies on successful demonstrations only. During deployment, small errors can compound and push the robot into states never seen during training (distribution shift). HIL data collection addresses this by:
+
+- Running the trained policy on the real robot
+- Having a human intervene when the robot is about to fail
+- Recording the human's recovery and correction as training data
+- Fine-tuning the policy on the combined dataset
+
+This produces a policy that not only knows how to perform the task, but also how to recover when things go wrong.
+
+---
+
+## How It Works
+
+During a HIL session, the human operator follows this loop within each episode:
+
+1. **Watch** the policy run autonomously
+2. **Pause** when failure is imminent, the robot holds its position
+3. **Take control** and teleoperate the robot back to a good state (recovery), then correct the behavior
+4. **Return control to the policy**, the policy resumes autonomous execution
+5. Repeat steps 2–4 as many times as needed during the episode
+6. **End the episode** when the task is complete, save and move on to the next rollout
+
+Both autonomous and human-controlled segments are recorded. The policy and human can alternate control multiple times within a single episode, and the episode continues from the current state after each handoff (no reset required just because intervention happened). This captures autonomous execution, recovery, and correction in one continuous trajectory. After collection, the combined dataset (original demonstrations + HIL data) is used to fine-tune the policy.
+
+This process can be repeated iteratively: deploy, collect, fine-tune, repeat. Each round targets the current policy's failure modes.
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│  Policy v0 (trained on demos)                                           │
+│       ↓                                                                 │
+│  HIL Collection (target current failure modes) → Fine-tune → Policy v1  │
+│       ↓                                                                 │
+│  HIL Collection (target new failure modes) → Fine-tune → Policy v2      │
+│       ↓                                                                 │
+│  ... (repeat until satisfactory performance)                            │
+└─────────────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Hardware Requirements
+
+### Teleoperator Requirements
+
+The `examples/hil` HIL scripts require **teleoperators with active motors** that can:
+
+- Enable/disable torque programmatically
+- Move to target positions (to mirror the robot state when pausing)
+
+**Compatible teleoperators in the current `examples/hil` scripts:**
+
+- `openarm_mini` - OpenArm Mini
+- `so_leader` - SO100 / SO101 leader arm
+
+> [!IMPORTANT]
+> The provided `examples/hil` commands default to `bi_openarm_follower` + `openarm_mini`.
+> `so_follower` + `so_leader` configs are also registered and can be used via CLI flags.
+
+---
+
+## Script
+
+A single script handles both synchronous and RTC-based inference. Toggle RTC with `--rtc.enabled=true`:
+
+| Mode                     | Flag                 | Models                |
+| ------------------------ | -------------------- | --------------------- |
+| Standard (default)       | _(no flag needed)_   | ACT, Diffusion Policy |
+| Real-Time Chunking (RTC) | `--rtc.enabled=true` | Pi0, Pi0.5, SmolVLA   |
+
+---
+
+## Step-by-Step Guide
+
+### Step 1: Pre-train a Base Policy
+
+First, train a policy on your demonstration dataset:
+
+```bash
+python src/lerobot/scripts/lerobot_train.py \
+    --dataset.repo_id=your-username/demo-dataset \
+    --policy.type=pi0 \
+    --output_dir=outputs/pretrain \
+    --batch_size=32 \
+    --steps=50000
+```
+
+### Step 2: Collect HIL Data
+
+**Standard inference (ACT, Diffusion Policy):**
+
+```bash
+python examples/hil/hil_data_collection.py \
+    --robot.type=bi_openarm_follower \
+    --robot.left_arm_config.port=can1 \
+    --robot.left_arm_config.side=left \
+    --robot.right_arm_config.port=can0 \
+    --robot.right_arm_config.side=right \
+    --robot.cameras='{left_wrist: {type: opencv, index_or_path: "/dev/video0", width: 1280, height: 720, fps: 30}, right_wrist: {type: opencv, index_or_path: "/dev/video4", width: 1280, height: 720, fps: 30}, base: {type: opencv, index_or_path: "/dev/video2", width: 640, height: 480, fps: 30}}' \
+    --teleop.type=openarm_mini \
+    --teleop.port_left=/dev/ttyACM0 \
+    --teleop.port_right=/dev/ttyACM1 \
+    --policy.path=outputs/pretrain/checkpoints/last/pretrained_model \
+    --dataset.repo_id=your-username/hil-dataset \
+    --dataset.single_task="Fold the T-shirt properly" \
+    --dataset.fps=30 \
+    --dataset.episode_time_s=1000 \
+    --dataset.num_episodes=50 \
+    --interpolation_multiplier=2
+```
+
+**With RTC for large models (Pi0, Pi0.5, SmolVLA):**
+
+For models with high inference latency, enable RTC for smooth execution:
+
+```bash
+python examples/hil/hil_data_collection.py \
+    --rtc.enabled=true \
+    --rtc.execution_horizon=20 \
+    --rtc.max_guidance_weight=5.0 \
+    --rtc.prefix_attention_schedule=LINEAR \
+    --robot.type=bi_openarm_follower \
+    --robot.left_arm_config.port=can1 \
+    --robot.left_arm_config.side=left \
+    --robot.right_arm_config.port=can0 \
+    --robot.right_arm_config.side=right \
+    --robot.cameras='{left_wrist: {type: opencv, index_or_path: "/dev/video0", width: 1280, height: 720, fps: 30}, right_wrist: {type: opencv, index_or_path: "/dev/video4", width: 1280, height: 720, fps: 30}, base: {type: opencv, index_or_path: "/dev/video2", width: 640, height: 480, fps: 30}}' \
+    --teleop.type=openarm_mini \
+    --teleop.port_left=/dev/ttyACM0 \
+    --teleop.port_right=/dev/ttyACM1 \
+    --policy.path=outputs/pretrain/checkpoints/last/pretrained_model \
+    --dataset.repo_id=your-username/hil-rtc-dataset \
+    --dataset.single_task="Fold the T-shirt properly" \
+    --dataset.fps=30 \
+    --dataset.episode_time_s=1000 \
+    --dataset.num_episodes=50 \
+    --interpolation_multiplier=3
+```
+
+**Controls (Conceptual):**
+
+The interaction model is:
+
+- **Pause input**: pause autonomous policy execution
+- **Takeover input**: transfer control to the human operator and record intervention data
+- **Return-to-policy input**: hand control back to the policy and continue the same episode
+- **Episode control inputs**: save/re-record/stop/reset as needed
+
+Exact key/pedal bindings can differ across scripts and hardware integrations. Use each script's printed controls as the source of truth for the concrete mapping on your setup.
+
+**The HIL Protocol:**
+
+1. Watch the policy run autonomously (teleop is idle/free)
+2. When you see imminent failure, trigger the **pause input**
+   - Policy stops
+   - Teleoperator moves to match robot position (torque enabled)
+   - No frames recorded during pause
+3. Trigger the **takeover input** to take control
+   - Teleoperator torque disabled, free to move
+   - **Recovery**: Teleoperate the robot back to a good state
+   - **Correction**: Correct the behavior
+   - All movements are recorded
+4. Trigger the **return-to-policy input**
+   - Policy resumes autonomous execution from the current state
+   - You can intervene again at any time (repeat steps 2–4)
+5. End and save the episode when the task is complete (or episode time limit is reached)
+6. **Reset**: Teleop moves to robot position, you can move the robot to the starting position
+7. Start the next episode
+
+**Foot Pedal Setup (Linux):**
+
+If using a USB foot pedal (PCsensor FootSwitch), ensure access:
+
+```bash
+sudo setfacl -m u:$USER:rw /dev/input/by-id/usb-PCsensor_FootSwitch-event-kbd
+```
+
+### Step 3: Fine-tune the Policy
+
+Fine-tune on the **combined** dataset (`demo-dataset` + `hil-dataset` merged together):
+
+```bash
+python src/lerobot/scripts/lerobot_train.py \
+    --dataset.repo_id=your-username/hil-dataset \
+    --policy.type=pi0 \
+    --policy.pretrained_path=outputs/pretrain/checkpoints/last/pretrained_model \
+    --output_dir=outputs/hil_finetune \
+    --steps=20000
+```
+
+Then deploy the fine-tuned policy and repeat from Step 2 to target its remaining failure modes.
+
+---
+
+## Tips for Effective HIL Collection
+
+### When to Intervene
+
+Intervene when you see:
+
+- Robot about to make an irreversible mistake
+- Robot hesitating or showing uncertain behavior
+- Robot deviating from the expected trajectory
+
+### Recovery: Teleoperating Back to a Good State
+
+During recovery, teleoperate the robot back to a state where:
+
+- The robot is in a familiar, in-distribution configuration
+- The current subtask can still be completed
+- The recovery trajectory itself is informative training data
+
+### Quality of Corrections
+
+During correction:
+
+- Provide **confident, clean** trajectories
+- Complete the current subtask fully
+- Don't overcorrect or add unnecessary movements
+
+---
+
+## Related Work
+
+This HIL data collection approach builds on ideas from interactive imitation learning:
+
+- **DAgger** (Ross et al., 2011) introduced the core idea: instead of only training on expert demonstrations, query the expert for corrections on states the _learner_ visits. This breaks the compounding-error cycle of standard behavioral cloning by iteratively collecting on-policy data.
+
+- **HG-DAgger** (Kelly et al., 2019) made this practical for robotics: a human expert monitors the robot and only intervenes when needed, rather than labeling every state. The gating between autonomous and human control is exactly the pause → takeover → return-to-policy loop used in the scripts here.
+
+- **RaC** (Hu et al., 2025) scales this loop to long-horizon tasks by explicitly decomposing interventions into **recovery** (teleoperating back to a good state) and **correction** (demonstrating the right behavior from there). This decomposition is the protocol followed by the HIL scripts in `examples/hil`.
+
+- **π0.6/RECAP** (Physical Intelligence, 2025) applies the same iterative collect-and-finetune loop at scale with VLA models, showing that even large pretrained policies benefit substantially from targeted human corrections on their own failure modes. π0.6 is trained using RECAP.
+
+```bibtex
+@article{ross2011dagger,
+  title={A Reduction of Imitation Learning and Structured Prediction to No-Regret Online Learning},
+  author={Ross, Stéphane and Gordon, Geoffrey and Bagnell, Drew},
+  journal={Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},
+  year={2011}
+}
+
+@article{kelly2019hgdagger,
+  title={HG-DAgger: Interactive Imitation Learning with Human Experts},
+  author={Kelly, Michael and Sidrane, Chelsea and Driggs-Campbell, Katherine and Kochenderfer, Mykel J},
+  journal={arXiv preprint arXiv:1810.02890},
+  year={2019}
+}
+
+@article{hu2025rac,
+  title={RaC: Robot Learning for Long-Horizon Tasks by Scaling Recovery and Correction},
+  author={Hu, Zheyuan and Wu, Robyn and Enock, Naveen and Li, Jasmine and Kadakia, Riya and Erickson, Zackory and Kumar, Aviral},
+  journal={arXiv preprint arXiv:2509.07953},
+  year={2025}
+}
+
+@article{pi2025recap,
+  title={π0.6: a VLA That Learns From Experience},
+  author={Physical Intelligence},
+  year={2025}
+}
+```
@@ -1,6 +1,6 @@
 # Installation

-This guide uses `conda` (via miniforge) to manage environments (recommended). If you prefer another environment manager (e.g. `uv`, `venv`), ensure you have Python >=3.12 and `ffmpeg` installed with the `libsvtav1` encoder, then skip ahead to [Environment Setup](#step-2-environment-setup).
+This guide uses `conda` (via miniforge) to manage environments (recommended). If you prefer another environment manager (e.g. `uv`, `venv`), ensure you have Python >=3.12 and support PyTorch >= 2.10, then skip ahead to [Environment Setup](#step-2-environment-setup).

 ## Step 1 (`conda` only): Install [`miniforge`](https://conda-forge.org/download/)

@@ -20,7 +20,7 @@ Create a virtual environment with Python 3.12:
 conda create -y -n lerobot python=3.12
 ```
 </hfoption>
-<hfoption id="uv">
+<hfoption id="uv (PyTorch >= 2.10 only)">
 ```bash
 uv python install 3.12
 uv venv --python 3.12
@@ -32,48 +32,87 @@ uv venv --python 3.12
 Then activate your virtual environment, you have to do this each time you open a shell to use lerobot:

 <!-- prettier-ignore-start -->
+
 <hfoptions id="activate_venv">
-<hfoption id="conda">```bash
+<hfoption id="conda">
+```bash
 conda activate lerobot
-```</hfoption>
-<hfoption id="uv">
-```bash
-# Linux/macOSsource
-source .venv/bin/activate
-# Windows PowerShell
-source .venv\Scripts\Activate.ps1
 ```
-</hfoption>
-</hfoptions>
-<!-- prettier-ignore-end -->
-
-When using `conda`, install `ffmpeg` in your environment:
-
-```bash
-conda install ffmpeg -c conda-forge
-ffmpeg -version  # ffmpeg 8.X is not yet supported !
-```
-
-> [!TIP]
-> This usually installs `ffmpeg 7.X` for your platform compiled with the `libsvtav1` encoder. If `libsvtav1` is not supported (check supported encoders with `ffmpeg -encoders`), you can:
->
-> - _[On any platform]_ Explicitly install `ffmpeg 7.X` using:
->
-> ```bash
-> conda install ffmpeg=7.1.1 -c conda-forge
-> ```
->
-> - _[On Linux only]_ If you want to bring your own ffmpeg: Install [ffmpeg build dependencies](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu#GettheDependencies) and [compile ffmpeg from source with libsvtav1](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu#libsvtav1), and make sure you use the corresponding ffmpeg binary to your install with `which ffmpeg`.

 > [!NOTE]
-> When installing LeRobot inside WSL (Windows Subsystem for Linux), make sure to install `evdev` with the following command:
+> When installing LeRobot inside WSL (Windows Subsystem for Linux), make sure to also install `evdev`:
 >
 > ```bash
 > conda install evdev -c conda-forge
 > ```

+</hfoption>
+<hfoption id="uv (PyTorch >= 2.10 only)">
+```bash
+# Linux/macOS
+source .venv/bin/activate
+# Windows PowerShell
+.venv\Scripts\activate
+```
+
+> [!NOTE]
+> When installing LeRobot inside WSL (Windows Subsystem for Linux), make sure to also install `evdev`:
+>
+> ```bash
+> sudo apt install libevdev-dev
+> uv pip install evdev
+> ```
+
+</hfoption>
+</hfoptions>
+<!-- prettier-ignore-end -->
+
+### Install `ffmpeg` (for video decoding)
+
+LeRobot uses [TorchCodec](https://github.com/meta-pytorch/torchcodec) for video decoding by default, which requires `ffmpeg`.
+
+> [!NOTE]
+> **Platform support:** TorchCodec is **not available** on macOS Intel (x86_64), Linux ARM (aarch64, arm64, armv7l), or Windows with PyTorch < 2.8. On these platforms, LeRobot automatically falls back to `pyav` — so you do not need to install `ffmpeg` and can skip to Step 3.
+
+If your platform supports TorchCodec, install `ffmpeg` using one of the methods below:
+
+<!-- prettier-ignore-start -->
+
+<hfoptions id="install_ffmpeg">
+<hfoption id="conda (any PyTorch version)">
+
+Install `ffmpeg` in your conda environment. This works with **all PyTorch versions** and is **required for PyTorch < 2.10**:
+
+```bash
+conda install ffmpeg -c conda-forge
+```
+
+> [!TIP]
+> This usually installs `ffmpeg 8.X` with the `libsvtav1` encoder. If you run into issues (e.g. `libsvtav1` missing — check with `ffmpeg -encoders` — or a version mismatch with `torchcodec`), you can explicitly install `ffmpeg 7.1.1` using:
+>
+> ```bash
+> conda install ffmpeg=7.1.1 -c conda-forge
+> ```
+
+</hfoption>
+<hfoption id="uv (PyTorch >= 2.10 only)">
+
+Starting with **PyTorch >= 2.10** (TorchCodec ≥ 0.10), TorchCodec can dynamically link to a system-wide `ffmpeg` installation. This is useful when using `uv` or other non-`conda` environment managers:
+
+```bash
+# Ubuntu/Debian
+sudo apt install ffmpeg
+
+# macOS (Apple Silicon)
+brew install ffmpeg
+```
+
 > [!IMPORTANT]
-> If you are using `uv` you will have to install `ffmpeg` system-wide (outside of the virtual environment). You rely on `uv` and `torchcodec` ability to dynamically link to the system `ffmpeg`.
+> System-wide `ffmpeg` is **only supported with PyTorch >= 2.10** (TorchCodec ≥ 0.10). For older PyTorch versions, you **must** use `conda install ffmpeg -c conda-forge` instead.
+
+</hfoption>
+</hfoptions>
+<!-- prettier-ignore-end -->

 ## Step 3: Install LeRobot 🤗

@@ -1,36 +1,61 @@
 # LIBERO

-**LIBERO** is a benchmark designed to study **lifelong robot learning**. The idea is that robots won’t just be pretrained once in a factory, they’ll need to keep learning and adapting with their human users over time. This ongoing adaptation is called **lifelong learning in decision making (LLDM)**, and it’s a key step toward building robots that become truly personalized helpers.
+LIBERO is a benchmark designed to study **lifelong robot learning** — the idea that robots need to keep learning and adapting with their users over time, not just be pretrained once. It provides a set of standardized manipulation tasks that focus on **knowledge transfer**: how well a robot can apply what it has already learned to new situations. By evaluating on LIBERO, different algorithms can be compared fairly and researchers can build on each other's work.

- 📄 [LIBERO paper](https://arxiv.org/abs/2306.03310)
- 💻 [Original LIBERO repo](https://github.com/Lifelong-Robot-Learning/LIBERO)
-
-To make progress on this challenge, LIBERO provides a set of standardized tasks that focus on **knowledge transfer**: how well a robot can apply what it has already learned to new situations. By evaluating on LIBERO, different algorithms can be compared fairly and researchers can build on each other’s work.
-
-LIBERO includes **five task suites**:
-
- **LIBERO-Spatial (`libero_spatial`)** – tasks that require reasoning about spatial relations.
- **LIBERO-Object (`libero_object`)** – tasks centered on manipulating different objects.
- **LIBERO-Goal (`libero_goal`)** – goal-conditioned tasks where the robot must adapt to changing targets.
- **LIBERO-90 (`libero_90`)** – 90 short-horizon tasks from the LIBERO-100 collection.
- **LIBERO-Long (`libero_10`)** – 10 long-horizon tasks from the LIBERO-100 collection.
-
-Together, these suites cover **130 tasks**, ranging from simple object manipulations to complex multi-step scenarios. LIBERO is meant to grow over time, and to serve as a shared benchmark where the community can test and improve lifelong learning algorithms.
+- Paper: [Benchmarking Knowledge Transfer for Lifelong Robot Learning](https://arxiv.org/abs/2306.03310)
+- GitHub: [Lifelong-Robot-Learning/LIBERO](https://github.com/Lifelong-Robot-Learning/LIBERO)
+- Project website: [libero-project.github.io](https://libero-project.github.io)

 ![An overview of the LIBERO benchmark](https://libero-project.github.io/assets/img/libero/fig1.png)

-## Evaluating with LIBERO
+## Available tasks

-At **LeRobot**, we ported [LIBERO](https://github.com/Lifelong-Robot-Learning/LIBERO) into our framework and used it mainly to **evaluate [SmolVLA](https://huggingface.co/docs/lerobot/en/smolvla)**, our lightweight Vision-Language-Action model.
+LIBERO includes **five task suites** covering **130 tasks**, ranging from simple object manipulations to complex multi-step scenarios:

-LIBERO is now part of our **multi-eval supported simulation**, meaning you can benchmark your policies either on a **single suite of tasks** or across **multiple suites at once** with just a flag.
+| Suite          | CLI name         | Tasks | Description                                        |
+| -------------- | ---------------- | ----- | -------------------------------------------------- |
+| LIBERO-Spatial | `libero_spatial` | 10    | Tasks requiring reasoning about spatial relations  |
+| LIBERO-Object  | `libero_object`  | 10    | Tasks centered on manipulating different objects   |
+| LIBERO-Goal    | `libero_goal`    | 10    | Goal-conditioned tasks with changing targets       |
+| LIBERO-90      | `libero_90`      | 90    | Short-horizon tasks from the LIBERO-100 collection |
+| LIBERO-Long    | `libero_10`      | 10    | Long-horizon tasks from the LIBERO-100 collection  |

-To Install LIBERO, after following LeRobot official instructions, just do:
-`pip install -e ".[libero]"`
+## Installation
+
+After following the LeRobot installation instructions:
+
+```bash
+pip install -e ".[libero]"
+```
+
+<Tip>
+LIBERO requires Linux (`sys_platform == 'linux'`). LeRobot uses MuJoCo for simulation — set the rendering backend before training or evaluation:
+
+```bash
+export MUJOCO_GL=egl  # for headless servers (HPC, cloud)
+```
+
+</Tip>
+
+## Evaluation
+
+### Default evaluation (recommended)
+
+Evaluate across the four standard suites (10 episodes per task):
+
+```bash
+lerobot-eval \
+  --policy.path="your-policy-id" \
+  --env.type=libero \
+  --env.task=libero_spatial,libero_object,libero_goal,libero_10 \
+  --eval.batch_size=1 \
+  --eval.n_episodes=10 \
+  --env.max_parallel_tasks=1
+```

 ### Single-suite evaluation

-Evaluate a policy on one LIBERO suite:
+Evaluate on one LIBERO suite:

 ```bash
 lerobot-eval \
@@ -42,15 +67,13 @@ lerobot-eval \
 ```

 - `--env.task` picks the suite (`libero_object`, `libero_spatial`, etc.).
- `--env.task_ids` picks task ids to run (`[0]`, `[1,2,3]`, etc.). Omit this flag (or set it to `null`) to run all tasks in the suite.
+- `--env.task_ids` restricts to specific task indices (`[0]`, `[1,2,3]`, etc.). Omit to run all tasks in the suite.
 - `--eval.batch_size` controls how many environments run in parallel.
- `--eval.n_episodes` sets how many episodes to run in total.
-
---
+- `--eval.n_episodes` sets how many episodes to run per task.

 ### Multi-suite evaluation

-Benchmark a policy across multiple suites at once:
+Benchmark a policy across multiple suites at once by passing a comma-separated list:

 ```bash
 lerobot-eval \
@@ -61,50 +84,49 @@ lerobot-eval \
  --eval.n_episodes=2
 ```

- Pass a comma-separated list to `--env.task` for multi-suite evaluation.
+### Control mode

-### Control Mode
+LIBERO supports two control modes — `relative` (default) and `absolute`. Different VLA checkpoints are trained with different action parameterizations, so make sure the mode matches your policy:

-LIBERO now supports two control modes: relative and absolute. This matters because different VLA checkpoints are trained with different mode of action to output hence control parameterizations.
-You can switch them with: `env.control_mode = "relative"` and `env.control_mode = "absolute"`
+```bash
+--env.control_mode=relative   # or "absolute"
+```

 ### Policy inputs and outputs

-When using LIBERO through LeRobot, policies interact with the environment via **observations** and **actions**:
+**Observations:**

- **Observations**
-  - `observation.state` – proprioceptive features (agent state).
-  - `observation.images.image` – main camera view (`agentview_image`).
-  - `observation.images.image2` – wrist camera view (`robot0_eye_in_hand_image`).
+- `observation.state` — 8-dim proprioceptive features (eef position, axis-angle orientation, gripper qpos)
+- `observation.images.image` — main camera view (`agentview_image`), HWC uint8
+- `observation.images.image2` — wrist camera view (`robot0_eye_in_hand_image`), HWC uint8

-  ⚠️ **Note:** LeRobot enforces the `.images.*` prefix for any multi-modal visual features. Always ensure that your policy config `input_features` use the same naming keys, and that your dataset metadata keys follow this convention during evaluation.
-  If your data contains different keys, you must rename the observations to match what the policy expects, since naming keys are encoded inside the normalization statistics layer.
-  This will be fixed with the upcoming Pipeline PR.
+<Tip warning={true}>
+  LeRobot enforces the `.images.*` prefix for visual features. Ensure your
+  policy config `input_features` use the same naming keys, and that your dataset
+  metadata keys follow this convention. If your data contains different keys,
+  you must rename the observations to match what the policy expects, since
+  naming keys are encoded inside the normalization statistics layer.
+</Tip>

- **Actions**
-  - Continuous control values in a `Box(-1, 1, shape=(7,))` space.
+**Actions:**

-We also provide a notebook for quick testing:
-Training with LIBERO
+- Continuous control in `Box(-1, 1, shape=(7,))` — 6D end-effector delta + 1D gripper

-## Training with LIBERO
+### Recommended evaluation episodes

-When training on LIBERO tasks, make sure your dataset parquet and metadata keys follow the LeRobot convention.
+For reproducible benchmarking, use **10 episodes per task** across all four standard suites (Spatial, Object, Goal, Long). This gives 400 total episodes and matches the protocol used for published results.

-The environment expects:
+## Training

- `observation.state` → 8-dim agent state
- `observation.images.image` → main camera (`agentview_image`)
- `observation.images.image2` → wrist camera (`robot0_eye_in_hand_image`)
+### Dataset

-⚠️ Cleaning the dataset upfront is **cleaner and more efficient** than remapping keys inside the code.
-To avoid potential mismatches and key errors, we provide a **preprocessed LIBERO dataset** that is fully compatible with the current LeRobot codebase and requires no additional manipulation:
-👉 [HuggingFaceVLA/libero](https://huggingface.co/datasets/HuggingFaceVLA/libero)
+We provide a preprocessed LIBERO dataset fully compatible with LeRobot:

-For reference, here is the **original dataset** published by Physical Intelligence:
-👉 [physical-intelligence/libero](https://huggingface.co/datasets/physical-intelligence/libero)
+- [HuggingFaceVLA/libero](https://huggingface.co/datasets/HuggingFaceVLA/libero)

---
+For reference, the original dataset published by Physical Intelligence:
+
+- [physical-intelligence/libero](https://huggingface.co/datasets/physical-intelligence/libero)

 ### Example training command

@@ -121,52 +143,39 @@ lerobot-train \
  --batch_size=4 \
  --eval.batch_size=1 \
  --eval.n_episodes=1 \
-  --eval_freq=1000 \
+  --eval_freq=1000
 ```

---
+## Reproducing published results

-### Note on rendering
+We reproduce the results of Pi0.5 on the LIBERO benchmark. We take the Physical Intelligence LIBERO base model (`pi05_libero`) and finetune for an additional 6k steps in bfloat16, with batch size of 256 on 8 H100 GPUs using the [HuggingFace LIBERO dataset](https://huggingface.co/datasets/HuggingFaceVLA/libero).

-LeRobot uses MuJoCo for simulation. You need to set the rendering backend before training or evaluation:
+The finetuned model: [lerobot/pi05_libero_finetuned](https://huggingface.co/lerobot/pi05_libero_finetuned)

- `export MUJOCO_GL=egl` → for headless servers (e.g. HPC, cloud)
-
-## Reproducing π₀.₅ results
-
-We reproduce the results of π₀.₅ on the LIBERO benchmark using the LeRobot implementation. We take the Physical Intelligence LIBERO base model (`pi05_libero`) and finetune for an additional 6k steps in bfloat16, with batch size of 256 on 8 H100 GPUs using the [HuggingFace LIBERO dataset](https://huggingface.co/datasets/HuggingFaceVLA/libero).
-
-The finetuned model can be found here:
-
- **π₀.₅ LIBERO**: [lerobot/pi05_libero_finetuned](https://huggingface.co/lerobot/pi05_libero_finetuned)
-
-We then evaluate the finetuned model using the LeRobot LIBERO implementation, by running the following command:
+### Evaluation command

 ```bash
 lerobot-eval \
-  --output_dir=/logs/ \
+  --output_dir=./eval_logs/ \
  --env.type=libero \
  --env.task=libero_spatial,libero_object,libero_goal,libero_10 \
  --eval.batch_size=1 \
  --eval.n_episodes=10 \
  --policy.path=pi05_libero_finetuned \
  --policy.n_action_steps=10 \
-  --output_dir=./eval_logs/ \
  --env.max_parallel_tasks=1
 ```

-**Note:** We set `n_action_steps=10`, similar to the original OpenPI implementation.
+We set `n_action_steps=10`, matching the original OpenPI implementation.

 ### Results

-We obtain the following results on the LIBERO benchmark:
+| Model               | LIBERO Spatial | LIBERO Object | LIBERO Goal | LIBERO 10 | Average  |
+| ------------------- | -------------- | ------------- | ----------- | --------- | -------- |
+| **Pi0.5 (LeRobot)** | 97.0           | 99.0          | 98.0        | 96.0      | **97.5** |

-| Model    | LIBERO Spatial | LIBERO Object | LIBERO Goal | LIBERO 10 | Average  |
-| -------- | -------------- | ------------- | ----------- | --------- | -------- |
-| **π₀.₅** | 97.0           | 99.0          | 98.0        | 96.0      | **97.5** |
+These results are consistent with the [original results](https://github.com/Physical-Intelligence/openpi/tree/main/examples/libero#results) reported by Physical Intelligence:

-These results are consistent with the original [results](https://github.com/Physical-Intelligence/openpi/tree/main/examples/libero#results) reported by Physical Intelligence:
-
-| Model    | LIBERO Spatial | LIBERO Object | LIBERO Goal | LIBERO 10 | Average   |
-| -------- | -------------- | ------------- | ----------- | --------- | --------- |
-| **π₀.₅** | 98.8           | 98.2          | 98.0        | 92.4      | **96.85** |
+| Model              | LIBERO Spatial | LIBERO Object | LIBERO Goal | LIBERO 10 | Average   |
+| ------------------ | -------------- | ------------- | ----------- | --------- | --------- |
+| **Pi0.5 (OpenPI)** | 98.8           | 98.2          | 98.0        | 92.4      | **96.85** |
@@ -1,32 +1,111 @@
 # Meta-World

-Meta-World is a well-designed, open-source simulation benchmark for multi-task and meta reinforcement learning in continuous-control robotic manipulation. It gives researchers a shared, realistic playground to test whether algorithms can _learn many different tasks_ and _generalize quickly to new ones_ — two central challenges for real-world robotics.
+Meta-World is an open-source simulation benchmark for **multi-task and meta reinforcement learning** in continuous-control robotic manipulation. It bundles 50 diverse manipulation tasks using everyday objects and a common tabletop Sawyer arm, providing a standardized playground to test whether algorithms can learn many different tasks and generalize quickly to new ones.

- 📄 [MetaWorld paper](https://arxiv.org/pdf/1910.10897)
- 💻 [Original MetaWorld repo](https://github.com/Farama-Foundation/Metaworld)
+- Paper: [Meta-World: A Benchmark and Evaluation for Multi-Task and Meta Reinforcement Learning](https://arxiv.org/abs/1910.10897)
+- GitHub: [Farama-Foundation/Metaworld](https://github.com/Farama-Foundation/Metaworld)
+- Project website: [metaworld.farama.org](https://metaworld.farama.org)

 ![MetaWorld MT10 demo](https://meta-world.github.io/figures/ml45.gif)

-## Why Meta-World matters
+## Available tasks

- **Diverse, realistic tasks.** Meta-World bundles a large suite of simulated manipulation tasks (50 in the MT50 suite) using everyday objects and a common tabletop Sawyer arm. This diversity exposes algorithms to a wide variety of dynamics, contacts and goal specifications while keeping a consistent control and observation structure.
- **Focus on generalization and multi-task learning.** By evaluating across task distributions that share structure but differ in goals and objects, Meta-World reveals whether an agent truly learns transferable skills rather than overfitting to a narrow task.
- **Standardized evaluation protocol.** It provides clear evaluation modes and difficulty splits, so different methods can be compared fairly across easy, medium, hard and very-hard regimes.
- **Empirical insight.** Past evaluations on Meta-World show impressive progress on some fronts, but also highlight that current multi-task and meta-RL methods still struggle with large, diverse task sets. That gap points to important research directions.
+Meta-World provides 50 tasks organized into difficulty groups. In LeRobot, you can evaluate on individual tasks, difficulty groups, or the full MT50 suite:

-## What it enables in LeRobot
+| Group      | CLI name             | Tasks | Description                                            |
+| ---------- | -------------------- | ----- | ------------------------------------------------------ |
+| Easy       | `easy`               | 28    | Tasks with simple dynamics and single-step goals       |
+| Medium     | `medium`             | 11    | Tasks requiring multi-step reasoning                   |
+| Hard       | `hard`               | 6     | Tasks with complex contacts and precise manipulation   |
+| Very Hard  | `very_hard`          | 5     | The most challenging tasks in the suite                |
+| MT50 (all) | Comma-separated list | 50    | All 50 tasks — the most challenging multi-task setting |

-In LeRobot, you can evaluate any policy or vision-language-action (VLA) model on Meta-World tasks and get a clear success-rate measure. The integration is designed to be straightforward:
+You can also pass individual task names directly (e.g., `assembly-v3`, `dial-turn-v3`).

- We provide a LeRobot-ready dataset for Meta-World (MT50) on the HF Hub: `https://huggingface.co/datasets/lerobot/metaworld_mt50`.
-  - This dataset is formatted for the MT50 evaluation that uses all 50 tasks (the most challenging multi-task setting).
-  - MT50 gives the policy a one-hot task vector and uses fixed object/goal positions for consistency.
+We provide a LeRobot-ready dataset for Meta-World MT50 on the HF Hub: [lerobot/metaworld_mt50](https://huggingface.co/datasets/lerobot/metaworld_mt50). This dataset is formatted for the MT50 evaluation that uses all 50 tasks with fixed object/goal positions and one-hot task vectors for consistency.

- Task descriptions and the exact keys required for evaluation are available in the repo/dataset — use these to ensure your policy outputs the right success signals.
+## Installation

-## Quick start, train a SmolVLA policy on Meta-World
+After following the LeRobot installation instructions:

-Example command to train a SmolVLA policy on a subset of tasks:
+```bash
+pip install -e ".[metaworld]"
+```
+
+<Tip warning={true}>
+If you encounter an `AssertionError: ['human', 'rgb_array', 'depth_array']` when running Meta-World environments, this is a mismatch between Meta-World and your Gymnasium version. Fix it with:
+
+```bash
+pip install "gymnasium==1.1.0"
+```
+
+</Tip>
+
+## Evaluation
+
+### Default evaluation (recommended)
+
+Evaluate on the medium difficulty split (a good balance of coverage and compute):
+
+```bash
+lerobot-eval \
+  --policy.path="your-policy-id" \
+  --env.type=metaworld \
+  --env.task=medium \
+  --eval.batch_size=1 \
+  --eval.n_episodes=10
+```
+
+### Single-task evaluation
+
+Evaluate on a specific task:
+
+```bash
+lerobot-eval \
+  --policy.path="your-policy-id" \
+  --env.type=metaworld \
+  --env.task=assembly-v3 \
+  --eval.batch_size=1 \
+  --eval.n_episodes=10
+```
+
+### Multi-task evaluation
+
+Evaluate across multiple tasks or difficulty groups:
+
+```bash
+lerobot-eval \
+  --policy.path="your-policy-id" \
+  --env.type=metaworld \
+  --env.task=assembly-v3,dial-turn-v3,handle-press-side-v3 \
+  --eval.batch_size=1 \
+  --eval.n_episodes=10
+```
+
+- `--env.task` accepts explicit task lists (comma-separated) or difficulty groups (e.g., `easy`, `medium`, `hard`, `very_hard`).
+- `--eval.batch_size` controls how many environments run in parallel.
+- `--eval.n_episodes` sets how many episodes to run per task.
+
+### Policy inputs and outputs
+
+**Observations:**
+
+- `observation.image` — single camera view (`corner2`), 480x480 HWC uint8
+- `observation.state` — 4-dim proprioceptive state (end-effector position + gripper)
+
+**Actions:**
+
+- Continuous control in `Box(-1, 1, shape=(4,))` — 3D end-effector delta + 1D gripper
+
+### Recommended evaluation episodes
+
+For reproducible benchmarking, use **10 episodes per task**. For the full MT50 suite this gives 500 total episodes. If you care about generalization, run on the full MT50 — it is intentionally challenging and reveals strengths/weaknesses better than a few narrow tasks.
+
+## Training
+
+### Example training command
+
+Train a SmolVLA policy on a subset of Meta-World tasks:

 ```bash
 lerobot-train \
@@ -44,37 +123,8 @@ lerobot-train \
  --eval_freq=1000
 ```

-Notes:
-
- `--env.task` accepts explicit task lists (comma separated) or difficulty groups (e.g., `env.task="hard"`).
- Adjust `batch_size`, `steps`, and `eval_freq` to match your compute budget.
- **Gymnasium Assertion Error**: if you encounter an error like
-  `AssertionError: ['human', 'rgb_array', 'depth_array']` when running MetaWorld environments, this comes from a mismatch between MetaWorld and your Gymnasium version.
-  We recommend using:
-
-```bash
-  pip install "gymnasium==1.1.0"
-```
-
-to ensure proper compatibility.
-
-## Quick start — evaluate a trained policy
-
-To evaluate a trained policy on the Meta-World medium difficulty split:
-
-```bash
-lerobot-eval \
-  --policy.path="your-policy-id" \
-  --env.type=metaworld \
-  --env.task=medium \
-  --eval.batch_size=1 \
-  --eval.n_episodes=2
-```
-
-This will run episodes and return per-task success rates using the standard Meta-World evaluation keys.
-
 ## Practical tips

- If you care about generalization, run on the full MT50 suite — it’s intentionally challenging and reveals strengths/weaknesses better than a few narrow tasks.
- Use the one-hot task conditioning for multi-task training (MT10 / MT50 conventions) so policies have explicit task context.
+- Use the one-hot task conditioning for multi-task training (MT10/MT50 conventions) so policies have explicit task context.
 - Inspect the dataset task descriptions and the `info["is_success"]` keys when writing post-processing or logging so your success metrics line up with the benchmark.
+- Adjust `batch_size`, `steps`, and `eval_freq` to match your compute budget.
@@ -331,6 +331,54 @@ lerobot-train \
  --wandb.project=multitask_dit
 ```

+## Libero Results
+
+```
+python -m lerobot.scripts.lerobot_train \
+  --dataset.repo_id=HuggingFaceVLA/libero \
+  --policy.type=multi_task_dit \
+  --policy.push_to_hub=false \
+  --output_dir="./outputs/multitask_dit_libero" \
+  --job_name="multitask-dit-libero" \
+  --wandb.enable=true \
+  --wandb.project=multitask_dit_libero \
+  --dataset.image_transforms.enable=true \
+  --dataset.image_transforms.max_num_transforms=4 \
+  --dataset.image_transforms.tfs='{"brightness":{"type":"ColorJitter","kwargs":{"brightness":[0.75,1.25]}},"contrast":{"type":"ColorJitter","kwargs":{"contrast":[0.6,1.4]}},"saturation":{"type":"ColorJitter","kwargs":{"saturation":[0.8,1.2]}},"hue":{"type":"ColorJitter","kwargs":{"hue":[-0.05,0.05]}},"sharpness":{"type":"SharpnessJitter","kwargs":{"sharpness":[0.6,1.4]}},"rotation":{"type":"RandomRotation","kwargs":{"degrees":[-5,5]}},"translation":{"type":"RandomAffine","kwargs":{"degrees":0,"translate":[0.1,0.1]}}}' \
+  --dataset.video_backend=torchcodec \
+  --policy.use_amp=true \
+  --policy.horizon=48 \
+  --policy.n_obs_steps=2 \
+  --policy.use_rope=true \
+  --policy.use_positional_encoding=false \
+  --policy.hidden_dim=768 \
+  --policy.num_layers=8 \
+  --policy.num_heads=12 \
+  --policy.dropout=0.1 \
+  --policy.timestep_embed_dim=256 \
+  --policy.objective=diffusion \
+  --policy.optimizer_lr=3e-4 \
+  --policy.optimizer_weight_decay=0 \
+  --policy.scheduler_warmup_steps=0 \
+  --policy.vision_encoder_name=openai/clip-vit-base-patch16 \
+  --policy.image_resize_shape=[256,256] \
+  --policy.image_crop_is_random=true \
+  --policy.text_encoder_name=openai/clip-vit-base-patch16 \
+  --policy.vision_encoder_lr_multiplier=0.1 \
+  --policy.device=cuda \
+  --num_workers=8 \
+  --save_freq=4000 \
+  --log_freq=100 \
+  --steps=100000 \
+  --batch_size=320
+```
+
+Results:
+
+| LIBERO Spatial | LIBERO Object | LIBERO Goal | LIBERO 10 | Average |
+| -------------- | ------------- | ----------- | --------- | ------- |
+| 87.0           | 98.2          | 93.8        | 83.2      | 90.6    |
+
 ## References

 For more details on the technical implementation and architecture, see:
@@ -0,0 +1,91 @@
+# π₀.₅ (pi05)
+
+This repository contains the Hugging Face port of **π₀.₅**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
+It is designed as a **Vision-Language-Action model with open-world generalization**.
+
+---
+
+## Model Overview
+
+| Feature              | π₀                                                     | π₀.₅                                      |
+| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
+| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
+| AdaRMS               | Not used                                               | Used in action expert                     |
+| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
+| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
+| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
+
+---
+
+## Relative Actions
+
+π₀.₅ supports training with **relative actions**, where the model learns relative offsets
+from the current robot state instead of absolute joint positions. This mirrors the
+relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
+
+### How it works
+
+1. **During preprocessing**, absolute actions are converted to relative offsets:
+   `relative = action - state` (for selected joints).
+2. The relative actions are normalized using statistics computed from the relative distribution.
+3. **During postprocessing**, predicted relative actions are converted back to absolute:
+   `absolute = relative + state`.
+
+Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
+
+### Configuration
+
+| Parameter                 | Type        | Default       | Description                                                      |
+| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
+| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
+| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
+| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
+
+### Training example
+
+```bash
+python -m lerobot.scripts.lerobot_train \
+  --policy.type=pi05 \
+  --dataset.repo_id=your_org/your_dataset \
+  --policy.use_relative_actions=true \
+  --policy.relative_exclude_joints='["gripper"]'
+```
+
+When `use_relative_actions=true`, the training script automatically:
+
+- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
+- Replaces the standard action stats with relative stats for normalization
+- Broadcasts these stats across all ranks in distributed training
+
+---
+
+## Citation
+
+If you use this work, please cite both **OpenPI** and the π₀.₅ paper:
+
+```bibtex
+@misc{openpi2024,
+  author       = {Physical Intelligence Lab},
+  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
+  year         = {2024},
+  publisher    = {GitHub},
+  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
+  license      = {Apache-2.0}
+}
+
+@misc{intelligence2025pi05visionlanguageactionmodelopenworld,
+  title        = {π₀.₅: a Vision-Language-Action Model with Open-World Generalization},
+  author       = {Physical Intelligence and Kevin Black and Noah Brown and James Darpinian and Karan Dhabalia and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Manuel Y. Galliker and Dibya Ghosh and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Devin LeBlanc and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Allen Z. Ren and Lucy Xiaoyang Shi and Laura Smith and Jost Tobias Springenberg and Kyle Stachowicz and James Tanner and Quan Vuong and Homer Walke and Anna Walling and Haohuan Wang and Lili Yu and Ury Zhilinsky},
+  year         = {2025},
+  eprint       = {2504.16054},
+  archivePrefix= {arXiv},
+  primaryClass = {cs.LG},
+  url          = {https://arxiv.org/abs/2504.16054},
+}
+```
+
+---
+
+## License
+
+This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1,108 @@
+# π₀ (pi0)
+
+This repository contains the Hugging Face port of **π₀**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
+It is designed as a **Vision-Language-Action model for general robot control**.
+
+---
+
+## Model Overview
+
+| Feature              | π₀                                                     | π₀.₅                                      |
+| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
+| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
+| AdaRMS               | Not used                                               | Used in action expert                     |
+| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
+| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
+| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
+
+---
+
+## Relative Actions
+
+π₀ supports training with **relative actions**, where the model learns relative offsets
+from the current robot state instead of absolute joint positions. This mirrors the
+relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
+
+### How it works
+
+1. **During preprocessing**, absolute actions are converted to relative offsets:
+   `relative = action - state` (for selected joints).
+2. The relative actions are normalized using statistics computed from the relative distribution.
+3. **During postprocessing**, predicted relative actions are converted back to absolute:
+   `absolute = relative + state`.
+
+Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
+
+### Configuration
+
+| Parameter                 | Type        | Default       | Description                                                      |
+| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
+| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
+| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
+| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
+
+### Training example
+
+```bash
+python -m lerobot.scripts.lerobot_train \
+  --policy.type=pi0 \
+  --dataset.repo_id=your_org/your_dataset \
+  --policy.use_relative_actions=true \
+  --policy.relative_exclude_joints='["gripper"]'
+```
+
+When `use_relative_actions=true`, the training script automatically:
+
+- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
+- Replaces the standard action stats with relative stats for normalization
+- Broadcasts these stats across all ranks in distributed training
+
+### Recomputing stats for an existing dataset
+
+If you want to precompute relative action stats offline, use `recompute_stats` from
+`lerobot.datasets.dataset_tools`:
+
+```python
+from lerobot.datasets.lerobot_dataset import LeRobotDataset
+from lerobot.datasets.dataset_tools import recompute_stats
+
+dataset = LeRobotDataset("your_org/your_dataset")
+dataset = recompute_stats(
+    dataset,
+    relative_action=True,
+    relative_exclude_joints=["gripper"],
+)
+```
+
+---
+
+## Citation
+
+If you use this work, please cite both **OpenPI** and the π₀ paper:
+
+```bibtex
+@misc{openpi2024,
+  author       = {Physical Intelligence Lab},
+  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
+  year         = {2024},
+  publisher    = {GitHub},
+  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
+  license      = {Apache-2.0}
+}
+
+@misc{black2024pi0visionlanguageactionflowmodel,
+  title        = {π₀: A Vision-Language-Action Flow Model for General Robot Control},
+  author       = {Kevin Black and Noah Brown and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Lucy Xiaoyang Shi and James Tanner and Quan Vuong and Anna Walling and Haohuan Wang and Ury Zhilinsky},
+  year         = {2024},
+  eprint       = {2410.24164},
+  archivePrefix= {arXiv},
+  primaryClass = {cs.LG},
+  url          = {https://arxiv.org/abs/2410.24164},
+}
+```
+
+---
+
+## License
+
+This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1,38 @@
+# Real-Time Chunking (RTC)
+
+This module contains the LeRobot implementation of **Real-Time Chunking (RTC)**, an inference-time technique for flow-matching based policies.
+
+**Note**: RTC is not a policy itself, but rather an inference enhancement that works with flow-matching based policies including [π₀](../pi0/), [π₀.₅](../pi05/), and [SmolVLA](../smolvla/).
+
+---
+
+## Citation
+
+If you use Real-Time Chunking in your work, please cite:
+
+```bibtex
+@misc{openpi2024,
+  author       = {Physical Intelligence Lab},
+  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
+  year         = {2024},
+  publisher    = {GitHub},
+  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
+  license      = {Apache-2.0}
+}
+
+@misc{black2025realtimeexecutionactionchunking,
+      title={Real-Time Execution of Action Chunking Flow Policies},
+      author={Kevin Black and Manuel Y. Galliker and Sergey Levine},
+      year={2025},
+      eprint={2506.07339},
+      archivePrefix={arXiv},
+      primaryClass={cs.RO},
+      url={https://arxiv.org/abs/2506.07339},
+}
+```
+
+---
+
+## License
+
+This implementation follows the **Apache 2.0 License**, consistent with the LeRobot project.
@@ -0,0 +1,14 @@
+## Paper
+
+https://arxiv.org/abs/2509.25358
+
+## Citation
+
+```bibtex
+@article{chen2025sarm,
+  title={SARM: Stage-Aware Reward Modeling for Long Horizon Robot Manipulation},
+  author={Chen, Qianzhong and Yu, Justin and Schwager, Mac and Abbeel, Pieter and Shentu, Yide and Wu, Philipp},
+  journal={arXiv preprint arXiv:2509.25358},
+  year={2025}
+}
+```
@@ -0,0 +1,680 @@
+#!/usr/bin/env python
+
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Create MP4 (or GIF) videos with sarm_progress overlay for specified episodes.
+
+Downloads datasets from HuggingFace, seeks directly into the episode segment
+of the source video, draws a progress line on each frame, and writes the result.
+
+Usage:
+    python examples/dataset/create_progress_videos.py \
+        --repo-id lerobot-data-collection/level2_final_quality3 \
+        --episode 1100
+
+    python examples/dataset/create_progress_videos.py \
+        --repo-id lerobot-data-collection/level2_final_quality3 \
+        --episode 1100 \
+        --camera-key observation.images.top \
+        --output-dir ./my_videos \
+        --gif
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import subprocess
+from pathlib import Path
+
+import cv2
+import numpy as np
+import pandas as pd
+from huggingface_hub import snapshot_download
+
+GRAPH_Y_TOP_FRAC = 0.01
+GRAPH_Y_BOT_FRAC = 0.99
+LINE_THICKNESS = 3
+SHADOW_THICKNESS = 6
+REF_ALPHA = 0.45
+FILL_ALPHA = 0.55
+SCORE_FONT_SCALE = 0.8
+TASK_FONT_SCALE = 0.55
+
+
+def download_episode_metadata(repo_id: str, episode: int) -> Path:
+    """Download only the metadata and sarm_progress files for a dataset.
+
+    Args:
+        repo_id: HuggingFace dataset repository ID.
+        episode: Episode index (used for logging only; all meta is fetched).
+
+    Returns:
+        Local cache path for the downloaded snapshot.
+    """
+    logging.info("[1/4] Downloading metadata for %s (episode %d) ...", repo_id, episode)
+    local_path = Path(
+        snapshot_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            allow_patterns=["meta/**", "sarm_progress.parquet"],
+            ignore_patterns=["*.mp4"],
+        )
+    )
+    return local_path
+
+
+def load_episode_meta(local_path: Path, episode: int, camera_key: str | None) -> dict:
+    """Read info.json and episode parquet to resolve fps, video path, and timestamps.
+
+    Args:
+        local_path: Local cache directory containing meta/.
+        episode: Episode index to look up.
+        camera_key: Camera observation key (e.g. "observation.images.base").
+            If None, the first available video key is used.
+
+    Returns:
+        Dict with keys: fps, camera, video_rel, chunk_index, file_index,
+        from_ts, to_ts, task_name.
+    """
+    info = json.loads((local_path / "meta" / "info.json").read_text())
+    fps = info["fps"]
+    features = info["features"]
+
+    video_keys = [k for k, v in features.items() if v.get("dtype") == "video"]
+    if not video_keys:
+        raise RuntimeError("No video keys found in dataset features")
+
+    if camera_key is not None:
+        if camera_key not in video_keys:
+            raise RuntimeError(f"camera_key='{camera_key}' not found. Available: {video_keys}")
+        selected_camera = camera_key
+    else:
+        selected_camera = video_keys[0]
+    logging.info("   fps=%d  camera='%s'  all_cams=%s", fps, selected_camera, video_keys)
+
+    episode_rows = []
+    for parquet_file in sorted((local_path / "meta" / "episodes").glob("**/*.parquet")):
+        episode_rows.append(pd.read_parquet(parquet_file))
+    episode_df = pd.concat(episode_rows, ignore_index=True)
+    row = episode_df[episode_df["episode_index"] == episode]
+    if row.empty:
+        raise RuntimeError(f"Episode {episode} not found in episode metadata")
+    row = row.iloc[0]
+
+    chunk_col = f"videos/{selected_camera}/chunk_index"
+    file_col = f"videos/{selected_camera}/file_index"
+    ts_from_col = f"videos/{selected_camera}/from_timestamp"
+    ts_to_col = f"videos/{selected_camera}/to_timestamp"
+
+    if chunk_col not in row.index:
+        chunk_col = f"{selected_camera}/chunk_index"
+        file_col = f"{selected_camera}/file_index"
+        ts_from_col = f"{selected_camera}/from_timestamp"
+        ts_to_col = f"{selected_camera}/to_timestamp"
+    if chunk_col not in row.index:
+        raise RuntimeError(
+            f"Cannot find video metadata columns for {selected_camera}.\nAvailable: {list(row.index)}"
+        )
+
+    chunk_index = int(row[chunk_col])
+    file_index = int(row[file_col])
+    from_timestamp = float(row[ts_from_col])
+    to_timestamp = float(row[ts_to_col])
+
+    video_template = info.get(
+        "video_path", "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4"
+    )
+    video_rel = video_template.format(
+        video_key=selected_camera,
+        chunk_index=chunk_index,
+        file_index=file_index,
+    )
+
+    task_name = _resolve_task_name(row, local_path)
+
+    return {
+        "fps": fps,
+        "camera": selected_camera,
+        "video_rel": video_rel,
+        "chunk_index": chunk_index,
+        "file_index": file_index,
+        "from_ts": from_timestamp,
+        "to_ts": to_timestamp,
+        "task_name": task_name,
+    }
+
+
+def _resolve_task_name(row: pd.Series, local_path: Path) -> str:
+    """Best-effort extraction of the task name for an episode row.
+
+    Args:
+        row: Single-episode row from the episodes parquet.
+        local_path: Dataset cache root.
+
+    Returns:
+        Task name string, or empty string if unavailable.
+    """
+    try:
+        if "tasks" in row.index and row["tasks"] is not None:
+            tasks_val = row["tasks"]
+            if isinstance(tasks_val, (list, tuple, np.ndarray)) and len(tasks_val) > 0:
+                return str(tasks_val[0])
+            return str(tasks_val).strip("[]'")
+
+        tasks_parquet = local_path / "meta" / "tasks.parquet"
+        if tasks_parquet.exists():
+            tasks_df = pd.read_parquet(tasks_parquet)
+            task_idx = int(row.get("task_index", 0)) if "task_index" in row.index else 0
+            match = tasks_df[tasks_df["task_index"] == task_idx]
+            if not match.empty:
+                return str(match.index[0])
+    except Exception as exc:
+        logging.warning("Could not load task name: %s", exc)
+    return ""
+
+
+def download_video_file(repo_id: str, local_path: Path, video_rel: str) -> Path:
+    """Download the specific video file if not already cached.
+
+    Args:
+        repo_id: HuggingFace dataset repository ID.
+        local_path: Local cache directory.
+        video_rel: Relative path to the video file within the dataset.
+
+    Returns:
+        Absolute path to the downloaded video file.
+    """
+    video_path = local_path / video_rel
+    if video_path.exists():
+        logging.info("   Video already cached: %s", video_path)
+        return video_path
+    logging.info("[2/4] Downloading video file %s ...", video_rel)
+    snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",
+        local_dir=str(local_path),
+        allow_patterns=[video_rel],
+    )
+    if not video_path.exists():
+        raise RuntimeError(f"Video not found after download: {video_path}")
+    return video_path
+
+
+def load_progress_data(local_path: Path, episode: int) -> np.ndarray | None:
+    """Load sarm_progress values for an episode.
+
+    Args:
+        local_path: Dataset cache root.
+        episode: Episode index.
+
+    Returns:
+        Sorted (N, 2) array of (frame_index, progress), or None if unavailable.
+    """
+    parquet_path = local_path / "sarm_progress.parquet"
+    if not parquet_path.exists():
+        logging.warning("sarm_progress.parquet not found")
+        return None
+    df = pd.read_parquet(parquet_path)
+    logging.info("   sarm_progress.parquet columns: %s", list(df.columns))
+    episode_df = df[df["episode_index"] == episode].copy()
+    if episode_df.empty:
+        logging.warning("No sarm_progress rows for episode %d", episode)
+        return None
+    episode_df = episode_df.sort_values("frame_index")
+
+    if "progress_dense" in episode_df.columns and episode_df["progress_dense"].notna().any():
+        progress_column = "progress_dense"
+    elif "progress_sparse" in episode_df.columns:
+        progress_column = "progress_sparse"
+    else:
+        progress_columns = [c for c in episode_df.columns if "progress" in c.lower()]
+        if not progress_columns:
+            return None
+        progress_column = progress_columns[0]
+
+    logging.info("   Using progress column: '%s'", progress_column)
+    return episode_df[["frame_index", progress_column]].rename(columns={progress_column: "progress"}).values
+
+
+def _precompute_pixel_coords(
+    progress_data: np.ndarray,
+    num_frames: int,
+    frame_width: int,
+    frame_height: int,
+) -> np.ndarray:
+    """Map progress samples to pixel coordinates for overlay drawing.
+
+    Args:
+        progress_data: (N, 2) array of (frame_index, progress).
+        num_frames: Total number of video frames.
+        frame_width: Video width in pixels.
+        frame_height: Video height in pixels.
+
+    Returns:
+        (N, 2) array of (x, y) pixel coordinates.
+    """
+    frame_indices = progress_data[:, 0].astype(float)
+    progress_values = np.clip(progress_data[:, 1].astype(float), 0.0, 1.0)
+
+    y_top = int(frame_height * GRAPH_Y_TOP_FRAC)
+    y_bot = int(frame_height * GRAPH_Y_BOT_FRAC)
+    graph_height = y_bot - y_top
+
+    x_coords = (frame_indices / (num_frames - 1) * (frame_width - 1)).astype(int)
+    y_coords = (y_bot - progress_values * graph_height).astype(int)
+
+    return np.stack([x_coords, y_coords], axis=1)
+
+
+def _progress_color(normalized_position: float) -> tuple[int, int, int]:
+    """Interpolate BGR color from red to green based on position in [0, 1].
+
+    Args:
+        normalized_position: Value in [0, 1] indicating how far along the episode.
+
+    Returns:
+        BGR color tuple.
+    """
+    red = int(255 * (1.0 - normalized_position))
+    green = int(255 * normalized_position)
+    return (0, green, red)
+
+
+def _prerender_fill_polygon(
+    pixel_coords: np.ndarray,
+    frame_width: int,
+    frame_height: int,
+) -> np.ndarray:
+    """Pre-render the grey fill polygon under the progress curve as a BGRA image.
+
+    Args:
+        pixel_coords: (N, 2) array of (x, y) pixel coordinates.
+        frame_width: Video width in pixels.
+        frame_height: Video height in pixels.
+
+    Returns:
+        BGRA image array of shape (frame_height, frame_width, 4).
+    """
+    y_bot = int(frame_height * GRAPH_Y_BOT_FRAC)
+    fill_image = np.zeros((frame_height, frame_width, 4), dtype=np.uint8)
+    polygon = np.concatenate(
+        [
+            pixel_coords,
+            [[pixel_coords[-1][0], y_bot], [pixel_coords[0][0], y_bot]],
+        ],
+        axis=0,
+    ).astype(np.int32)
+    cv2.fillPoly(fill_image, [polygon], color=(128, 128, 128, int(255 * FILL_ALPHA)))
+    return fill_image
+
+
+def _alpha_composite_region(base: np.ndarray, overlay_bgra: np.ndarray, x_limit: int) -> None:
+    """Blend BGRA overlay onto BGR base in-place, up to x_limit columns.
+
+    Args:
+        base: BGR frame to draw on (modified in-place).
+        overlay_bgra: BGRA overlay image.
+        x_limit: Only blend columns [0, x_limit).
+    """
+    if x_limit <= 0:
+        return
+    region_base = base[:, :x_limit]
+    region_overlay = overlay_bgra[:, :x_limit]
+    alpha = region_overlay[:, :, 3:4].astype(np.float32) / 255.0
+    region_base[:] = np.clip(
+        region_overlay[:, :, :3].astype(np.float32) * alpha + region_base.astype(np.float32) * (1.0 - alpha),
+        0,
+        255,
+    ).astype(np.uint8)
+
+
+def _draw_text_outlined(
+    frame: np.ndarray,
+    text: str,
+    position: tuple[int, int],
+    font_scale: float,
+    thickness: int = 1,
+) -> None:
+    """Draw white text with a dark outline for readability on any background.
+
+    Args:
+        frame: BGR image to draw on (modified in-place).
+        text: String to render.
+        position: (x, y) bottom-left corner of the text.
+        font_scale: OpenCV font scale.
+        thickness: Text stroke thickness.
+    """
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    cv2.putText(frame, text, position, font, font_scale, (0, 0, 0), thickness + 2, cv2.LINE_AA)
+    cv2.putText(frame, text, position, font, font_scale, (255, 255, 255), thickness, cv2.LINE_AA)
+
+
+def composite_progress_video(
+    video_path: Path,
+    from_timestamp: float,
+    to_timestamp: float,
+    progress_data: np.ndarray,
+    output_path: Path,
+    fps: float,
+    task_name: str = "",
+) -> Path:
+    """Read episode frames by seeking into the source video, draw progress overlay, write output.
+
+    Uses cv2.CAP_PROP_POS_MSEC to seek directly into the source video,
+    eliminating the need for an intermediate clip file.
+
+    Args:
+        video_path: Path to the full source video file.
+        from_timestamp: Start timestamp of the episode in seconds.
+        to_timestamp: End timestamp of the episode in seconds.
+        progress_data: (N, 2) array of (frame_index, progress).
+        output_path: Path to write the output MP4.
+        fps: Frames per second for the output video.
+        task_name: Optional task name to display at the top of the video.
+
+    Returns:
+        Path to the written output file (MP4).
+    """
+    capture = cv2.VideoCapture(str(video_path))
+    try:
+        capture.set(cv2.CAP_PROP_POS_MSEC, from_timestamp * 1000)
+
+        frame_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        frame_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        duration_seconds = to_timestamp - from_timestamp
+        num_frames = int(round(duration_seconds * fps))
+
+        logging.info(
+            "   Video: %dx%d, %d frames @ %.1f fps (%.2fs)",
+            frame_width,
+            frame_height,
+            num_frames,
+            fps,
+            duration_seconds,
+        )
+
+        pixel_coords = _precompute_pixel_coords(progress_data, num_frames, frame_width, frame_height)
+        y_ref = int(frame_height * GRAPH_Y_TOP_FRAC)
+
+        fill_image = _prerender_fill_polygon(pixel_coords, frame_width, frame_height)
+
+        ref_line_image = np.zeros((frame_height, frame_width, 4), dtype=np.uint8)
+        cv2.line(
+            ref_line_image,
+            (0, y_ref),
+            (frame_width - 1, y_ref),
+            (200, 200, 200, int(255 * REF_ALPHA)),
+            1,
+            cv2.LINE_AA,
+        )
+
+        frame_indices = progress_data[:, 0].astype(int)
+        progress_values = progress_data[:, 1].astype(float)
+
+        logging.info("[3/4] Compositing %d frames ...", num_frames)
+        fourcc = cv2.VideoWriter_fourcc(*"mp4v")
+        writer = cv2.VideoWriter(str(output_path), fourcc, fps, (frame_width, frame_height))
+
+        for frame_idx in range(num_frames):
+            ret, frame = capture.read()
+            if not ret:
+                break
+
+            drawn_count = int(np.searchsorted(frame_indices, frame_idx, side="right"))
+            x_current = (
+                int(pixel_coords[min(drawn_count, len(pixel_coords)) - 1][0]) + 1 if drawn_count > 0 else 0
+            )
+
+            _alpha_composite_region(frame, ref_line_image, frame_width)
+            _alpha_composite_region(frame, fill_image, x_current)
+
+            if drawn_count >= 2:
+                time_position = (drawn_count - 1) / max(len(progress_values) - 1, 1)
+                line_color = _progress_color(time_position)
+                points = pixel_coords[:drawn_count].reshape(-1, 1, 2).astype(np.int32)
+                cv2.polylines(
+                    frame,
+                    [points],
+                    isClosed=False,
+                    color=(255, 255, 255),
+                    thickness=SHADOW_THICKNESS,
+                    lineType=cv2.LINE_AA,
+                )
+                cv2.polylines(
+                    frame,
+                    [points],
+                    isClosed=False,
+                    color=line_color,
+                    thickness=LINE_THICKNESS,
+                    lineType=cv2.LINE_AA,
+                )
+
+            if drawn_count > 0:
+                score = float(progress_values[min(drawn_count, len(progress_values)) - 1])
+                score_text = f"{score:.2f}"
+                (text_width, _), _ = cv2.getTextSize(
+                    score_text, cv2.FONT_HERSHEY_SIMPLEX, SCORE_FONT_SCALE, 2
+                )
+                score_x = frame_width - text_width - 12
+                score_y = frame_height - 12
+                time_position = (drawn_count - 1) / max(len(progress_values) - 1, 1)
+                score_color = _progress_color(time_position)
+                cv2.putText(
+                    frame,
+                    score_text,
+                    (score_x, score_y),
+                    cv2.FONT_HERSHEY_SIMPLEX,
+                    SCORE_FONT_SCALE,
+                    (0, 0, 0),
+                    4,
+                    cv2.LINE_AA,
+                )
+                cv2.putText(
+                    frame,
+                    score_text,
+                    (score_x, score_y),
+                    cv2.FONT_HERSHEY_SIMPLEX,
+                    SCORE_FONT_SCALE,
+                    score_color,
+                    2,
+                    cv2.LINE_AA,
+                )
+
+            if task_name:
+                (text_width, _), _ = cv2.getTextSize(task_name, cv2.FONT_HERSHEY_SIMPLEX, TASK_FONT_SCALE, 1)
+                task_x = max((frame_width - text_width) // 2, 4)
+                _draw_text_outlined(frame, task_name, (task_x, 22), TASK_FONT_SCALE)
+
+            writer.write(frame)
+            if frame_idx % 100 == 0:
+                logging.info("   Frame %d/%d ...", frame_idx, num_frames)
+
+        writer.release()
+    finally:
+        capture.release()
+
+    logging.info("   MP4 written: %s", output_path)
+    return output_path
+
+
+def convert_mp4_to_gif(mp4_path: Path) -> Path:
+    """Convert an MP4 to an optimized GIF using ffmpeg palette generation.
+
+    Args:
+        mp4_path: Path to the source MP4 file.
+
+    Returns:
+        Path to the generated GIF file.
+    """
+    capture = cv2.VideoCapture(str(mp4_path))
+    frame_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+    capture.release()
+
+    gif_path = mp4_path.with_suffix(".gif")
+    palette_path = mp4_path.parent / "_palette.png"
+
+    logging.info("[4/4] Converting to GIF ...")
+    result_palette = subprocess.run(  # nosec B607
+        [
+            "ffmpeg",
+            "-y",
+            "-i",
+            str(mp4_path),
+            "-vf",
+            f"fps=10,scale={frame_width}:-1:flags=lanczos,palettegen=max_colors=128:stats_mode=diff",
+            "-update",
+            "1",
+            str(palette_path),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if result_palette.returncode != 0:
+        logging.warning("palettegen failed:\n%s", result_palette.stderr[-500:])
+
+    result_gif = subprocess.run(  # nosec B607
+        [
+            "ffmpeg",
+            "-y",
+            "-i",
+            str(mp4_path),
+            "-i",
+            str(palette_path),
+            "-filter_complex",
+            f"fps=10,scale={frame_width}:-1:flags=lanczos[v];[v][1:v]paletteuse=dither=bayer:bayer_scale=3",
+            str(gif_path),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if result_gif.returncode != 0:
+        logging.warning("GIF encode failed:\n%s", result_gif.stderr[-500:])
+
+    palette_path.unlink(missing_ok=True)
+    logging.info("   GIF written: %s", gif_path)
+    return gif_path
+
+
+def process_dataset(
+    repo_id: str,
+    episode: int,
+    camera_key: str | None,
+    output_dir: Path,
+    create_gif: bool = False,
+) -> Path | None:
+    """Full pipeline: download, extract metadata, composite progress, write output.
+
+    Args:
+        repo_id: HuggingFace dataset repository ID.
+        episode: Episode index.
+        camera_key: Camera key to use, or None for auto-selection.
+        output_dir: Directory to write output files.
+        create_gif: If True, also generate a GIF from the MP4.
+
+    Returns:
+        Path to the final output file, or None on failure.
+    """
+    safe_name = repo_id.replace("/", "_")
+    logging.info("Processing: %s  |  episode %d", repo_id, episode)
+
+    local_path = download_episode_metadata(repo_id, episode)
+    logging.info("   Local cache: %s", local_path)
+
+    episode_meta = load_episode_meta(local_path, episode, camera_key)
+    logging.info("   Episode meta: %s", episode_meta)
+
+    video_path = download_video_file(repo_id, local_path, episode_meta["video_rel"])
+
+    progress_data = load_progress_data(local_path, episode)
+    if progress_data is None:
+        logging.error("Could not load sarm_progress data. Skipping overlay.")
+        return None
+
+    logging.info("   Progress frames: %d", len(progress_data))
+
+    output_path = output_dir / f"{safe_name}_ep{episode}_progress.mp4"
+    final_path = composite_progress_video(
+        video_path=video_path,
+        from_timestamp=episode_meta["from_ts"],
+        to_timestamp=episode_meta["to_ts"],
+        progress_data=progress_data,
+        output_path=output_path,
+        fps=episode_meta["fps"],
+        task_name=episode_meta.get("task_name", ""),
+    )
+
+    if create_gif:
+        final_path = convert_mp4_to_gif(final_path)
+
+    logging.info("Done: %s", final_path)
+    return final_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Create MP4/GIF videos with sarm_progress overlay for dataset episodes."
+    )
+    parser.add_argument(
+        "--repo-id",
+        type=str,
+        required=True,
+        help="HuggingFace dataset repository ID (e.g. 'lerobot-data-collection/level2_final_quality3').",
+    )
+    parser.add_argument(
+        "--episode",
+        type=int,
+        required=True,
+        help="Episode index to visualize.",
+    )
+    parser.add_argument(
+        "--camera-key",
+        type=str,
+        default=None,
+        help="Camera observation key (e.g. 'observation.images.base'). Auto-selects first camera if omitted.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path("progress_videos"),
+        help="Directory to write output files (default: ./progress_videos).",
+    )
+    parser.add_argument(
+        "--gif",
+        action="store_true",
+        help="Also generate a GIF from the MP4 output.",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    result = process_dataset(
+        repo_id=args.repo_id,
+        episode=args.episode,
+        camera_key=args.camera_key,
+        output_dir=args.output_dir,
+        create_gif=args.gif,
+    )
+
+    if result:
+        logging.info("Output: %s", result)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,228 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared utilities for Human-in-the-Loop data collection scripts."""
+
+import logging
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from lerobot.processor import (
+    IdentityProcessorStep,
+    RobotAction,
+    RobotObservation,
+    RobotProcessorPipeline,
+)
+from lerobot.processor.converters import (
+    observation_to_transition,
+    robot_action_observation_to_transition,
+    transition_to_observation,
+    transition_to_robot_action,
+)
+from lerobot.robots import Robot
+from lerobot.teleoperators import Teleoperator
+from lerobot.utils.control_utils import is_headless
+from lerobot.utils.robot_utils import precise_sleep
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class HILDatasetConfig:
+    repo_id: str
+    single_task: str
+    root: str | Path | None = None
+    fps: int = 30
+    episode_time_s: float = 120
+    num_episodes: int = 50
+    video: bool = True
+    push_to_hub: bool = True
+    private: bool = False
+    tags: list[str] | None = None
+    num_image_writer_processes: int = 0
+    num_image_writer_threads_per_camera: int = 4
+    video_encoding_batch_size: int = 1
+    vcodec: str = "auto"
+    streaming_encoding: bool = True
+    encoder_queue_maxsize: int = 30
+    encoder_threads: int | None = None
+    rename_map: dict[str, str] = field(default_factory=dict)
+
+
+def teleop_has_motor_control(teleop: Teleoperator) -> bool:
+    """Check if teleoperator has motor control capabilities."""
+    return all(hasattr(teleop, attr) for attr in ("enable_torque", "disable_torque", "write_goal_positions"))
+
+
+def teleop_disable_torque(teleop: Teleoperator) -> None:
+    """Disable teleop torque if supported."""
+    if hasattr(teleop, "disable_torque"):
+        teleop.disable_torque()
+
+
+def teleop_enable_torque(teleop: Teleoperator) -> None:
+    """Enable teleop torque if supported."""
+    if hasattr(teleop, "enable_torque"):
+        teleop.enable_torque()
+
+
+def teleop_smooth_move_to(teleop: Teleoperator, target_pos: dict, duration_s: float = 2.0, fps: int = 50):
+    """Smoothly move teleop to target position if motor control is available."""
+    if not teleop_has_motor_control(teleop):
+        logger.warning("Teleop does not support motor control - cannot mirror robot position")
+        return
+
+    teleop_enable_torque(teleop)
+    current = teleop.get_action()
+    steps = max(int(duration_s * fps), 1)
+
+    for step in range(steps + 1):
+        t = step / steps
+        interp = {}
+        for k in current:
+            if k in target_pos:
+                interp[k] = current[k] * (1 - t) + target_pos[k] * t
+            else:
+                interp[k] = current[k]
+        teleop.write_goal_positions(interp)
+        time.sleep(1 / fps)
+
+
+def init_keyboard_listener():
+    """Initialize keyboard listener with HIL controls."""
+    events = {
+        "exit_early": False,
+        "rerecord_episode": False,
+        "stop_recording": False,
+        "policy_paused": False,
+        "correction_active": False,
+        "resume_policy": False,
+        "in_reset": False,
+        "start_next_episode": False,
+    }
+
+    if is_headless():
+        logger.warning("Headless environment - keyboard controls unavailable")
+        return None, events
+
+    from pynput import keyboard
+
+    def on_press(key):
+        try:
+            if events["in_reset"]:
+                if key in [keyboard.Key.space, keyboard.Key.right]:
+                    logger.info("[HIL] Starting next episode...")
+                    events["start_next_episode"] = True
+                elif hasattr(key, "char") and key.char == "c":
+                    events["start_next_episode"] = True
+                elif key == keyboard.Key.esc:
+                    logger.info("[HIL] ESC - Stop recording, pushing to hub...")
+                    events["stop_recording"] = True
+                    events["start_next_episode"] = True
+            else:
+                if key == keyboard.Key.space:
+                    if not events["policy_paused"] and not events["correction_active"]:
+                        logger.info("[HIL] PAUSED - Press 'c' to take control or 'p' to resume policy")
+                        events["policy_paused"] = True
+                elif hasattr(key, "char") and key.char == "c":
+                    if events["policy_paused"] and not events["correction_active"]:
+                        logger.info("[HIL] Taking control...")
+                        events["start_next_episode"] = True
+                elif hasattr(key, "char") and key.char == "p":
+                    if events["policy_paused"] or events["correction_active"]:
+                        logger.info("[HIL] Resuming policy...")
+                        events["resume_policy"] = True
+                elif key == keyboard.Key.right:
+                    logger.info("[HIL] End episode")
+                    events["exit_early"] = True
+                elif key == keyboard.Key.left:
+                    logger.info("[HIL] Re-record episode")
+                    events["rerecord_episode"] = True
+                    events["exit_early"] = True
+                elif key == keyboard.Key.esc:
+                    logger.info("[HIL] ESC - Stop recording...")
+                    events["stop_recording"] = True
+                    events["exit_early"] = True
+        except Exception as e:
+            logger.info(f"Key error: {e}")
+
+    listener = keyboard.Listener(on_press=on_press)
+    listener.start()
+    return listener, events
+
+
+def make_identity_processors():
+    """Create identity processors for recording."""
+    teleop_proc = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction](
+        steps=[IdentityProcessorStep()],
+        to_transition=robot_action_observation_to_transition,
+        to_output=transition_to_robot_action,
+    )
+    obs_proc = RobotProcessorPipeline[RobotObservation, RobotObservation](
+        steps=[IdentityProcessorStep()],
+        to_transition=observation_to_transition,
+        to_output=transition_to_observation,
+    )
+    return teleop_proc, obs_proc
+
+
+def reset_loop(robot: Robot, teleop: Teleoperator, events: dict, fps: int):
+    """Reset period where human repositions environment."""
+    logger.info("[HIL] RESET")
+
+    events["in_reset"] = True
+    events["start_next_episode"] = False
+
+    obs = robot.get_observation()
+    robot_pos = {k: v for k, v in obs.items() if k.endswith(".pos") and k in robot.observation_features}
+    teleop_smooth_move_to(teleop, robot_pos, duration_s=2.0, fps=50)
+
+    logger.info("Press any key to enable teleoperation")
+    while not events["start_next_episode"] and not events["stop_recording"]:
+        precise_sleep(0.05)
+
+    if events["stop_recording"]:
+        return
+
+    events["start_next_episode"] = False
+    teleop_disable_torque(teleop)
+    logger.info("Teleop enabled - press any key to start episode")
+
+    while not events["start_next_episode"] and not events["stop_recording"]:
+        loop_start = time.perf_counter()
+        action = teleop.get_action()
+        robot.send_action(action)
+        precise_sleep(1 / fps - (time.perf_counter() - loop_start))
+
+    events["in_reset"] = False
+    events["start_next_episode"] = False
+    events["exit_early"] = False
+    events["policy_paused"] = False
+    events["correction_active"] = False
+    events["resume_policy"] = False
+
+
+def print_controls(rtc: bool = False):
+    """Print control instructions."""
+    mode = "Human-in-the-Loop Data Collection" + (" (RTC)" if rtc else "")
+    logger.info(
+        "%s\n  Controls:\n"
+        "    SPACE  - Pause policy\n"
+        "    c      - Take control\n"
+        "    p      - Resume policy after pause/correction\n"
+        "    →      - End episode\n"
+        "    ESC    - Stop and push to hub",
+        mode,
+    )
@@ -69,15 +69,20 @@ Usage:
        --policy.path=lerobot-data-collection/folding_final \
        --robot.type=bi_openarm_follower \
        --robot.cameras='{left_wrist: {type: opencv, index_or_path: "/dev/video4", width: 1280, height: 720, fps: 30}, base: {type: opencv, index_or_path: "/dev/video2", width: 640, height: 480, fps: 30}, right_wrist: {type: opencv, index_or_path: "/dev/video0", width: 1280, height: 720, fps: 30}}' \
-        --robot.left_arm_config.port=can1 \
+        --robot.left_arm_config.port=can0 \
        --robot.left_arm_config.side=left \
        --robot.left_arm_config.can_interface=socketcan \
-        --robot.right_arm_config.port=can0 \
+        --robot.left_arm_config.disable_torque_on_disconnect=true \
+        --robot.left_arm_config.max_relative_target=8.0 \
+        --robot.right_arm_config.port=can1 \
        --robot.right_arm_config.side=right \
        --robot.right_arm_config.can_interface=socketcan \
+        --robot.right_arm_config.disable_torque_on_disconnect=true \
+        --robot.right_arm_config.max_relative_target=8.0 \
        --task="Fold the T-shirt properly" \
        --fps=30 \
        --duration=2000 \
+        --interpolation_multiplier=3 \
        --rtc.enabled=true \
        --rtc.execution_horizon=20 \
        --rtc.max_guidance_weight=5.0 \
@@ -104,9 +109,7 @@ from lerobot.configs.policies import PreTrainedConfig
 from lerobot.configs.types import RTCAttentionSchedule
 from lerobot.datasets.feature_utils import build_dataset_frame, hw_to_dataset_features
 from lerobot.policies.factory import get_policy_class, make_pre_post_processors
-from lerobot.policies.rtc.action_queue import ActionQueue
-from lerobot.policies.rtc.configuration_rtc import RTCConfig
-from lerobot.policies.rtc.latency_tracker import LatencyTracker
+from lerobot.policies.rtc import ActionInterpolator, ActionQueue, LatencyTracker, RTCConfig
 from lerobot.processor import (
    NormalizerProcessorStep,
    RelativeActionsProcessorStep,
@@ -181,6 +184,7 @@ class RTCDemoConfig(HubMixin):
    # Demo parameters
    duration: float = 30.0  # Duration to run the demo (seconds)
    fps: float = 10.0  # Action execution frequency (Hz)
+    interpolation_multiplier: int = 1  # Control rate multiplier (1=off, 2=2x, 3=3x)

    # Compute device
    device: str | None = None  # Device to run on (cuda, cpu, auto)
@@ -461,20 +465,23 @@ def actor_control(
        action_keys = [k for k in robot.action_features() if k.endswith(".pos")]

        action_count = 0
-        action_interval = 1.0 / cfg.fps
+        interpolator = ActionInterpolator(multiplier=cfg.interpolation_multiplier)
+        action_interval = interpolator.get_control_interval(cfg.fps)

        while not shutdown_event.is_set():
            start_time = time.perf_counter()

-            # Try to get an action from the queue with timeout
-            action = action_queue.get()
+            if interpolator.needs_new_action():
+                new_action = action_queue.get()
+                if new_action is not None:
+                    interpolator.add(new_action.cpu())

+            action = interpolator.get()
            if action is not None:
                action = action.cpu()
                action_dict = {key: action[i].item() for i, key in enumerate(action_keys)}
                action_processed = robot_action_processor((action_dict, None))
                robot.send_action(action_processed)
-
                action_count += 1

            dt_s = time.perf_counter() - start_time
@@ -71,9 +71,9 @@ dependencies = [
    "cmake>=3.29.0.1,<4.2.0",
    "packaging>=24.2,<26.0",

-    "torch>=2.2.1,<2.11.0",
-    "torchcodec>=0.2.1,<0.11.0; sys_platform != 'win32' and (sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')) and (sys_platform != 'darwin' or platform_machine != 'x86_64')",
-    "torchvision>=0.21.0,<0.26.0",
+    "torch>=2.7,<2.11.0",
+    "torchcodec>=0.3.0,<0.11.0; sys_platform != 'win32' and (sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')) and (sys_platform != 'darwin' or platform_machine != 'x86_64')", # NOTE: Windows support starts at version 0.7 (needs torch==2.8), ffmpeg>=8 support starts at version 0.8.1 (needs torch==2.9), system-wide ffmpeg support starts at version 0.10 (needs torch==2.10).
+    "torchvision>=0.22.0,<0.26.0",

    "einops>=0.8.0,<0.9.0",
    "opencv-python-headless>=4.9.0,<4.14.0",
@@ -1,37 +0,0 @@
-# Multitask DiT Policy
-
-## Citation
-
-If you use this work, please cite the following works:
-
-```bibtex
-@misc{jones2025multitaskditpolicy,
-  author = {Bryson Jones},
-  title = {Dissecting and Open-Sourcing Multitask Diffusion Transformer Policy},
-  year = {2025},
-  url = {https://brysonkjones.substack.com/p/dissecting-and-open-sourcing-multitask-diffusion-transformer-policy},
-  note = {Blog post}
-}
-```
-
-```bibtex
-@misc{trilbmteam2025carefulexaminationlargebehaviormodels,
-  author       = {TRI LBM Team},
-  title        = {A Careful Examination of Large Behavior Models for Multitask Dexterous Manipulation},
-  year         = {2025},
-  eprint       = {arXiv:2507.05331},
-  archivePrefix = {arXiv},
-  primaryClass = {cs.RO},
-  url          = {https://arxiv.org/abs/2507.05331}
-}
-```
-
-```bibtex
-@misc{bostondynamics2025largebehaviormodelsatlas,
-  author       = {Boston Dynamics and TRI Research Team},
-  title        = {Large Behavior Models and Atlas Find New Footing},
-  year         = {2025},
-  url          = {https://bostondynamics.com/blog/large-behavior-models-atlas-find-new-footing/},
-  note         = {Blog post}
-}
-```
@@ -0,0 +1 @@
+../../../../docs/source/policy_multi_task_dit_README.md
@@ -1,108 +0,0 @@
-# π₀ (pi0)
-
-This repository contains the Hugging Face port of **π₀**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
-It is designed as a **Vision-Language-Action model for general robot control**.
-
---
-
-## Model Overview
-
-| Feature              | π₀                                                     | π₀.₅                                      |
-| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
-| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
-| AdaRMS               | Not used                                               | Used in action expert                     |
-| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
-| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
-| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
-
---
-
-## Relative Actions
-
-π₀ supports training with **relative actions**, where the model learns relative offsets
-from the current robot state instead of absolute joint positions. This mirrors the
-relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
-
-### How it works
-
-1. **During preprocessing**, absolute actions are converted to relative offsets:
-   `relative = action - state` (for selected joints).
-2. The relative actions are normalized using statistics computed from the relative distribution.
-3. **During postprocessing**, predicted relative actions are converted back to absolute:
-   `absolute = relative + state`.
-
-Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
-
-### Configuration
-
-| Parameter                 | Type        | Default       | Description                                                      |
-| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
-| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
-| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
-| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
-
-### Training example
-
-```bash
-python -m lerobot.scripts.lerobot_train \
-  --policy.type=pi0 \
-  --dataset.repo_id=your_org/your_dataset \
-  --policy.use_relative_actions=true \
-  --policy.relative_exclude_joints='["gripper"]'
-```
-
-When `use_relative_actions=true`, the training script automatically:
-
- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
- Replaces the standard action stats with relative stats for normalization
- Broadcasts these stats across all ranks in distributed training
-
-### Recomputing stats for an existing dataset
-
-If you want to precompute relative action stats offline, use `recompute_stats` from
-`lerobot.datasets.dataset_tools`:
-
-```python
-from lerobot.datasets.lerobot_dataset import LeRobotDataset
-from lerobot.datasets.dataset_tools import recompute_stats
-
-dataset = LeRobotDataset("your_org/your_dataset")
-dataset = recompute_stats(
-    dataset,
-    relative_action=True,
-    relative_exclude_joints=["gripper"],
-)
-```
-
---
-
-## Citation
-
-If you use this work, please cite both **OpenPI** and the π₀ paper:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{black2024pi0visionlanguageactionflowmodel,
-  title        = {π₀: A Vision-Language-Action Flow Model for General Robot Control},
-  author       = {Kevin Black and Noah Brown and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Lucy Xiaoyang Shi and James Tanner and Quan Vuong and Anna Walling and Haohuan Wang and Ury Zhilinsky},
-  year         = {2024},
-  eprint       = {2410.24164},
-  archivePrefix= {arXiv},
-  primaryClass = {cs.LG},
-  url          = {https://arxiv.org/abs/2410.24164},
-}
-```
-
---
-
-## License
-
-This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1 @@
+../../../../docs/source/policy_pi0_README.md
@@ -1,91 +0,0 @@
-# π₀.₅ (pi05)
-
-This repository contains the Hugging Face port of **π₀.₅**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
-It is designed as a **Vision-Language-Action model with open-world generalization**.
-
---
-
-## Model Overview
-
-| Feature              | π₀                                                     | π₀.₅                                      |
-| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
-| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
-| AdaRMS               | Not used                                               | Used in action expert                     |
-| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
-| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
-| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
-
---
-
-## Relative Actions
-
-π₀.₅ supports training with **relative actions**, where the model learns relative offsets
-from the current robot state instead of absolute joint positions. This mirrors the
-relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
-
-### How it works
-
-1. **During preprocessing**, absolute actions are converted to relative offsets:
-   `relative = action - state` (for selected joints).
-2. The relative actions are normalized using statistics computed from the relative distribution.
-3. **During postprocessing**, predicted relative actions are converted back to absolute:
-   `absolute = relative + state`.
-
-Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
-
-### Configuration
-
-| Parameter                 | Type        | Default       | Description                                                      |
-| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
-| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
-| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
-| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
-
-### Training example
-
-```bash
-python -m lerobot.scripts.lerobot_train \
-  --policy.type=pi05 \
-  --dataset.repo_id=your_org/your_dataset \
-  --policy.use_relative_actions=true \
-  --policy.relative_exclude_joints='["gripper"]'
-```
-
-When `use_relative_actions=true`, the training script automatically:
-
- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
- Replaces the standard action stats with relative stats for normalization
- Broadcasts these stats across all ranks in distributed training
-
---
-
-## Citation
-
-If you use this work, please cite both **OpenPI** and the π₀.₅ paper:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{intelligence2025pi05visionlanguageactionmodelopenworld,
-  title        = {π₀.₅: a Vision-Language-Action Model with Open-World Generalization},
-  author       = {Physical Intelligence and Kevin Black and Noah Brown and James Darpinian and Karan Dhabalia and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Manuel Y. Galliker and Dibya Ghosh and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Devin LeBlanc and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Allen Z. Ren and Lucy Xiaoyang Shi and Laura Smith and Jost Tobias Springenberg and Kyle Stachowicz and James Tanner and Quan Vuong and Homer Walke and Anna Walling and Haohuan Wang and Lili Yu and Ury Zhilinsky},
-  year         = {2025},
-  eprint       = {2504.16054},
-  archivePrefix= {arXiv},
-  primaryClass = {cs.LG},
-  url          = {https://arxiv.org/abs/2504.16054},
-}
-```
-
---
-
-## License
-
-This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1 @@
+../../../../docs/source/policy_pi05_README.md
@@ -1,38 +0,0 @@
-# Real-Time Chunking (RTC)
-
-This module contains the LeRobot implementation of **Real-Time Chunking (RTC)**, an inference-time technique for flow-matching based policies.
-
-**Note**: RTC is not a policy itself, but rather an inference enhancement that works with flow-matching based policies including [π₀](../pi0/), [π₀.₅](../pi05/), and [SmolVLA](../smolvla/).
-
---
-
-## Citation
-
-If you use Real-Time Chunking in your work, please cite:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{black2025realtimeexecutionactionchunking,
-      title={Real-Time Execution of Action Chunking Flow Policies},
-      author={Kevin Black and Manuel Y. Galliker and Sergey Levine},
-      year={2025},
-      eprint={2506.07339},
-      archivePrefix={arXiv},
-      primaryClass={cs.RO},
-      url={https://arxiv.org/abs/2506.07339},
-}
-```
-
---
-
-## License
-
-This implementation follows the **Apache 2.0 License**, consistent with the LeRobot project.
@@ -0,0 +1 @@
+../../../../docs/source/policy_rtc_README.md
@@ -0,0 +1,29 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Real-Time Chunking (RTC) utilities for action-chunking policies."""
+
+from lerobot.policies.rtc.action_interpolator import ActionInterpolator
+from lerobot.policies.rtc.action_queue import ActionQueue
+from lerobot.policies.rtc.configuration_rtc import RTCConfig
+from lerobot.policies.rtc.latency_tracker import LatencyTracker
+from lerobot.policies.rtc.modeling_rtc import RTCProcessor
+
+__all__ = [
+    "ActionInterpolator",
+    "ActionQueue",
+    "LatencyTracker",
+    "RTCConfig",
+    "RTCProcessor",
+]
@@ -0,0 +1,116 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Action interpolation for smoother robot control.
+
+Provides configurable Nx control rate by interpolating between consecutive actions.
+Useful with RTC and action-chunking policies to reduce jerkiness.
+"""
+
+from torch import Tensor
+
+
+class ActionInterpolator:
+    """Interpolates between consecutive actions for smoother control.
+
+    When enabled with multiplier N, produces N actions per policy action
+    by linearly interpolating between the previous and current action.
+
+    Example with multiplier=3:
+        prev_action -> [1/3 interpolated, 2/3 interpolated, current_action]
+
+    This effectively multiplies the control rate for smoother motion.
+
+    Usage:
+        interpolator = ActionInterpolator(multiplier=2)  # 2x control rate
+
+        # In control loop:
+        if interpolator.needs_new_action():
+            new_action = queue.get()
+            if new_action:
+                interpolator.add(new_action.cpu())
+
+        action = interpolator.get()
+        if action:
+            robot.send_action(action)
+    """
+
+    def __init__(self, multiplier: int = 1):
+        """Initialize the interpolator.
+
+        Args:
+            multiplier: Control rate multiplier (1 = no interpolation, 2 = 2x, 3 = 3x, etc.)
+        """
+        if multiplier < 1:
+            raise ValueError(f"multiplier must be >= 1, got {multiplier}")
+        self.multiplier = multiplier
+        self._prev: Tensor | None = None
+        self._buffer: list[Tensor] = []
+        self._idx = 0
+
+    @property
+    def enabled(self) -> bool:
+        """Whether interpolation is active (multiplier > 1)."""
+        return self.multiplier > 1
+
+    def reset(self):
+        """Reset interpolation state (call between episodes)."""
+        self._prev = None
+        self._buffer = []
+        self._idx = 0
+
+    def needs_new_action(self) -> bool:
+        """Check if a new action is needed from the queue."""
+        return self._idx >= len(self._buffer)
+
+    def add(self, action: Tensor) -> None:
+        """Add a new action and compute interpolated sequence.
+
+        Args:
+            action: New action tensor from policy/queue (already on CPU).
+        """
+        if self.multiplier > 1 and self._prev is not None:
+            self._buffer = []
+            for i in range(1, self.multiplier + 1):
+                t = i / self.multiplier
+                interp = self._prev + t * (action - self._prev)
+                self._buffer.append(interp)
+        else:
+            # First step: no previous action yet, so run at base FPS without interpolation.
+            self._buffer = [action.clone()]
+        self._prev = action.clone()
+        self._idx = 0
+
+    def get(self) -> Tensor | None:
+        """Get the next interpolated action.
+
+        Returns:
+            Next action tensor, or None if buffer is exhausted.
+        """
+        if self._idx >= len(self._buffer):
+            return None
+        action = self._buffer[self._idx]
+        self._idx += 1
+        return action
+
+    def get_control_interval(self, fps: float) -> float:
+        """Get the control interval based on interpolation multiplier.
+
+        Args:
+            fps: Base frames per second.
+
+        Returns:
+            Control interval in seconds (divided by multiplier).
+        """
+        return 1.0 / (fps * self.multiplier)
@@ -79,6 +79,13 @@ class ActionQueue:
            self.last_index += 1
            return action.clone()

+    def clear(self) -> None:
+        """Clear queued actions and reset consumption index."""
+        with self.lock:
+            self.queue = None
+            self.original_queue = None
+            self.last_index = 0
+
    def qsize(self) -> int:
        """Get the number of remaining actions in the queue.

@@ -123,14 +130,26 @@ class ActionQueue:
        with self.lock:
            if self.original_queue is None:
                return None
-            return self.original_queue[self.last_index :]
+            return self.original_queue[self.last_index :].clone()
+
+    def get_processed_left_over(self) -> Tensor | None:
+        """Get leftover processed actions (the actions currently executed by the robot).
+
+        Returns:
+            Tensor | None: Remaining processed actions (remaining_steps, action_dim),
+                or None if no processed queue exists.
+        """
+        with self.lock:
+            if self.queue is None:
+                return None
+            return self.queue[self.last_index :].clone()

    def merge(
        self,
        original_actions: Tensor,
        processed_actions: Tensor,
        real_delay: int,
-        action_index_before_inference: int | None = 0,
+        action_index_before_inference: int | None = None,
    ):
        """Merge new actions into the queue.

@@ -145,10 +164,10 @@ class ActionQueue:
            action_index_before_inference: Index before inference started, for validation.
        """
        with self.lock:
-            self._check_delays(real_delay, action_index_before_inference)
+            delay = self._check_and_resolve_delays(real_delay, action_index_before_inference)

            if self.cfg.enabled:
-                self._replace_actions_queue(original_actions, processed_actions, real_delay)
+                self._replace_actions_queue(original_actions, processed_actions, delay)
                return

            self._append_actions_queue(original_actions, processed_actions)
@@ -164,12 +183,13 @@ class ActionQueue:
            processed_actions: Post-processed actions for robot.
            real_delay: Number of time steps to skip due to inference delay.
        """
-        self.original_queue = original_actions[real_delay:].clone()
-        self.queue = processed_actions[real_delay:].clone()
+        clamped_delay = max(0, min(real_delay, len(original_actions), len(processed_actions)))
+        self.original_queue = original_actions[clamped_delay:].clone()
+        self.queue = processed_actions[clamped_delay:].clone()

        logger.debug(f"original_actions shape: {self.original_queue.shape}")
        logger.debug(f"processed_actions shape: {self.queue.shape}")
-        logger.debug(f"real_delay: {real_delay}")
+        logger.debug(f"real_delay: {real_delay}, clamped_delay: {clamped_delay}")

        self.last_index = 0

@@ -196,7 +216,9 @@ class ActionQueue:

        self.last_index = 0

-    def _check_delays(self, real_delay: int, action_index_before_inference: int | None = None):
+    def _check_and_resolve_delays(
+        self, real_delay: int, action_index_before_inference: int | None = None
+    ) -> int:
        """Validate that computed delays match expectations.

        Compares the delay computed from inference latency with the actual
@@ -205,15 +227,20 @@ class ActionQueue:
        Args:
            real_delay: Delay computed from inference latency.
            action_index_before_inference: Action index when inference started.
-        """
-        if action_index_before_inference is None:
-            return

-        indexes_diff = self.last_index - action_index_before_inference
-        if indexes_diff != real_delay:
-            # Let's check that action index difference (real delay calculated based on action queue)
-            # is the same as delay calculated based on inference latency
-            logger.warning(
-                f"[ACTION_QUEUE] Indexes diff is not equal to real delay. "
-                f"Indexes diff: {indexes_diff}, real delay: {real_delay}"
-            )
+        Returns:
+            int: Delay to use.
+        """
+        effective_delay = max(0, real_delay)
+
+        if action_index_before_inference is not None:
+            indexes_diff = max(0, self.last_index - action_index_before_inference)
+            if indexes_diff != real_delay:
+                logger.warning(
+                    "Indexes diff is not equal to real delay. indexes_diff=%d, real_delay=%d",
+                    indexes_diff,
+                    real_delay,
+                )
+                return real_delay
+
+        return effective_delay
@@ -1,14 +0,0 @@
-## Paper
-
-https://arxiv.org/abs/2509.25358
-
-## Citation
-
-```bibtex
-@article{chen2025sarm,
-  title={SARM: Stage-Aware Reward Modeling for Long Horizon Robot Manipulation},
-  author={Chen, Qianzhong and Yu, Justin and Schwager, Mac and Abbeel, Pieter and Shentu, Yide and Wu, Philipp},
-  journal={arXiv preprint arXiv:2509.25358},
-  year={2025}
-}
-```
@@ -0,0 +1 @@
+../../../../docs/source/policy_sarm_README.md
@@ -96,9 +96,11 @@ class BiOpenArmFollower(Robot):
        left_arm_motors_ft = self.left_arm._motors_ft
        right_arm_motors_ft = self.right_arm._motors_ft

+        # Right first, then left — matches the teleoperator (OpenArmMini) ordering
+        # and the dataset feature names recorded during data collection.
        return {
-            **{f"left_{k}": v for k, v in left_arm_motors_ft.items()},
            **{f"right_{k}": v for k, v in right_arm_motors_ft.items()},
+            **{f"left_{k}": v for k, v in left_arm_motors_ft.items()},
        }

    @property
@@ -150,14 +152,16 @@ class BiOpenArmFollower(Robot):
        left_cam_keys = set(self.left_arm.cameras.keys())
        right_cam_keys = set(self.right_arm.cameras.keys())

-        left_obs = self.left_arm.get_observation()
-        for key, value in left_obs.items():
-            obs_dict[key if key in left_cam_keys else f"left_{key}"] = value
-
+        # Right first, then left — matches the teleoperator (OpenArmMini) ordering
+        # and the dataset feature names recorded during data collection.
        right_obs = self.right_arm.get_observation()
        for key, value in right_obs.items():
            obs_dict[key if key in right_cam_keys else f"right_{key}"] = value

+        left_obs = self.left_arm.get_observation()
+        for key, value in left_obs.items():
+            obs_dict[key if key in left_cam_keys else f"left_{key}"] = value
+
        return obs_dict

    @check_if_not_connected
@@ -183,7 +187,7 @@ class BiOpenArmFollower(Robot):
        prefixed_sent_action_left = {f"left_{key}": value for key, value in sent_action_left.items()}
        prefixed_sent_action_right = {f"right_{key}": value for key, value in sent_action_right.items()}

-        return {**prefixed_sent_action_left, **prefixed_sent_action_right}
+        return {**prefixed_sent_action_right, **prefixed_sent_action_left}

    @check_if_not_connected
    def disconnect(self):
@@ -23,10 +23,12 @@ from ..config import RobotConfig


@RobotConfig.register_subclass("bi_openarm_follower")
-@dataclass
+@dataclass(kw_only=True)
 class BiOpenArmFollowerConfig(RobotConfig):
    """Configuration class for Bi OpenArm Follower robots."""

+    id: str | None = "bi_openarm_follower"
+
    left_arm_config: OpenArmFollowerConfigBase
    right_arm_config: OpenArmFollowerConfigBase

@@ -74,6 +74,8 @@ from pathlib import Path
 from pprint import pformat
 from typing import Any

+import torch
+
 from lerobot.cameras import (  # noqa: F401
    CameraConfig,  # noqa: F401
 )
@@ -90,6 +92,7 @@ from lerobot.datasets.pipeline_features import aggregate_pipeline_dataset_featur
 from lerobot.datasets.video_utils import VideoEncodingManager
 from lerobot.policies.factory import make_policy, make_pre_post_processors
 from lerobot.policies.pretrained import PreTrainedPolicy
+from lerobot.policies.rtc import ActionInterpolator
 from lerobot.policies.utils import make_robot_action
 from lerobot.processor import (
    PolicyAction,
@@ -226,6 +229,9 @@ class RecordConfig:
    play_sounds: bool = True
    # Resume recording on an existing dataset.
    resume: bool = False
+    # Action interpolation multiplier for smoother policy control (1=off, 2=2x, 3=3x)
+    # Only applies when using a policy (not teleop)
+    interpolation_multiplier: int = 1

    def __post_init__(self):
        # HACK: We parse again the cli args here to get the pretrained path if there was one.
@@ -298,6 +304,7 @@ def record_loop(
    control_time_s: int | None = None,
    single_task: str | None = None,
    display_data: bool = False,
+    interpolator: ActionInterpolator | None = None,
    display_compressed_images: bool = False,
 ):
    if dataset is not None and dataset.fps != fps:
@@ -334,6 +341,16 @@ def record_loop(
        preprocessor.reset()
        postprocessor.reset()

+    # Reset interpolator if provided
+    if interpolator is not None:
+        interpolator.reset()
+
+    # Calculate control interval based on interpolation
+    use_interpolation = interpolator is not None and interpolator.enabled and policy is not None
+    control_interval = interpolator.get_control_interval(fps) if interpolator else 1 / fps
+    # Pre-compute action key order outside the hot loop — it won't change mid-episode.
+    action_keys = sorted(robot.action_features) if use_interpolation else []
+
    no_action_count = 0
    timestamp = 0
    start_episode_t = time.perf_counter()
@@ -353,28 +370,67 @@ def record_loop(
        if policy is not None or dataset is not None:
            observation_frame = build_dataset_frame(dataset.features, obs_processed, prefix=OBS_STR)

+        # Track whether this iteration should be recorded to the dataset.
+        # Interpolated-only iterations send actions to the robot but don't record frames,
+        # keeping the dataset at the original fps while the robot moves at the higher rate.
+        is_record_frame = True
+
        # Get action from either policy or teleop
        if policy is not None and preprocessor is not None and postprocessor is not None:
-            action_values = predict_action(
-                observation=observation_frame,
-                policy=policy,
-                device=get_safe_torch_device(policy.config.device),
-                preprocessor=preprocessor,
-                postprocessor=postprocessor,
-                use_amp=policy.config.use_amp,
-                task=single_task,
-                robot_type=robot.robot_type,
-            )
+            # With interpolation: only call policy when interpolator needs new action
+            if use_interpolation:
+                ran_inference = False

-            act_processed_policy: RobotAction = make_robot_action(action_values, dataset.features)
+                if interpolator.needs_new_action():
+                    action_values = predict_action(
+                        observation=observation_frame,
+                        policy=policy,
+                        device=get_safe_torch_device(policy.config.device),
+                        preprocessor=preprocessor,
+                        postprocessor=postprocessor,
+                        use_amp=policy.config.use_amp,
+                        task=single_task,
+                        robot_type=robot.robot_type,
+                    )
+                    act_processed_policy = make_robot_action(action_values, dataset.features)
+                    robot_action_to_send = robot_action_processor((act_processed_policy, obs))
+
+                    action_tensor = torch.tensor([robot_action_to_send[k] for k in action_keys])
+                    interpolator.add(action_tensor)
+                    ran_inference = True
+
+                interp_action = interpolator.get()
+                if interp_action is not None:
+                    robot_action_to_send = {k: interp_action[i].item() for i, k in enumerate(action_keys)}
+                    action_values = robot_action_to_send
+                else:
+                    continue
+
+                is_record_frame = ran_inference
+            else:
+                action_values = predict_action(
+                    observation=observation_frame,
+                    policy=policy,
+                    device=get_safe_torch_device(policy.config.device),
+                    preprocessor=preprocessor,
+                    postprocessor=postprocessor,
+                    use_amp=policy.config.use_amp,
+                    task=single_task,
+                    robot_type=robot.robot_type,
+                )
+                act_processed_policy: RobotAction = make_robot_action(action_values, dataset.features)
+                # Applies a pipeline to the action, default is IdentityProcessor
+                robot_action_to_send = robot_action_processor((act_processed_policy, obs))

        elif policy is None and isinstance(teleop, Teleoperator):
+            act = teleop.get_action()
            if robot.name == "unitree_g1":
                teleop.send_feedback(obs)
-            act = teleop.get_action()

            # Applies a pipeline to the raw teleop action, default is IdentityProcessor
            act_processed_teleop = teleop_action_processor((act, obs))
+            action_values = act_processed_teleop
+            robot_action_to_send = robot_action_processor((act_processed_teleop, obs))

        elif policy is None and isinstance(teleop, list):
            arm_action = teleop_arm.get_action()
@@ -383,6 +439,8 @@ def record_loop(
            base_action = robot._from_keyboard_to_base_action(keyboard_action)
            act = {**arm_action, **base_action} if len(base_action) > 0 else arm_action
            act_processed_teleop = teleop_action_processor((act, obs))
+            action_values = act_processed_teleop
+            robot_action_to_send = robot_action_processor((act_processed_teleop, obs))
        else:
            no_action_count += 1
            if no_action_count == 1 or no_action_count % 10 == 0:
@@ -393,22 +451,14 @@ def record_loop(
                )
            continue

-        # Applies a pipeline to the action, default is IdentityProcessor
-        if policy is not None and act_processed_policy is not None:
-            action_values = act_processed_policy
-            robot_action_to_send = robot_action_processor((act_processed_policy, obs))
-        else:
-            action_values = act_processed_teleop
-            robot_action_to_send = robot_action_processor((act_processed_teleop, obs))
-
        # Send action to robot
        # Action can eventually be clipped using `max_relative_target`,
        # so action actually sent is saved in the dataset. action = postprocessor.process(action)
        # TODO(steven, pepijn, adil): we should use a pipeline step to clip the action, so the sent action is the action that we input to the robot.
        _sent_action = robot.send_action(robot_action_to_send)

-        # Write to dataset
-        if dataset is not None:
+        # Write to dataset (only on real policy frames, not interpolated-only iterations)
+        if dataset is not None and is_record_frame:
            action_frame = build_dataset_frame(dataset.features, action_values, prefix=ACTION)
            frame = {**observation_frame, **action_frame, "task": single_task}
            dataset.add_frame(frame)
@@ -420,7 +470,7 @@ def record_loop(

        dt_s = time.perf_counter() - start_loop_t

-        sleep_time_s: float = 1 / fps - dt_s
+        sleep_time_s: float = control_interval - dt_s
        if sleep_time_s < 0:
            logging.warning(
                f"Record loop is running slower ({1 / dt_s:.1f} Hz) than the target FPS ({fps} Hz). Dataset frames might be dropped and robot control might be unstable. Common causes are: 1) Camera FPS not keeping up 2) Policy inference taking too long 3) CPU starvation"
@@ -506,6 +556,7 @@ def record(cfg: RecordConfig) -> LeRobotDataset:
        policy = None if cfg.policy is None else make_policy(cfg.policy, ds_meta=dataset.meta)
        preprocessor = None
        postprocessor = None
+        interpolator = None
        if cfg.policy is not None:
            preprocessor, postprocessor = make_pre_post_processors(
                policy_cfg=cfg.policy,
@@ -516,6 +567,10 @@ def record(cfg: RecordConfig) -> LeRobotDataset:
                    "rename_observations_processor": {"rename_map": cfg.dataset.rename_map},
                },
            )
+            # Create interpolator for smoother policy control
+            if cfg.interpolation_multiplier > 1:
+                interpolator = ActionInterpolator(multiplier=cfg.interpolation_multiplier)
+                logging.info(f"Action interpolation enabled: {cfg.interpolation_multiplier}x control rate")

        robot.connect()
        if teleop is not None:
@@ -547,6 +602,7 @@ def record(cfg: RecordConfig) -> LeRobotDataset:
                    control_time_s=cfg.dataset.episode_time_s,
                    single_task=cfg.dataset.single_task,
                    display_data=cfg.display_data,
+                    interpolator=interpolator,
                    display_compressed_images=display_compressed_images,
                )

@@ -32,9 +32,15 @@ from .config_openarm_mini import OpenArmMiniConfig
 logger = logging.getLogger(__name__)

 # Motors whose direction is inverted during readout
-RIGHT_MOTORS_TO_FLIP = ["joint_1", "joint_2", "joint_3", "joint_4", "joint_5"]
+RIGHT_MOTORS_TO_FLIP = ["joint_1", "joint_2", "joint_3", "joint_4", "joint_5", "joint_7"]
 LEFT_MOTORS_TO_FLIP = ["joint_1", "joint_3", "joint_4", "joint_5", "joint_6", "joint_7"]

+# Leader joint 6 maps to follower joint 7 and vice versa
+JOINT_REMAP = {"joint_6": "joint_7", "joint_7": "joint_6"}
+JOINT_REMAP_REVERSE = {"joint_7": "joint_6", "joint_6": "joint_7"}
+
+GRIPPER_TELEOP_TO_DEGREES = -0.65
+

 class OpenArmMini(Teleoperator):
    """
@@ -95,6 +101,8 @@ class OpenArmMini(Teleoperator):

    @property
    def action_features(self) -> dict[str, type]:
+        # Right first, then left — matches the robot (BiOpenArmFollower) ordering
+        # and the dataset feature names recorded during data collection.
        features: dict[str, type] = {}
        for motor in self.bus_right.motors:
            features[f"right_{motor}.pos"] = float
@@ -276,16 +284,70 @@ class OpenArmMini(Teleoperator):
        right_positions = self.bus_right.sync_read("Present_Position")
        left_positions = self.bus_left.sync_read("Present_Position")

+        # Right first, then left — matches the robot (BiOpenArmFollower) ordering
+        # and the dataset feature names recorded during data collection.
+        # Joint 6↔7 remap: leader joint_6 → follower joint_7 and vice versa.
        action: dict[str, Any] = {}
        for motor, val in right_positions.items():
-            action[f"right_{motor}.pos"] = -val if motor in RIGHT_MOTORS_TO_FLIP else val
+            target = JOINT_REMAP.get(motor, motor)
+            if motor == "gripper":
+                # Convert gripper from teleop 0-100 to openarms degrees: 0→0°, 100→-65°
+                action[f"right_{target}.pos"] = val * GRIPPER_TELEOP_TO_DEGREES
+            else:
+                action[f"right_{target}.pos"] = -val if motor in RIGHT_MOTORS_TO_FLIP else val
        for motor, val in left_positions.items():
-            action[f"left_{motor}.pos"] = -val if motor in LEFT_MOTORS_TO_FLIP else val
+            target = JOINT_REMAP.get(motor, motor)
+            if motor == "gripper":
+                action[f"left_{target}.pos"] = val * GRIPPER_TELEOP_TO_DEGREES
+            else:
+                action[f"left_{target}.pos"] = -val if motor in LEFT_MOTORS_TO_FLIP else val

        dt_ms = (time.perf_counter() - start) * 1e3
        logger.debug(f"{self} read action: {dt_ms:.1f}ms")
        return action

+    def enable_torque(self) -> None:
+        """Enable torque on both arms for position control."""
+        self.bus_right.enable_torque()
+        self.bus_left.enable_torque()
+
+    def disable_torque(self) -> None:
+        """Disable torque on both arms for free movement."""
+        self.bus_right.disable_torque()
+        self.bus_left.disable_torque()
+
+    def write_goal_positions(self, positions: dict[str, float]) -> None:
+        """Write goal positions to motors (inverse of get_action flip/gripper/remap logic)."""
+        right_goals: dict[str, float] = {}
+        left_goals: dict[str, float] = {}
+
+        for key, val in positions.items():
+            if not key.endswith(".pos"):
+                continue
+            motor_name = key.removesuffix(".pos")
+            if motor_name.startswith("right_"):
+                base = motor_name.removeprefix("right_")
+                # Reverse remap: follower joint_7 → leader joint_6 and vice versa
+                target = JOINT_REMAP_REVERSE.get(base, base)
+                if base == "gripper":
+                    # Convert robot degrees to teleop 0-100: 0°→0, -65°→100
+                    right_goals[target] = val / GRIPPER_TELEOP_TO_DEGREES
+                else:
+                    # Un-flip using the ORIGINAL motor name (target = leader motor)
+                    right_goals[target] = -val if target in RIGHT_MOTORS_TO_FLIP else val
+            elif motor_name.startswith("left_"):
+                base = motor_name.removeprefix("left_")
+                target = JOINT_REMAP_REVERSE.get(base, base)
+                if base == "gripper":
+                    left_goals[target] = val / GRIPPER_TELEOP_TO_DEGREES
+                else:
+                    left_goals[target] = -val if target in LEFT_MOTORS_TO_FLIP else val
+
+        if right_goals:
+            self.bus_right.sync_write("Goal_Position", right_goals)
+        if left_goals:
+            self.bus_left.sync_write("Goal_Position", left_goals)
+
    def send_feedback(self, feedback: dict[str, float]) -> None:
        raise NotImplementedError("Feedback is not yet implemented for OpenArm Mini.")

@@ -0,0 +1,559 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for ActionInterpolator and its interaction with ActionQueue (RTC)."""
+
+import pytest
+import torch
+
+from lerobot.policies.rtc.action_interpolator import ActionInterpolator
+from lerobot.policies.rtc.action_queue import ActionQueue
+from lerobot.policies.rtc.configuration_rtc import RTCConfig
+
+# ====================== Fixtures ======================
+
+
+@pytest.fixture
+def interp2():
+    """Create an ActionInterpolator with multiplier=2."""
+    return ActionInterpolator(multiplier=2)
+
+
+@pytest.fixture
+def interp3():
+    """Create an ActionInterpolator with multiplier=3."""
+    return ActionInterpolator(multiplier=3)
+
+
+# ====================== Initialization Tests ======================
+
+
+def test_interpolator_multiplier_1_no_interpolation():
+    """Test multiplier=1 creates a disabled interpolator."""
+    interp = ActionInterpolator(multiplier=1)
+    assert interp.multiplier == 1
+    assert not interp.enabled
+
+
+def test_interpolator_multiplier_2_enabled():
+    """Test multiplier=2 creates an enabled interpolator."""
+    interp = ActionInterpolator(multiplier=2)
+    assert interp.multiplier == 2
+    assert interp.enabled
+
+
+def test_interpolator_multiplier_0_raises():
+    """Test multiplier=0 raises ValueError."""
+    with pytest.raises(ValueError, match="multiplier must be >= 1"):
+        ActionInterpolator(multiplier=0)
+
+
+def test_interpolator_negative_multiplier_raises():
+    """Test negative multiplier raises ValueError."""
+    with pytest.raises(ValueError, match="multiplier must be >= 1"):
+        ActionInterpolator(multiplier=-1)
+
+
+def test_interpolator_default_multiplier_is_1():
+    """Test default multiplier is 1 (disabled)."""
+    interp = ActionInterpolator()
+    assert interp.multiplier == 1
+    assert not interp.enabled
+
+
+# ====================== needs_new_action Tests ======================
+
+
+def test_needs_new_action_true_initially(interp2):
+    """Test needs_new_action() returns True before any action is added."""
+    assert interp2.needs_new_action()
+
+
+def test_needs_new_action_false_after_add(interp2):
+    """Test needs_new_action() returns False right after add()."""
+    interp2.add(torch.tensor([1.0, 2.0]))
+    assert not interp2.needs_new_action()
+
+
+def test_needs_new_action_true_after_buffer_exhausted(interp2):
+    """Test needs_new_action() returns True after consuming all buffered actions."""
+    interp2.add(torch.tensor([1.0, 2.0]))
+    interp2.get()
+    assert interp2.needs_new_action()
+
+
+def test_needs_new_action_true_after_all_interpolated_consumed(interp2):
+    """Test needs_new_action() tracks interpolated sub-steps correctly."""
+    interp2.add(torch.tensor([0.0, 0.0]))
+    interp2.get()
+    assert interp2.needs_new_action()
+
+    interp2.add(torch.tensor([2.0, 4.0]))
+    interp2.get()
+    assert not interp2.needs_new_action()
+    interp2.get()
+    assert interp2.needs_new_action()
+
+
+# ====================== Passthrough Tests (multiplier=1) ======================
+
+
+def test_passthrough_single_action_returned_as_is():
+    """Test multiplier=1 returns the action unchanged."""
+    interp = ActionInterpolator(multiplier=1)
+    action = torch.tensor([3.0, 5.0])
+    interp.add(action)
+
+    result = interp.get()
+    assert result is not None
+    torch.testing.assert_close(result, action)
+
+
+def test_passthrough_none_after_single_get():
+    """Test multiplier=1 returns None after consuming the single action."""
+    interp = ActionInterpolator(multiplier=1)
+    interp.add(torch.tensor([1.0]))
+    interp.get()
+    assert interp.get() is None
+
+
+def test_passthrough_sequential_actions():
+    """Test multiplier=1 passes through consecutive actions one at a time."""
+    interp = ActionInterpolator(multiplier=1)
+    for val in [1.0, 2.0, 3.0]:
+        action = torch.tensor([val])
+        interp.add(action)
+        result = interp.get()
+        torch.testing.assert_close(result, action)
+        assert interp.get() is None
+
+
+# ====================== Interpolation Tests (multiplier=2) ======================
+
+
+def test_interpolation_2x_first_action_no_interpolation(interp2):
+    """Test first action has no previous, so buffer is just [action]."""
+    interp2.add(torch.tensor([0.0, 0.0]))
+    result = interp2.get()
+    torch.testing.assert_close(result, torch.tensor([0.0, 0.0]))
+    assert interp2.get() is None
+
+
+def test_interpolation_2x_second_action_produces_two_steps(interp2):
+    """Test second action produces 2 interpolated sub-steps."""
+    interp2.add(torch.tensor([0.0, 0.0]))
+    interp2.get()
+
+    interp2.add(torch.tensor([2.0, 4.0]))
+    step1 = interp2.get()
+    step2 = interp2.get()
+
+    torch.testing.assert_close(step1, torch.tensor([1.0, 2.0]))
+    torch.testing.assert_close(step2, torch.tensor([2.0, 4.0]))
+    assert interp2.get() is None
+
+
+def test_interpolation_2x_three_consecutive_actions(interp2):
+    """Test interpolation across three consecutive actions."""
+    a0 = torch.tensor([0.0])
+    a1 = torch.tensor([4.0])
+    a2 = torch.tensor([10.0])
+
+    interp2.add(a0)
+    torch.testing.assert_close(interp2.get(), a0)
+
+    interp2.add(a1)
+    torch.testing.assert_close(interp2.get(), torch.tensor([2.0]))
+    torch.testing.assert_close(interp2.get(), torch.tensor([4.0]))
+
+    interp2.add(a2)
+    torch.testing.assert_close(interp2.get(), torch.tensor([7.0]))
+    torch.testing.assert_close(interp2.get(), torch.tensor([10.0]))
+
+
+# ====================== Interpolation Tests (multiplier=3) ======================
+
+
+def test_interpolation_3x_produces_three_steps(interp3):
+    """Test multiplier=3 produces 3 interpolated sub-steps."""
+    interp3.add(torch.tensor([0.0, 0.0]))
+    interp3.get()
+
+    interp3.add(torch.tensor([3.0, 6.0]))
+    s1 = interp3.get()
+    s2 = interp3.get()
+    s3 = interp3.get()
+
+    torch.testing.assert_close(s1, torch.tensor([1.0, 2.0]))
+    torch.testing.assert_close(s2, torch.tensor([2.0, 4.0]))
+    torch.testing.assert_close(s3, torch.tensor([3.0, 6.0]))
+    assert interp3.get() is None
+
+
+def test_interpolation_3x_last_step_equals_target(interp3):
+    """Test last interpolated step equals the target action exactly."""
+    interp3.add(torch.tensor([10.0]))
+    interp3.get()
+
+    target = torch.tensor([100.0])
+    interp3.add(target)
+    interp3.get()
+    interp3.get()
+    last = interp3.get()
+    torch.testing.assert_close(last, target)
+
+
+# ====================== Reset Tests ======================
+
+
+def test_reset_clears_buffer(interp2):
+    """Test reset() clears the action buffer."""
+    interp2.add(torch.tensor([1.0]))
+    interp2.reset()
+    assert interp2.needs_new_action()
+    assert interp2.get() is None
+
+
+def test_reset_clears_prev(interp2):
+    """Test after reset, next add produces single-element buffer (no prev)."""
+    interp2.add(torch.tensor([0.0]))
+    interp2.get()
+    interp2.add(torch.tensor([10.0]))
+    interp2.get()
+    interp2.get()
+
+    interp2.reset()
+    interp2.add(torch.tensor([5.0]))
+    result = interp2.get()
+    torch.testing.assert_close(result, torch.tensor([5.0]))
+    assert interp2.get() is None
+
+
+def test_reset_episode_boundary(interp2):
+    """Test reset between two simulated episodes."""
+    interp2.add(torch.tensor([0.0]))
+    interp2.get()
+    interp2.add(torch.tensor([10.0]))
+    interp2.get()
+    interp2.get()
+
+    interp2.reset()
+
+    interp2.add(torch.tensor([100.0]))
+    result = interp2.get()
+    torch.testing.assert_close(result, torch.tensor([100.0]))
+    assert interp2.get() is None
+
+
+# ====================== get_control_interval Tests ======================
+
+
+def test_control_interval_30fps_multiplier_1():
+    """Test control interval at 30fps with no interpolation."""
+    interp = ActionInterpolator(multiplier=1)
+    assert interp.get_control_interval(30.0) == pytest.approx(1.0 / 30.0)
+
+
+def test_control_interval_30fps_multiplier_2(interp2):
+    """Test control interval at 30fps with 2x interpolation."""
+    assert interp2.get_control_interval(30.0) == pytest.approx(1.0 / 60.0)
+
+
+def test_control_interval_30fps_multiplier_3(interp3):
+    """Test control interval at 30fps with 3x interpolation."""
+    assert interp3.get_control_interval(30.0) == pytest.approx(1.0 / 90.0)
+
+
+def test_control_interval_60fps_multiplier_2(interp2):
+    """Test control interval at 60fps with 2x interpolation."""
+    assert interp2.get_control_interval(60.0) == pytest.approx(1.0 / 120.0)
+
+
+# ====================== get() on Empty Tests ======================
+
+
+def test_get_returns_none_before_any_add():
+    """Test get() returns None when no action has been added."""
+    interp = ActionInterpolator(multiplier=2)
+    assert interp.get() is None
+
+
+def test_get_returns_none_after_reset(interp2):
+    """Test get() returns None after reset."""
+    interp2.add(torch.tensor([1.0]))
+    interp2.reset()
+    assert interp2.get() is None
+
+
+# ====================== Multi-Dimensional Action Tests ======================
+
+
+def test_6dof_interpolation(interp2):
+    """Test interpolation works correctly with 6-dimensional actions."""
+    prev = torch.zeros(6)
+    target = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
+
+    interp2.add(prev)
+    interp2.get()
+
+    interp2.add(target)
+    mid = interp2.get()
+    end = interp2.get()
+
+    torch.testing.assert_close(mid, target / 2)
+    torch.testing.assert_close(end, target)
+
+
+# ====================== Simulated Control Loop Tests ======================
+
+
+def test_control_loop_produces_correct_action_count():
+    """Test N policy actions with multiplier M yields 1 + (N-1)*M robot commands."""
+    multiplier = 3
+    n_policy_actions = 5
+    interp = ActionInterpolator(multiplier=multiplier)
+
+    robot_commands = 0
+    for i in range(n_policy_actions):
+        action = torch.tensor([float(i)])
+        if interp.needs_new_action():
+            interp.add(action)
+        while True:
+            a = interp.get()
+            if a is None:
+                break
+            robot_commands += 1
+
+    expected = 1 + (n_policy_actions - 1) * multiplier
+    assert robot_commands == expected
+
+
+def test_control_loop_monotonic_increase():
+    """Test actions [0, 1, 2, 3] with multiplier=2 produce monotonically increasing values."""
+    interp = ActionInterpolator(multiplier=2)
+    all_values = []
+
+    for i in range(4):
+        interp.add(torch.tensor([float(i)]))
+        while True:
+            a = interp.get()
+            if a is None:
+                break
+            all_values.append(a.item())
+
+    for i in range(1, len(all_values)):
+        assert all_values[i] >= all_values[i - 1]
+
+
+# ====================== ActionQueue + ActionInterpolator Integration Tests ======================
+
+
+def _make_chunk(n_steps: int, action_dim: int = 2, offset: float = 0.0) -> torch.Tensor:
+    """Create a simple action chunk: each row is [offset + step_idx, offset + step_idx]."""
+    return torch.arange(n_steps, dtype=torch.float32).unsqueeze(1).expand(-1, action_dim) + offset
+
+
+def test_queue_interpolator_consumption_rate_matches_base_fps():
+    """Test queue.get() is called at base fps rate, not multiplied fps."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=3)
+
+    chunk = _make_chunk(10)
+    queue.merge(chunk, chunk.clone(), real_delay=0)
+
+    queue_gets = 0
+    control_ticks = 0
+
+    while True:
+        if interp.needs_new_action():
+            if queue.empty():
+                break
+            action = queue.get()
+            if action is None:
+                break
+            interp.add(action)
+            queue_gets += 1
+
+        result = interp.get()
+        if result is not None:
+            control_ticks += 1
+
+    assert queue_gets == 10
+    assert control_ticks == 1 + 9 * 3
+
+
+def test_queue_interpolator_leftover_decreases_only_on_queue_get():
+    """Test get_left_over() shrinks only on queue.get(), not on interpolator sub-steps."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=3)
+
+    chunk = _make_chunk(6)
+    queue.merge(chunk, chunk.clone(), real_delay=0)
+
+    assert interp.needs_new_action()
+    interp.add(queue.get())
+    leftover_after_first_get = queue.get_left_over()
+    assert leftover_after_first_get is not None
+    assert len(leftover_after_first_get) == 5
+
+    interp.get()
+    assert len(queue.get_left_over()) == 5
+
+    interp.add(queue.get())
+    assert len(queue.get_left_over()) == 4
+
+    for _ in range(3):
+        assert interp.get() is not None
+    assert len(queue.get_left_over()) == 4
+
+
+def test_queue_interpolator_processed_leftover_tracks_queue_index():
+    """Test get_processed_left_over() reflects queue's last_index, not interpolator state."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=2)
+
+    original = _make_chunk(8, offset=0.0)
+    processed = _make_chunk(8, offset=100.0)
+    queue.merge(original, processed, real_delay=0)
+
+    left = queue.get_processed_left_over()
+    assert len(left) == 8
+
+    for _ in range(3):
+        if interp.needs_new_action():
+            action = queue.get()
+            if action is not None:
+                interp.add(action)
+        interp.get()
+
+    proc_left = queue.get_processed_left_over()
+    orig_left = queue.get_left_over()
+    assert proc_left is not None and orig_left is not None
+    assert len(proc_left) == len(orig_left)
+    assert proc_left[0, 0].item() >= 100.0
+    assert orig_left[0, 0].item() < 100.0
+
+
+def test_queue_interpolator_merge_resets_queue_but_interpolator_keeps_prev():
+    """Test queue merge doesn't affect interpolator's prev, enabling smooth transitions."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=2)
+
+    chunk1 = torch.tensor([[0.0], [2.0], [4.0], [6.0], [8.0]])
+    queue.merge(chunk1, chunk1.clone(), real_delay=0)
+
+    consumed = []
+    for _ in range(5):
+        if interp.needs_new_action():
+            a = queue.get()
+            if a is not None:
+                interp.add(a)
+        r = interp.get()
+        if r is not None:
+            consumed.append(r.item())
+
+    assert interp.needs_new_action()
+    assert consumed[-1] == pytest.approx(4.0)
+
+    idx_before = queue.get_action_index()
+
+    chunk2 = torch.tensor([[10.0], [12.0], [14.0]])
+    queue.merge(chunk2, chunk2.clone(), real_delay=0, action_index_before_inference=idx_before)
+
+    first_action = queue.get()
+    assert first_action is not None
+    interp.add(first_action)
+    first_from_new = interp.get()
+    assert first_from_new is not None
+    assert first_from_new.item() == pytest.approx(7.0)
+
+
+def test_queue_interpolator_reset_does_not_affect_queue():
+    """Test interpolator reset leaves queue state untouched."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=2)
+
+    chunk = _make_chunk(5)
+    queue.merge(chunk, chunk.clone(), real_delay=0)
+
+    interp.add(queue.get())
+    interp.get()
+    interp.add(queue.get())
+    interp.get()
+    interp.get()
+
+    assert queue.qsize() == 3
+
+    interp.reset()
+
+    assert queue.qsize() == 3
+    assert len(queue.get_left_over()) == 3
+
+    interp.add(queue.get())
+    result = interp.get()
+    assert result is not None
+    assert queue.qsize() == 2
+
+
+def test_queue_interpolator_no_interpolation_1_to_1():
+    """Test multiplier=1 produces exactly 1 robot command per queue.get()."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=1)
+
+    chunk = _make_chunk(5)
+    queue.merge(chunk, chunk.clone(), real_delay=0)
+
+    robot_commands = 0
+    while not queue.empty():
+        if interp.needs_new_action():
+            action = queue.get()
+            if action is not None:
+                interp.add(action)
+        result = interp.get()
+        if result is not None:
+            robot_commands += 1
+
+    assert robot_commands == 5
+
+
+def test_queue_interpolator_delay_skips_stale_actions():
+    """Test merge with delay correctly skips stale actions for the interpolator."""
+    cfg = RTCConfig(enabled=True, execution_horizon=10)
+    queue = ActionQueue(cfg)
+    interp = ActionInterpolator(multiplier=2)
+
+    chunk1 = _make_chunk(10)
+    queue.merge(chunk1, chunk1.clone(), real_delay=0)
+
+    for _ in range(5):
+        if interp.needs_new_action():
+            a = queue.get()
+            if a is not None:
+                interp.add(a)
+        interp.get()
+
+    assert queue.get_action_index() == 3
+
+    chunk2 = _make_chunk(10, offset=100.0)
+    queue.merge(chunk2, chunk2.clone(), real_delay=3, action_index_before_inference=0)
+
+    first_action = queue.get()
+    assert first_action is not None
+    torch.testing.assert_close(first_action, torch.tensor([103.0, 103.0]))
Author	SHA1	Message	Date
Steven Palma	2e2c27da34	fix(ci): use GITHUB_TOKEN for automated PR	2026-04-06 21:09:21 +02:00
Steven Palma	913041e753	fix(ci): latest deps tests permissions (#3296 ) * fix(ci): latest deps tests permissions * fix(ci): force push dep update branch * fix(ci): change secret for permissions & Ci trigger	2026-04-06 14:56:05 +02:00
Steven Palma	2b541ddd4c	docs(ci): add readme for dockerfile (#3295 )	2026-04-06 13:22:45 +02:00
Steven Palma	50a1e67e94	feat(ci): add `uv.lock` (#3292 ) * feat(ci): add uv.lock * feat(ci): use uv.lock in CI PR testing * chore(ci): rename nightly to docker publish and test * feat(ci): automated update of uv.lock + remove unbound check + docker images now use uv.lock * fix(ci): add --force-with-lease + set -e for silent erros	2026-04-06 12:23:37 +02:00
Steven Palma	d60a700d2b	chore(policy): multi dit docs (#3285 ) * docs(policy): add libero results multi task dit + remove readme in src code * docs(policy): add hyperlink to doc file in src code * chore(style): pre-commit	2026-04-05 21:23:13 +02:00
Steven Palma	8c3d4cf900	chore(docs): no policy readme in src code (#3286 ) * chore(docs): move policies readme out of src code * chore(docs): create symlink for policy readme	2026-04-05 19:25:38 +02:00
Caroline Pascal	b6e60a6e30	chore(dependencies): bump minimum torch from 2.2.1 to 2.7 (#3156 ) * feat(ffmpeg): updating ffmpeg verion to 8.X * Revert "feat(ffmpeg): updating ffmpeg verion to 8.X" This reverts commit `bb0f03185c`. * chore(pyproject): updating pyproject to fit the minimally required version of torchcodec * chore(docs): updating doc with specific instructions for ffmpeg/torchcodec installation * fix(typo): reverting ceiling bound on pytorch to 2.11.0 * chore(format): removing empty line * chore(typo): fixing typo * chore(docs): adding warning in case of torchcodec/ffmpeg version mismatch * chore(docs): applying comments * chore(docs): adding uv commands for evdev on WSL * fix(typo): fixing typo * fix(typo): fixing typos again * chore(ruff): format * fix(evdev install): splitting evdev install instructions between conda and uv * chore(ruff): format --------- Co-authored-by: Steven Palma <imstevenpmwork@ieee.org>	2026-04-05 19:24:43 +02:00
Steven Palma	3596681d94	docs(policy): fix gr00t license docs (#3284 )	2026-04-05 19:09:15 +02:00
Pepijn	4dbbcca496	docs(benchmarks): add benchmark integration guide and standardize benchmark docs (#3270 ) * docs(benchmarks): add benchmark integration guide and standardize benchmark docs Add a comprehensive guide for adding new benchmarks to LeRobot, and refactor the existing LIBERO and Meta-World docs to follow the new standardized template. Made-with: Cursor * docs(benchmarks): clean up adding-benchmarks guide for clarity Rewrite for simpler language, better structure, and easier navigation. Move quick-reference table to the top, fold eval explanation into architecture section, condense the doc template to a bulleted outline. Made-with: Cursor * fix link * fix task count * Update docs/source/adding_benchmarks.mdx Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co> Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com> * Update docs/source/metaworld.mdx Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co> Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com> * Update docs/source/adding_benchmarks.mdx Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co> Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com> * Update docs/source/adding_benchmarks.mdx Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co> Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com> * Update docs/source/adding_benchmarks.mdx Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co> Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com> * docs(benchmarks): add verification checklist to adding-benchmarks guide Made-with: Cursor --------- Signed-off-by: Pepijn <138571049+pkooij@users.noreply.github.com> Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co>	2026-04-03 14:44:53 +02:00
Pepijn	818892a38b	feat(dagger): Add HIL/Dagger/HG-Dagger/RaC style data collection (#2833 ) * feat: HIL data collection, RTC interpolator, and action queue improvements - Add Human-in-the-Loop (HIL) data collection examples (sync + RTC) - Add HIL data collection documentation - Add ActionInterpolator for smoother policy control at higher rates - Integrate interpolator into lerobot-record and eval_with_real_robot - Add action queue clear() and get_processed_left_over() methods - Add rtc/__init__.py for cleaner imports * docs: expand Related Work section with paper summaries * fix: only record dataset frames at original fps, not at interpolated rate The interpolator speeds up robot control (e.g. 2x) but dataset frames should still be recorded at the original fps. Interpolated-only iterations now only send actions to the robot without writing to the dataset. * refactor: merge HIL sync and RTC scripts into single file with --rtc.enabled toggle Combines hil_data_collection.py and hil_data_collection_rtc.py into one script. RTC is toggled via --rtc.enabled=true (defaults to off for sync inference). Deletes the separate hil_data_collection_rtc.py and updates docs to reflect the single-script usage. * test: add ActionInterpolator test suite (29 tests) Covers constructor validation, passthrough (multiplier=1), 2x and 3x interpolation with exact value checks, reset/episode boundaries, control interval calculation, multi-dim actions, and simulated control loop integration. * test: add ActionQueue + ActionInterpolator integration tests Verifies the interpolator doesn't interfere with RTC's leftover chunk tracking: queue consumption rate matches base fps regardless of multiplier, get_left_over/get_processed_left_over only change on queue.get(), merge preserves smooth interpolation across chunks, and interpolator reset is independent of queue state. * feat: register SO follower/leader configs in HIL script Adds SOFollowerRobotConfig and SOLeaderTeleopConfig imports so SO100/SO101 robots can be used via --robot.type=so_follower and --teleop.type=so_leader. Updates docs accordingly. Made-with: Cursor * docs: remove em dashes from HIL documentation Made-with: Cursor * refactor: rename examples/rac to examples/hil Updates directory name and all references in docs and script docstrings. Made-with: Cursor * fix: encorperate pr feedback comments * refactor(tests): enhance ActionInterpolator test structure and add detailed docstrings * feedback pr and test fix * fix(test): pass correct real_delay in interpolator delay test The test was passing real_delay=0 and relying on _check_delays to silently override it with the index-based diff. Now passes real_delay=3 to match the 3 actions consumed during the simulated inference period. * fix pr feedback * ordering * update hil script * fix * default name * fix(bi_openarm): use kw_only=True to fix dataclass field ordering BiOpenArmFollowerConfig overrides `id` with a default, making it positional in the child — non-default `left_arm_config` then follows a default field, which Python dataclasses forbid. Adding kw_only=True (matching the parent RobotConfig) removes positional constraints. Made-with: Cursor * style: format long line in hil_data_collection.py Made-with: Cursor * pr feedback --------- Co-authored-by: Khalil Meftah <khalil.meftah@huggingface.co>	2026-04-02 19:53:59 +02:00
Pepijn	66fef25ded	docs(toctree): add Benchmarks section for LIBERO and Meta-World (#3268 ) * docs(toctree): add Benchmarks section for LIBERO and Meta-World Move LIBERO and Meta-World pages out of the Simulation section into a dedicated Benchmarks section so benchmark-specific docs are easier to find and the Simulation section stays focused on environment hubs. Made-with: Cursor * docs(toctree): move IsaacLab Arena into Benchmarks section Include NVIDIA IsaacLab Arena Environments alongside LIBERO and Meta-World in the Benchmarks section. Made-with: Cursor	2026-04-02 19:52:39 +02:00
Pepijn	2cf08b7a4b	Add create reward visualization (#3155 ) * Add create reward visualization and multimodal analysis tool * add example for creating progress video for sarm * nit * precommit * refactor: address review comments on create_progress_videos.py - Add shebang and Apache 2.0 license header - Replace hardcoded absolute OUTPUT_DIR with relative default (./progress_videos) - Add argparse CLI (--repo-id, --episode, --camera-key, --output-dir, --gif) - Wrap entrypoint in def main() - Replace all print() with logging - Use logging.error/warning instead of traceback.print_exc - Release VideoCapture via try/finally; consolidate triple-open into single seek - Eliminate intermediate clip file: seek directly via CAP_PROP_POS_MSEC - Make MP4 the default output, GIF opt-in via --gif flag - Add return types to all functions - Add Args/Returns docstrings - Use descriptive variable names throughout Made-with: Cursor * refactor: move create_progress_videos.py to examples/dataset/ for consistency Made-with: Cursor * refactor: address PR review comments on create_progress_videos.py - Replace Unicode ellipsis and multiplication sign with ASCII equivalents - Fix step numbering from 1-5 to 1-4 (only 4 actual steps) - Move frame_width reading into convert_mp4_to_gif - Remove unused text_height variable Made-with: Cursor	2026-04-02 16:58:07 +02:00
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_multi_task_dit_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_pi0_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_pi05_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_rtc_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_sarm_README.md`