feat(ci): extract task descriptions and embed in metrics artifact

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
feat(ci): add Libero train+eval smoke test (1 step, eval_freq=1)
2026-05-12 23:29:52 +00:00 · 2026-04-09 12:50:14 +02:00 · 2026-04-09 10:26:42 +02:00 · 2026-04-09 10:04:53 +02:00 · 2026-04-08 20:56:35 +02:00 · 2026-04-08 20:33:39 +02:00
51 changed files with 8037 additions and 564 deletions
@@ -0,0 +1,309 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Integration tests: build an isolated Docker image per benchmark and run a
+# 1-episode smoke eval. Each benchmark gets its own image so incompatible
+# dependency trees (e.g. hf-libero vs metaworld==3.0.0) can never collide.
+#
+# To add a new benchmark:
+#   1. Add docker/Dockerfile.benchmark.<name>  (install only lerobot[<name>])
+#   2. Copy one of the jobs below and adjust the image name and eval command.
+name: Benchmark Integration Tests
+
+on:
+  # Run manually from the Actions tab
+  workflow_dispatch:
+
+  # Run every Monday at 02:00 UTC.
+  schedule:
+    - cron: "0 2 * * 1"
+
+  push:
+    branches:
+      - feat/benchmark-ci
+      - main
+    paths:
+      - "src/lerobot/envs/**"
+      - "src/lerobot/scripts/lerobot_eval.py"
+      - "docker/Dockerfile.benchmark.*"
+      - ".github/workflows/benchmark_tests.yml"
+      - "pyproject.toml"
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "src/lerobot/envs/**"
+      - "src/lerobot/scripts/lerobot_eval.py"
+      - "docker/Dockerfile.benchmark.*"
+      - ".github/workflows/benchmark_tests.yml"
+      - "pyproject.toml"
+
+permissions:
+  contents: read
+
+env:
+  UV_VERSION: "0.8.0"
+  PYTHON_VERSION: "3.12"
+
+# Cancel in-flight runs for the same branch/PR.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # ── LIBERO ────────────────────────────────────────────────────────────────
+  # Isolated image: lerobot[libero] only (hf-libero, dm-control, mujoco chain)
+  libero-integration-test:
+    name: Libero — build image + 1-episode eval
+    runs-on:
+      group: aws-g6-4xlarge-plus
+    env:
+      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+          lfs: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses]
+        with:
+          cache-binary: false
+
+      # Build the benchmark-specific image; layer cache lives in the runner's
+      # local Docker daemon — reused across re-runs on the same machine.
+      - name: Build Libero benchmark image
+        uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses]
+        with:
+          context: .
+          file: docker/Dockerfile.benchmark.libero
+          push: false
+          load: true
+          tags: lerobot-benchmark-libero:ci
+          cache-from: type=local,src=/tmp/.buildx-cache-libero
+          cache-to: type=local,dest=/tmp/.buildx-cache-libero,mode=max
+
+      - name: Login to Hugging Face
+        if: env.HF_USER_TOKEN != ''
+        run: |
+          docker run --rm \
+            -e HF_HOME=/tmp/hf \
+            lerobot-benchmark-libero:ci \
+            bash -c "hf auth login --token '$HF_USER_TOKEN' --add-to-git-credential && hf auth whoami"
+
+      - name: Run Libero smoke eval (1 episode)
+        run: |
+          # Named container (no --rm) so we can docker cp artifacts out.
+          # Output to /tmp inside the container — user_lerobot cannot create
+          # root-level dirs like /artifacts.
+          docker run --name libero-eval --gpus all \
+            --shm-size=4g \
+            -e HF_HOME=/tmp/hf \
+            -e HF_USER_TOKEN="${HF_USER_TOKEN}" \
+            -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+            lerobot-benchmark-libero:ci \
+            bash -c "
+              hf auth login --token \"\$HF_USER_TOKEN\" --add-to-git-credential 2>/dev/null || true
+              lerobot-eval \
+                --policy.path=pepijn223/smolvla_libero \
+                --env.type=libero \
+                --env.task=libero_spatial \
+                --eval.batch_size=1 \
+                --eval.n_episodes=1 \
+                --eval.use_async_envs=false \
+                --policy.device=cuda \
+                '--env.camera_name_mapping={\"agentview_image\": \"camera1\", \"robot0_eye_in_hand_image\": \"camera2\"}' \
+                --policy.empty_cameras=1 \
+                --output_dir=/tmp/eval-artifacts
+              python3 /lerobot/scripts/ci/extract_task_descriptions.py \
+                --env libero --task libero_spatial \
+                --output /tmp/eval-artifacts/task_descriptions.json 2>/dev/null || true
+            "
+
+      - name: Copy Libero artifacts from container
+        if: always()
+        run: |
+          mkdir -p /tmp/libero-artifacts
+          docker cp libero-eval:/tmp/eval-artifacts/. /tmp/libero-artifacts/ 2>/dev/null || true
+          docker rm -f libero-eval || true
+
+      - name: Parse Libero eval metrics
+        if: always()
+        run: |
+          python3 scripts/ci/parse_eval_metrics.py \
+            --artifacts-dir /tmp/libero-artifacts \
+            --env libero \
+            --task libero_spatial \
+            --policy pepijn223/smolvla_libero
+
+      - name: Upload Libero rollout video
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: libero-rollout-video
+          path: /tmp/libero-artifacts/videos/
+          if-no-files-found: warn
+
+      - name: Upload Libero eval metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: libero-metrics
+          path: /tmp/libero-artifacts/metrics.json
+          if-no-files-found: warn
+
+      # ── LIBERO TRAIN+EVAL SMOKE ──────────────────────────────────────────────
+      # Train SmolVLA for 1 step (batch_size=1, dataset episode 0 only) then
+      # immediately runs eval inside the training loop (eval_freq=1, 1 episode).
+      # Tests the full train→eval-within-training pipeline end-to-end.
+      - name: Run Libero train+eval smoke (1 step, eval_freq=1)
+        run: |
+          docker run --name libero-train-smoke --gpus all \
+            --shm-size=4g \
+            -e HF_HOME=/tmp/hf \
+            -e HF_USER_TOKEN="${HF_USER_TOKEN}" \
+            -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+            lerobot-benchmark-libero:ci \
+            bash -c "
+              hf auth login --token \"\$HF_USER_TOKEN\" --add-to-git-credential 2>/dev/null || true
+              accelerate launch --num_processes=1 \$(which lerobot-train) \
+                --policy.path=lerobot/smolvla_base \
+                --policy.load_vlm_weights=true \
+                --policy.scheduler_decay_steps=25000 \
+                --policy.freeze_vision_encoder=false \
+                --policy.train_expert_only=false \
+                --dataset.repo_id=lerobot/libero \
+                --dataset.episodes=[0] \
+                --dataset.use_imagenet_stats=false \
+                --env.type=libero \
+                --env.task=libero_spatial \
+                '--env.camera_name_mapping={\"agentview_image\": \"camera1\", \"robot0_eye_in_hand_image\": \"camera2\"}' \
+                --policy.empty_cameras=1 \
+                --output_dir=/tmp/train-smoke \
+                --steps=1 \
+                --batch_size=1 \
+                --eval_freq=1 \
+                --eval.n_episodes=1 \
+                --eval.batch_size=1 \
+                --eval.use_async_envs=false \
+                --save_freq=1 \
+                --policy.push_to_hub=false \
+                '--rename_map={\"observation.images.image\": \"observation.images.camera1\", \"observation.images.image2\": \"observation.images.camera2\"}'
+            "
+
+      - name: Copy Libero train-smoke artifacts from container
+        if: always()
+        run: |
+          mkdir -p /tmp/libero-train-smoke-artifacts
+          docker cp libero-train-smoke:/tmp/train-smoke/. /tmp/libero-train-smoke-artifacts/ 2>/dev/null || true
+          docker rm -f libero-train-smoke || true
+
+      - name: Upload Libero train-smoke eval video
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: libero-train-smoke-video
+          path: /tmp/libero-train-smoke-artifacts/eval/
+          if-no-files-found: warn
+
+  # ── METAWORLD ─────────────────────────────────────────────────────────────
+  # Isolated image: lerobot[metaworld] only (metaworld==3.0.0, mujoco>=3 chain)
+  metaworld-integration-test:
+    name: MetaWorld — build image + 1-episode eval
+    runs-on:
+      group: aws-g6-4xlarge-plus
+    env:
+      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+          lfs: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses]
+        with:
+          cache-binary: false
+
+      - name: Build MetaWorld benchmark image
+        uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses]
+        with:
+          context: .
+          file: docker/Dockerfile.benchmark.metaworld
+          push: false
+          load: true
+          tags: lerobot-benchmark-metaworld:ci
+          cache-from: type=local,src=/tmp/.buildx-cache-metaworld
+          cache-to: type=local,dest=/tmp/.buildx-cache-metaworld,mode=max
+
+      - name: Run MetaWorld smoke eval (1 episode)
+        run: |
+          docker run --name metaworld-eval --gpus all \
+            --shm-size=4g \
+            -e HF_HOME=/tmp/hf \
+            -e HF_USER_TOKEN="${HF_USER_TOKEN}" \
+            -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+            lerobot-benchmark-metaworld:ci \
+            bash -c "
+              hf auth login --token \"\$HF_USER_TOKEN\" --add-to-git-credential 2>/dev/null || true
+              lerobot-eval \
+                --policy.path=pepijn223/smolvla_metaworld \
+                --env.type=metaworld \
+                --env.task=metaworld-push-v3 \
+                --eval.batch_size=1 \
+                --eval.n_episodes=1 \
+                --eval.use_async_envs=false \
+                --policy.device=cuda \
+                '--rename_map={\"observation.image\": \"observation.images.camera1\"}' \
+                --policy.empty_cameras=2 \
+                --output_dir=/tmp/eval-artifacts
+              python3 /lerobot/scripts/ci/extract_task_descriptions.py \
+                --env metaworld --task metaworld-push-v3 \
+                --output /tmp/eval-artifacts/task_descriptions.json 2>/dev/null || true
+            "
+
+      - name: Copy MetaWorld artifacts from container
+        if: always()
+        run: |
+          mkdir -p /tmp/metaworld-artifacts
+          docker cp metaworld-eval:/tmp/eval-artifacts/. /tmp/metaworld-artifacts/ 2>/dev/null || true
+          docker rm -f metaworld-eval || true
+
+      - name: Parse MetaWorld eval metrics
+        if: always()
+        run: |
+          python3 scripts/ci/parse_eval_metrics.py \
+            --artifacts-dir /tmp/metaworld-artifacts \
+            --env metaworld \
+            --task metaworld-push-v3 \
+            --policy pepijn223/smolvla_metaworld
+
+      - name: Upload MetaWorld rollout video
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metaworld-rollout-video
+          path: /tmp/metaworld-artifacts/videos/
+          if-no-files-found: warn
+
+      - name: Upload MetaWorld eval metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metaworld-metrics
+          path: /tmp/metaworld-artifacts/metrics.json
+          if-no-files-found: warn
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# This workflow handles nightly testing & docker images publishing.
-name: Nightly
+# This workflow handles Docker image publishing & testing.
+name: Docker Publish & Test
 permissions:
  contents: read

@@ -39,8 +39,8 @@ concurrency:

 jobs:
  # This job builds a CPU image for testing & distribution
-  build-docker-cpu-nightly:
-    name: Build CPU Docker for Nightly
+  build-docker-cpu:
+    name: Build CPU Docker
    runs-on:
      group: aws-general-8-plus
    if: github.repository == 'huggingface/lerobot'
@@ -74,8 +74,8 @@ jobs:
          tags: ${{ env.DOCKER_IMAGE_NAME_CPU }}

  # This job builds a GPU image for testing & distribution
-  build-docker-gpu-nightly:
-    name: Build GPU Docker for Nightly
+  build-docker-gpu:
+    name: Build GPU Docker
    runs-on:
      group: aws-general-8-plus
    if: github.repository == 'huggingface/lerobot'
@@ -109,9 +109,9 @@ jobs:
          tags: ${{ env.DOCKER_IMAGE_NAME_GPU }}

  # This job runs the E2E tests + pytest with all extras in the CPU image
-  nightly-cpu-tests:
-    name: Nightly CPU Tests
-    needs: [build-docker-cpu-nightly]
+  cpu-tests:
+    name: CPU Tests
+    needs: [build-docker-cpu]
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
@@ -121,7 +121,7 @@ jobs:
      TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    container:
-      image: ${{ needs.build-docker-cpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images]
+      image: ${{ needs.build-docker-cpu.outputs.image_tag }} # zizmor: ignore[unpinned-images]
      options: --shm-size "16gb"
      credentials:
        username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
@@ -142,9 +142,9 @@ jobs:
        run: make test-end-to-end

  # This job runs the E2E tests + pytest with all extras in the GPU image
-  nightly-gpu-tests:
-    name: Nightly GPU Tests
-    needs: [build-docker-gpu-nightly]
+  gpu-tests:
+    name: GPU Tests
+    needs: [build-docker-gpu]
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
@@ -154,7 +154,7 @@ jobs:
      TRITON_CACHE_DIR: /home/user_lerobot/.cache/triton
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    container:
-      image: ${{ needs.build-docker-gpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images]
+      image: ${{ needs.build-docker-gpu.outputs.image_tag }} # zizmor: ignore[unpinned-images]
      options: --gpus all --shm-size "16gb"
      credentials:
        username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
@@ -175,9 +175,9 @@ jobs:
        run: make test-end-to-end

  # This job runs multi-GPU training tests with 4 GPUs
-  nightly-multi-gpu-tests:
-    name: Nightly Multi-GPU Tests
-    needs: [build-docker-gpu-nightly]
+  multi-gpu-tests:
+    name: Multi-GPU Tests
+    needs: [build-docker-gpu]
    runs-on:
      group: aws-g4dn-12xlarge  # Instance with 4 GPUs
    env:
@@ -188,7 +188,7 @@ jobs:
      CUDA_VISIBLE_DEVICES: "0,1,2,3"
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    container:
-      image: ${{ needs.build-docker-gpu-nightly.outputs.image_tag }} # zizmor: ignore[unpinned-images]
+      image: ${{ needs.build-docker-gpu.outputs.image_tag }} # zizmor: ignore[unpinned-images]
      options: --gpus all --shm-size "16gb"
      credentials:
        username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
@@ -33,7 +33,7 @@ jobs:
      github.event.workflow_run.event == 'pull_request' &&
      github.event.workflow_run.conclusion == 'success' &&
      github.repository == 'huggingface/lerobot'
-    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main
+    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
    with:
      package_name: lerobot
    secrets:
@@ -55,7 +55,7 @@ jobs:
      github.repository == 'huggingface/lerobot'
    permissions:
      contents: read
-    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main
+    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
    with:
      commit_sha: ${{ github.sha }}
      package: lerobot
@@ -78,7 +78,7 @@ jobs:
    permissions:
      contents: read
      pull-requests: write
-    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
    with:
      commit_sha: ${{ github.event.pull_request.head.sha }}
      pr_number: ${{ github.event.number }}
@@ -27,6 +27,7 @@ on:
      - "tests/**"
      - ".github/workflows/**"
      - "pyproject.toml"
+      - "uv.lock"
      - "Makefile"
  push:
    branches:
@@ -36,6 +37,7 @@ on:
      - "tests/**"
      - ".github/workflows/**"
      - "pyproject.toml"
+      - "uv.lock"
      - "Makefile"

 permissions:
@@ -63,7 +65,7 @@ jobs:
      HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          lfs: true
@@ -81,14 +83,14 @@ jobs:
          libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev

      - name: Setup uv and Python
-        uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses]
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6
        with:
          enable-cache: true
          version: ${{ env.UV_VERSION }}
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install lerobot with test extras
-        run: uv sync --extra "test"
+        run: uv sync --locked --extra "test"

      - name: Login to Hugging Face
        if: env.HF_USER_TOKEN != ''
@@ -29,6 +29,7 @@ on:
      - "tests/**"
      - ".github/workflows/**"
      - "pyproject.toml"
+      - "uv.lock"
      - "Makefile"

 permissions:
@@ -62,7 +63,7 @@ jobs:
      HF_LEROBOT_HOME: /mnt/cache/.cache/huggingface/lerobot
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          lfs: true
          persist-credentials: false
@@ -79,14 +80,14 @@ jobs:
          speech-dispatcher libgeos-dev portaudio19-dev

      - name: Setup uv and Python
-        uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses]
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6
        with:
          enable-cache: true
          version: ${{ env.UV_VERSION }}
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install lerobot with all extras
-        run: uv sync --extra all # TODO(Steven): Make flash-attn optional
+        run: uv sync --locked --extra all # TODO(Steven): Make flash-attn optional

      - name: Login to Hugging Face
        if: env.HF_USER_TOKEN != ''
@@ -136,21 +137,21 @@ jobs:
          sudo apt-get update
          sudo apt-get install git-lfs
          git lfs install
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          lfs: true
          persist-credentials: false
      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses]
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f  # v3
        with:
          cache-binary: false
      - name: Login to Docker Hub
-        uses: docker/login-action@v3 # zizmor: ignore[unpinned-uses]
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9  # v3
        with:
          username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }}
          password: ${{ secrets.DOCKERHUB_LEROBOT_PASSWORD }}
      - name: Build and push Docker image
-        uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses]
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8  # v6
        with:
          context: .
          file: ./docker/Dockerfile.internal
@@ -12,38 +12,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# This workflow handles full testing with unboud dependencies versions.
-name: Unbound Dependency Tests
+# This workflow tests the project against the latest upstream dependencies
+# (within pyproject.toml constraints) and opens a PR to update uv.lock
+# if the tests pass and the lockfile has changed.
+name: Latest Dependency Tests

 on:
  # Allows running this workflow manually from the Actions tab
  workflow_dispatch:

-  # Run on the 1st and 15th of every month at 09:00 UTC
-  # schedule:
-  #  - cron: '0 2 1,15 * *'
-
-permissions:
-  contents: read
+  # Runs at 03:00 UTC
+  schedule:
+    - cron: "0 3 * * *"

 # Sets up the environment variables
 env:
  UV_VERSION: "0.8.0"
  PYTHON_VERSION: "3.12"
-  DOCKER_IMAGE_NAME: huggingface/lerobot-gpu:unbound
+  DOCKER_IMAGE_NAME: huggingface/lerobot-gpu:latest-deps

-# Ensures that only the latest action is built, canceling older runs.
+# Ensures that only the latest run is active, canceling older runs.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}
  cancel-in-progress: true

 jobs:

-  # This job runs the E2E tests + pytest with all unbound extras
-  full-tests:
-    name: Full Unbound Tests
+  # This job upgrades the lockfile and checks if dependencies have changed
+  upgrade-lock:
+    name: Upgrade Lockfile
    runs-on: ubuntu-latest
    if: github.repository == 'huggingface/lerobot'
+    permissions:
+      contents: read
+    outputs:
+      changed: ${{ steps.diff.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup uv and Python
+        uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses]
+        with:
+          version: ${{ env.UV_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Upgrade uv.lock
+        run: uv lock --upgrade
+
+      - name: Check for changes
+        id: diff
+        run: |
+          if git diff --quiet uv.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "uv.lock is up to date — no dependency changes."
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "uv.lock has changed — running tests."
+          fi
+
+      - name: Upload updated lockfile
+        if: steps.diff.outputs.changed == 'true'
+        uses: actions/upload-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+          path: uv.lock
+
+  # This job runs the full test suite with the upgraded dependencies
+  cpu-tests:
+    name: CPU Tests (Latest Deps)
+    needs: [upgrade-lock]
+    if: needs.upgrade-lock.outputs.changed == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
    env:
      MUJOCO_GL: egl
      HF_HOME: /mnt/cache/.cache/huggingface
@@ -55,6 +98,11 @@ jobs:
          lfs: true
          persist-credentials: false

+      - name: Download updated lockfile
+        uses: actions/download-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+
      # NOTE(Steven): Mount to `/mnt` to avoid the limited storage on `/home`. Consider cleaning default SDKs or using self-hosted runners for more space.
      # (As of 2024-06-10, the runner's `/home` has only 6.2 GB free—8% of its 72 GB total.)
      - name: Setup /mnt storage
@@ -73,34 +121,32 @@ jobs:
          version: ${{ env.UV_VERSION }}
          python-version: ${{ env.PYTHON_VERSION }}

-      - name: Unbound dependencies
-        run: |
-          sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml
-          echo "Dependencies unbound:" && cat pyproject.toml
-
      - name: Install lerobot with all extras
-        run: uv sync --extra all # TODO(Steven): Make flash-attn optional
+        run: uv sync --locked --extra all # TODO(Steven): Make flash-attn optional
+
      - name: Login to Hugging Face
        if: env.HF_USER_TOKEN != ''
        run: |
          uv run hf auth login --token "$HF_USER_TOKEN" --add-to-git-credential
          uv run hf auth whoami
+
      - name: Run pytest (all extras)
-        run: uv run pytest tests -vv
+        run: uv run pytest tests -vv --maxfail=10

      - name: Run end-to-end tests
        run: uv run make test-end-to-end

-  # This job builds a GPU enabled image for testing
+  # This job builds a GPU-enabled Docker image with the upgraded dependencies
  build-and-push-docker:
    name: Build and Push Docker
+    needs: [upgrade-lock]
+    if: needs.upgrade-lock.outputs.changed == 'true'
+    permissions:
+      contents: read
    runs-on:
      group: aws-general-8-plus
-    if: github.repository == 'huggingface/lerobot'
    outputs:
      image_tag: ${{ env.DOCKER_IMAGE_NAME }}
-    env:
-      GITHUB_REF: ${{ github.ref }}
    steps:
      - name: Install Git LFS
        run: |
@@ -111,6 +157,12 @@ jobs:
        with:
          lfs: true
          persist-credentials: false
+
+      - name: Download updated lockfile
+        uses: actions/download-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses]
        with:
@@ -127,14 +179,13 @@ jobs:
          file: ./docker/Dockerfile.internal
          push: true
          tags: ${{ env.DOCKER_IMAGE_NAME }}
-          build-args: |
-            UNBOUND_DEPS=true

-  # This job runs pytest with all unbound extras in a GPU enabled host
-  # It runs everytime a test image is created
+  # This job runs pytest with all extras on a GPU-enabled host
  gpu-tests:
-    name: GPU Unbound Tests
+    name: GPU Tests (Latest Deps)
    needs: [build-and-push-docker]
+    permissions:
+      contents: read
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
@@ -159,17 +210,69 @@ jobs:
        run: |
          hf auth login --token "$HF_USER_TOKEN" --add-to-git-credential
          hf auth whoami
+      - name: Fix ptxas permissions
+        run: chmod +x /lerobot/.venv/lib/python3.12/site-packages/triton/backends/nvidia/bin/ptxas
      - name: Run pytest on GPU
-        run: pytest tests -vv
+        run: pytest tests -vv --maxfail=10
      - name: Run end-to-end tests
        run: make test-end-to-end

-  # This job deletes the test image recently created
-  # It runs everytime after the gpu-tests have finished
-  delete-unbound-image:
-    name: Delete Unbound Image
+  # This job creates or updates a PR with the upgraded lockfile
+  open-pr:
+    name: Open PR
+    needs: [cpu-tests, gpu-tests, upgrade-lock]
+    if: success() && needs.upgrade-lock.outputs.changed == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ secrets.UPDATE_LOCK_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Download updated lockfile
+        uses: actions/download-artifact@v4 # zizmor: ignore[unpinned-uses]
+        with:
+          name: uv-lock
+
+      - name: Create or update PR
+        run: |
+          set -euo pipefail
+          BRANCH="auto/update-uv-lock"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git"
+
+          git checkout -B "$BRANCH"
+          git add uv.lock
+          git commit -m "chore(dependencies): update uv.lock"
+          git push --force origin "$BRANCH"
+
+          # Create PR only if one doesn't already exist for this branch
+          EXISTING_PR=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number')
+          if [ -z "$EXISTING_PR" ]; then
+            gh pr create \
+              --title "chore(dependencies): update uv.lock" \
+              --body "Automated update of \`uv.lock\` after successful latest dependency tests (CPU + GPU).
+
+          This PR upgrades all dependencies to their latest versions within the ranges specified in \`pyproject.toml\`." \
+              --head "$BRANCH" \
+              --base main
+          else
+            echo "PR #$EXISTING_PR already exists, branch has been updated."
+          fi
+
+  # This job deletes the temporary Docker image after tests complete
+  cleanup-docker:
+    name: Cleanup Docker Image
    needs: [gpu-tests, build-and-push-docker]
    if: always() && needs.build-and-push-docker.result == 'success'
+    permissions:
+      contents: read
    runs-on: ubuntu-latest
    steps:
      - name: Get Docker Hub Token and Delete Image
@@ -180,8 +283,7 @@ jobs:
          IMAGE_FULL: ${{ needs.build-and-push-docker.outputs.image_tag }}
        run: |
          IMAGE_NAME=$(echo "$IMAGE_FULL" | cut -d':' -f1)
-          IMAGE_TAG=$(echo "$IMAGE_FULL" | cut -d':' -f2)
-
+          IMAGE_TAG=$(echo "$IMAGE_FULL" | cut -d':' -f2-)
          echo "Attempting to delete image: $IMAGE_NAME:$IMAGE_TAG"

          TOKEN=$(curl -s -H "Content-Type: application/json" \
@@ -43,16 +43,16 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false

      - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
        with:
          python-version: '3.12'

      - name: Run pre-commit hooks
-        uses: pre-commit/action@v3.0.1 # zizmor: ignore[unpinned-uses]
+        uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1
        with:
          extra_args: --all-files --show-diff-on-failure --color=always
@@ -38,12 +38,12 @@ jobs:

    steps:
      - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false

      - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
        with:
          python-version: '3.12'

@@ -104,7 +104,7 @@ jobs:
      - name: Publish to TestPyPI for pre-releases
        # True for tags like 'v0.2.0-rc1'
        if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-')
-        uses: pypa/gh-action-pypi-publish@v1.13.0 # zizmor: ignore[unpinned-uses, use-trusted-publishing]
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e  # v1.13.0
        with:
          repository-url: https://test.pypi.org/legacy/
          verbose: true
@@ -112,7 +112,7 @@ jobs:

      - name: Publish to PyPI
        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
-        uses: pypa/gh-action-pypi-publish@v1.13.0 # zizmor: ignore[unpinned-uses, use-trusted-publishing]
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e  # v1.13.0
        with:
          verbose: true
          print-hash: true
@@ -127,7 +127,7 @@ jobs:
    env:
      MUJOCO_GL: egl
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          lfs: true
          persist-credentials: false
@@ -137,7 +137,7 @@ jobs:
          git curl libglib2.0-0 libegl1-mesa-dev ffmpeg libusb-1.0-0-dev \
          speech-dispatcher libgeos-dev portaudio19-dev
      - name: Setup uv and Python
-        uses: astral-sh/setup-uv@v6 # zizmor: ignore[unpinned-uses]
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6
        with:
          enable-cache: true # zizmor: ignore[cache-poisoning]
          version: ${{ env.UV_VERSION }}
@@ -43,12 +43,12 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
-        uses: actions/checkout@v6 # zizmor: ignore[unpinned-uses]
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Secret Scanning
-        uses: trufflesecurity/trufflehog@v3.90.0  # zizmor: ignore[unpinned-uses]
+        uses: trufflesecurity/trufflehog@eafb8c5f6a06175141c27f17bcc17941853d0047  # v3.90.0
        with:
          extra_args: --only-verified
@@ -25,7 +25,6 @@ node_modules/

 # Lock files
 poetry.lock
-uv.lock
 Pipfile.lock

 ### Build & Distribution ###
@@ -4,7 +4,8 @@

 <div align="center">

-[![Tests](https://github.com/huggingface/lerobot/actions/workflows/nightly.yml/badge.svg?branch=main)](https://github.com/huggingface/lerobot/actions/workflows/nightly.yml?query=branch%3Amain)
+[![Tests](https://github.com/huggingface/lerobot/actions/workflows/latest_deps_tests.yml/badge.svg?branch=main)](https://github.com/huggingface/lerobot/actions/workflows/latest_deps_tests.yml?query=branch%3Amain)
+[![Tests](https://github.com/huggingface/lerobot/actions/workflows/docker_publish.yml/badge.svg?branch=main)](https://github.com/huggingface/lerobot/actions/workflows/docker_publish.yml?query=branch%3Amain)
 [![Python versions](https://img.shields.io/pypi/pyversions/lerobot)](https://www.python.org/downloads/)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/huggingface/lerobot/blob/main/LICENSE)
 [![Status](https://img.shields.io/pypi/status/lerobot)](https://pypi.org/project/lerobot/)
@@ -0,0 +1,89 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Isolated benchmark image for LIBERO integration tests.
+# Installs only lerobot[libero] so its dep tree (hf-libero, dm-control, mujoco)
+# cannot conflict with other benchmarks.
+#
+# Build:  docker build -f docker/Dockerfile.benchmark.libero -t lerobot-benchmark-libero .
+# Run:    docker run --gpus all --rm lerobot-benchmark-libero lerobot-eval ...
+
+ARG CUDA_VERSION=12.4.1
+ARG OS_VERSION=22.04
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION}
+
+ARG PYTHON_VERSION=3.12
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    MUJOCO_GL=egl \
+    PATH=/lerobot/.venv/bin:$PATH \
+    CUDA_VISIBLE_DEVICES=0 \
+    DEVICE=cuda
+
+# System deps — same set as Dockerfile.internal
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    software-properties-common build-essential git curl \
+    libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \
+    libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \
+    cmake pkg-config ninja-build \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+       python${PYTHON_VERSION} \
+       python${PYTHON_VERSION}-venv \
+       python${PYTHON_VERSION}-dev \
+    && curl -LsSf https://astral.sh/uv/install.sh | sh \
+    && mv /root/.local/bin/uv /usr/local/bin/uv \
+    && useradd --create-home --shell /bin/bash user_lerobot \
+    && usermod -aG sudo user_lerobot \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /lerobot
+RUN chown -R user_lerobot:user_lerobot /lerobot
+USER user_lerobot
+
+ENV HOME=/home/user_lerobot \
+    HF_HOME=/home/user_lerobot/.cache/huggingface \
+    HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \
+    TORCH_HOME=/home/user_lerobot/.cache/torch \
+    TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton
+
+RUN uv venv --python python${PYTHON_VERSION}
+
+# Install only lerobot[libero] — completely isolated from metaworld's dep tree
+COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
+COPY --chown=user_lerobot:user_lerobot src/ src/
+
+RUN uv sync --locked --extra libero --extra smolvla --no-cache
+
+# Pre-download lerobot/libero-assets from HF Hub so nothing is fetched at
+# runtime (which times out on CI). Point the libero config at the cached path.
+# libero/libero/__init__.py calls input() when ~/.libero/config.yaml is missing,
+# so we write the config before any libero import can happen.
+RUN LIBERO_DIR=$(python${PYTHON_VERSION} -c \
+      "import importlib.util, os; s=importlib.util.find_spec('libero'); \
+       print(os.path.join(os.path.dirname(s.origin), 'libero'))") && \
+    mkdir -p /home/user_lerobot/.libero && \
+    python${PYTHON_VERSION} -c "\
+from huggingface_hub import snapshot_download; \
+snapshot_download(repo_id='lerobot/libero-assets', repo_type='dataset', \
+                  local_dir='/home/user_lerobot/.libero/assets')" && \
+    printf "assets: /home/user_lerobot/.libero/assets\nbddl_files: ${LIBERO_DIR}/bddl_files\ndatasets: ${LIBERO_DIR}/../datasets\ninit_states: ${LIBERO_DIR}/init_files\n" \
+    > /home/user_lerobot/.libero/config.yaml
+
+RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas
+
+COPY --chown=user_lerobot:user_lerobot . .
+
+CMD ["/bin/bash"]
@@ -0,0 +1,74 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Isolated benchmark image for MetaWorld integration tests.
+# Installs only lerobot[metaworld] so its dep tree (metaworld==3.0.0, mujoco>=3)
+# cannot conflict with other benchmarks.
+#
+# Build:  docker build -f docker/Dockerfile.benchmark.metaworld -t lerobot-benchmark-metaworld .
+# Run:    docker run --gpus all --rm lerobot-benchmark-metaworld lerobot-eval ...
+
+ARG CUDA_VERSION=12.4.1
+ARG OS_VERSION=22.04
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION}
+
+ARG PYTHON_VERSION=3.12
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    MUJOCO_GL=egl \
+    PATH=/lerobot/.venv/bin:$PATH \
+    CUDA_VISIBLE_DEVICES=0 \
+    DEVICE=cuda
+
+# System deps — same set as Dockerfile.internal
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    software-properties-common build-essential git curl \
+    libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \
+    libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \
+    cmake pkg-config ninja-build \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+       python${PYTHON_VERSION} \
+       python${PYTHON_VERSION}-venv \
+       python${PYTHON_VERSION}-dev \
+    && curl -LsSf https://astral.sh/uv/install.sh | sh \
+    && mv /root/.local/bin/uv /usr/local/bin/uv \
+    && useradd --create-home --shell /bin/bash user_lerobot \
+    && usermod -aG sudo user_lerobot \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /lerobot
+RUN chown -R user_lerobot:user_lerobot /lerobot
+USER user_lerobot
+
+ENV HOME=/home/user_lerobot \
+    HF_HOME=/home/user_lerobot/.cache/huggingface \
+    HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \
+    TORCH_HOME=/home/user_lerobot/.cache/torch \
+    TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton
+
+RUN uv venv --python python${PYTHON_VERSION}
+
+# Install only lerobot[metaworld] — completely isolated from libero's dep tree
+COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
+COPY --chown=user_lerobot:user_lerobot src/ src/
+
+RUN uv sync --locked --extra metaworld --extra smolvla --no-cache
+
+RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas
+
+COPY --chown=user_lerobot:user_lerobot . .
+
+CMD ["/bin/bash"]
@@ -73,17 +73,10 @@ ENV HOME=/home/user_lerobot \
 RUN uv venv --python python${PYTHON_VERSION}

 # Install Python dependencies for caching
-COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml README.md MANIFEST.in ./
+COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
 COPY --chown=user_lerobot:user_lerobot src/ src/

-ARG UNBOUND_DEPS=false
-
-RUN if [ "$UNBOUND_DEPS" = "true" ]; then \
-    sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml; \
-    echo "Dependencies unbound:" && cat pyproject.toml; \
-    fi
-
-RUN uv pip install --no-cache ".[all]"
+RUN uv sync --locked --extra all --no-cache

 RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas

@@ -61,17 +61,10 @@ ENV HOME=/home/user_lerobot \
 RUN uv venv

 # Install Python dependencies for caching
-COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml README.md MANIFEST.in ./
+COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
 COPY --chown=user_lerobot:user_lerobot src/ src/

-ARG UNBOUND_DEPS=false
-
-RUN if [ "$UNBOUND_DEPS" = "true" ]; then \
-    sed -i 's/,[[:space:]]*<[0-9\.]*//g' pyproject.toml; \
-    echo "Dependencies unbound:" && cat pyproject.toml; \
-    fi
-
-RUN uv pip install --no-cache ".[all]"
+RUN uv sync --locked --extra all --no-cache

 # Copy the rest of the application code
 # Make sure to have the git-LFS files for testing
@@ -0,0 +1,77 @@
+# Docker
+
+This directory contains Dockerfiles for running LeRobot in containerized environments. Both images are **built nightly from `main`** and published to Docker Hub with the full environment pre-baked — no dependency setup required.
+
+## Pre-built Images
+
+```bash
+# CPU-only image (based on Dockerfile.user)
+docker pull huggingface/lerobot-cpu:latest
+
+# GPU image with CUDA support (based on Dockerfile.internal)
+docker pull huggingface/lerobot-gpu:latest
+```
+
+## Quick Start
+
+The fastest way to start training is to pull the GPU image and run `lerobot-train` directly. This is the same environment used for all of our CI, so it is a well-tested, batteries-included setup.
+
+```bash
+docker run -it --rm --gpus all --shm-size 16gb huggingface/lerobot-gpu:latest
+
+# inside the container:
+lerobot-train --policy.type=act --dataset.repo_id=lerobot/aloha_sim_transfer_cube_human
+```
+
+## Dockerfiles
+
+### `Dockerfile.user` (CPU)
+
+A lightweight image based on `python:3.12-slim`. Includes all Python dependencies and system libraries but does not include CUDA — there is no GPU support. Useful for exploring the codebase, running scripts, or working with robots, but not practical for training.
+
+### `Dockerfile.internal` (GPU)
+
+A CUDA-enabled image based on `nvidia/cuda`. This is the image for training — mostly used for internal interactions with the GPU cluster.
+
+## Usage
+
+### Running a pre-built image
+
+```bash
+# CPU
+docker run -it --rm huggingface/lerobot-cpu:latest
+
+# GPU
+docker run -it --rm --gpus all --shm-size 16gb huggingface/lerobot-gpu:latest
+```
+
+### Building locally
+
+From the repo root:
+
+```bash
+# CPU
+docker build -f docker/Dockerfile.user -t lerobot-user .
+docker run -it --rm lerobot-user
+
+# GPU
+docker build -f docker/Dockerfile.internal -t lerobot-internal .
+docker run -it --rm --gpus all --shm-size 16gb lerobot-internal
+```
+
+### Multi-GPU training
+
+To select specific GPUs, set `CUDA_VISIBLE_DEVICES` when launching the container:
+
+```bash
+# Use 4 GPUs
+docker run -it --rm --gpus all --shm-size 16gb \
+  -e CUDA_VISIBLE_DEVICES=0,1,2,3 \
+  huggingface/lerobot-gpu:latest
+```
+
+### USB device access (e.g. robots, cameras)
+
+```bash
+docker run -it --device=/dev/ -v /dev/:/dev/ --rm huggingface/lerobot-cpu:latest
+```
@@ -73,6 +73,8 @@
    title: Control & Train Robots in Sim (LeIsaac)
  title: "Simulation"
 - sections:
+  - local: evaluation
+    title: Evaluation (lerobot-eval)
  - local: adding_benchmarks
    title: Adding a New Benchmark
  - local: libero
@@ -26,7 +26,7 @@ During evaluation, data moves through four stages:
 1. gym.Env  ──→  raw observations (numpy dicts)

 2. Preprocessing  ──→  standard LeRobot keys + task description
-   (preprocess_observation, add_envs_task in envs/utils.py)
+   (preprocess_observation in envs/utils.py, env.call("task_description"))

 3. Processors  ──→  env-specific then policy-specific transforms
   (env_preprocessor, policy_preprocessor)
@@ -122,15 +122,17 @@ Each `EnvConfig` subclass declares two dicts that tell the policy what to expect

 ### Checklist

-| File                                     | Required | Why                                                          |
-| ---------------------------------------- | -------- | ------------------------------------------------------------ |
-| `src/lerobot/envs/<benchmark>.py`        | Yes      | Wraps the simulator as a standard gym.Env                    |
-| `src/lerobot/envs/configs.py`            | Yes      | Registers your benchmark and its `create_envs()` for the CLI |
-| `src/lerobot/processor/env_processor.py` | Optional | Custom observation/action transforms                         |
-| `src/lerobot/envs/utils.py`              | Optional | Only if you need new raw observation keys                    |
-| `pyproject.toml`                         | Yes      | Declares benchmark-specific dependencies                     |
-| `docs/source/<benchmark>.mdx`            | Yes      | User-facing documentation page                               |
-| `docs/source/_toctree.yml`               | Yes      | Adds your page to the docs sidebar                           |
+| File                                      | Required | Why                                                          |
+| ----------------------------------------- | -------- | ------------------------------------------------------------ |
+| `src/lerobot/envs/<benchmark>.py`         | Yes      | Wraps the simulator as a standard gym.Env                    |
+| `src/lerobot/envs/configs.py`             | Yes      | Registers your benchmark and its `create_envs()` for the CLI |
+| `src/lerobot/processor/env_processor.py`  | Optional | Custom observation/action transforms                         |
+| `src/lerobot/envs/utils.py`               | Optional | Only if you need new raw observation keys                    |
+| `pyproject.toml`                          | Yes      | Declares benchmark-specific dependencies                     |
+| `docs/source/<benchmark>.mdx`             | Yes      | User-facing documentation page                               |
+| `docs/source/_toctree.yml`                | Yes      | Adds your page to the docs sidebar                           |
+| `docker/Dockerfile.benchmark.<benchmark>` | Yes      | Isolated Docker image for CI smoke tests                     |
+| `.github/workflows/benchmark_tests.yml`   | Yes      | CI job that builds the image and runs a 1-episode smoke eval |

 ### 1. The gym.Env wrapper (`src/lerobot/envs/<benchmark>.py`)

@@ -161,6 +163,8 @@ class MyBenchmarkEnv(gym.Env):
        ...
 ```

+**GPU-based simulators (e.g. MuJoCo with EGL rendering):** If your simulator allocates GPU/EGL contexts during `__init__`, defer that allocation to a `_ensure_env()` helper called on first `reset()`/`step()`. This avoids inheriting stale GPU handles when `AsyncVectorEnv` spawns worker processes. See `LiberoEnv._ensure_env()` for the pattern.
+
 Also provide a factory function that returns the nested dict structure:

 ```python
@@ -207,7 +211,7 @@ class MyBenchmarkEnvConfig(EnvConfig):
    def gym_kwargs(self) -> dict:
        return {"obs_type": self.obs_type, "render_mode": self.render_mode}

-    def create_envs(self, n_envs: int, use_async_envs: bool = False):
+    def create_envs(self, n_envs: int, use_async_envs: bool = True):
        """Override for multi-task benchmarks or custom env creation."""
        from lerobot.envs.<benchmark> import create_<benchmark>_envs
        return create_<benchmark>_envs(task=self.task, n_envs=n_envs, ...)
@@ -293,14 +297,87 @@ Add your benchmark to the "Benchmarks" section:
  title: "Benchmarks"
 ```

+### 7. CI smoke test (`docker/` + `.github/workflows/benchmark_tests.yml`)
+
+Each benchmark must have an isolated Docker image and a CI job that runs a 1-episode eval. This catches install-time regressions (broken transitive deps, import errors, interactive prompts) before they reach users.
+
+**Create `docker/Dockerfile.benchmark.<benchmark>`** — copy an existing one and change only the extra name:
+
+```dockerfile
+# Isolated benchmark image — installs lerobot[<benchmark>] only.
+# Build: docker build -f docker/Dockerfile.benchmark.<benchmark> -t lerobot-benchmark-<benchmark> .
+ARG CUDA_VERSION=12.4.1
+ARG OS_VERSION=22.04
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION}
+ARG PYTHON_VERSION=3.12
+# ... (same system deps as Dockerfile.benchmark.libero) ...
+RUN uv sync --locked --extra <benchmark> --no-cache
+```
+
+Each benchmark gets its own image so its dependency tree (pinned simulator packages, specific mujoco/scipy versions) cannot conflict with other benchmarks.
+
+**Add a job to `.github/workflows/benchmark_tests.yml`** — copy an existing job block and adjust:
+
+```yaml
+<benchmark>-integration-test:
+  name: <Benchmark> — build image + 1-episode eval
+  runs-on:
+    group: aws-g6-4xlarge-plus
+  env:
+    HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
+  steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        persist-credentials: false
+        lfs: true
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses]
+      with:
+        cache-binary: false
+    - name: Build <Benchmark> image
+      uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses]
+      with:
+        context: .
+        file: docker/Dockerfile.benchmark.<benchmark>
+        push: false
+        load: true
+        tags: lerobot-benchmark-<benchmark>:ci
+        cache-from: type=local,src=/tmp/.buildx-cache-<benchmark>
+        cache-to: type=local,dest=/tmp/.buildx-cache-<benchmark>,mode=max
+    - name: Run <Benchmark> smoke eval (1 episode)
+      run: |
+        docker run --rm --gpus all \
+          --shm-size=4g \
+          -e HF_HOME=/tmp/hf \
+          -e HF_USER_TOKEN="${HF_USER_TOKEN}" \
+          lerobot-benchmark-<benchmark>:ci \
+          bash -c "
+            hf auth login --token \"\$HF_USER_TOKEN\" --add-to-git-credential 2>/dev/null || true
+            lerobot-eval \
+              --policy.path=<hub_policy_path> \
+              --env.type=<benchmark> \
+              --env.task=<task> \
+              --eval.batch_size=1 \
+              --eval.n_episodes=1 \
+              --eval.use_async_envs=false \
+              --policy.device=cuda
+          "
+```
+
+**Tips:**
+
+- If the benchmark library prompts for user input on import (like LIBERO asking for a dataset folder), pass the relevant env var in the `docker run` command (e.g. `-e LIBERO_DATA_FOLDER=/tmp/libero_data`).
+- The job is scoped to only trigger on changes to `src/lerobot/envs/**`, `src/lerobot/scripts/lerobot_eval.py`, and the Dockerfiles — it won't run on unrelated PRs.
+
 ## Verifying your integration

 After completing the steps above, confirm that everything works:

 1. **Install** — `pip install -e ".[mybenchmark]"` and verify the dependency group installs cleanly.
 2. **Smoke test env creation** — call `make_env()` with your config in Python, check that the returned dict has the expected `{suite: {task_id: VectorEnv}}` shape, and that `reset()` returns observations with the right keys.
-3. **Run a full eval** — `lerobot-eval --env.type=<name> --env.task=<task> --eval.n_episodes=1 --eval.batch_size=1 --policy.path=<any_compatible_policy>` to exercise the full pipeline end-to-end.
+3. **Run a full eval** — `lerobot-eval --env.type=<name> --env.task=<task> --eval.n_episodes=1 --policy.path=<any_compatible_policy>` to exercise the full pipeline end-to-end. (`batch_size` defaults to auto-tuning based on CPU cores; pass `--eval.batch_size=1` to force a single environment.)
 4. **Check success detection** — verify that `info["is_success"]` flips to `True` when the task is actually completed. This is what the eval loop uses to compute success rates.
+5. **Add CI smoke test** — follow step 7 above to add a Dockerfile and CI job. This ensures the install stays green as dependencies evolve.

 ## Writing a benchmark doc page

@@ -311,7 +388,7 @@ Each benchmark `.mdx` page should include:
 - **Overview image or GIF.**
 - **Available tasks** — table of task suites with counts and brief descriptions.
 - **Installation** — `pip install -e ".[<benchmark>]"` plus any extra steps (env vars, system packages).
- **Evaluation** — recommended `lerobot-eval` command with `n_episodes` and `batch_size` for reproducible results. Include single-task and multi-task examples if applicable.
+- **Evaluation** — recommended `lerobot-eval` command with `n_episodes` for reproducible results. `batch_size` defaults to auto; only specify it if needed. Include single-task and multi-task examples if applicable. See the [Evaluation guide](evaluation) for details.
 - **Policy inputs and outputs** — observation keys with shapes, action space description.
 - **Recommended evaluation episodes** — how many episodes per task is standard.
 - **Training** — example `lerobot-train` command.
@@ -151,7 +151,7 @@ observation = {

 ### Factory Function

-The `make_env_pre_post_processors` function delegates to `env_cfg.get_env_processors()`:
+The `make_env_pre_post_processors` function follows the same pattern as `make_pre_post_processors` for policies:

 ```python
 from lerobot.envs.factory import make_env_pre_post_processors
@@ -159,30 +159,46 @@ from lerobot.envs.configs import LiberoEnv, PushtEnv

 # For LIBERO: Returns LiberoProcessorStep in preprocessor
 libero_cfg = LiberoEnv(task="libero_spatial", camera_name=["agentview"])
-env_preprocessor, env_postprocessor = make_env_pre_post_processors(libero_cfg, policy_cfg)
+env_preprocessor, env_postprocessor = make_env_pre_post_processors(libero_cfg)

 # For other environments: Returns identity processors (no-op)
 pusht_cfg = PushtEnv()
-env_preprocessor, env_postprocessor = make_env_pre_post_processors(pusht_cfg, policy_cfg)
+env_preprocessor, env_postprocessor = make_env_pre_post_processors(pusht_cfg)
 ```

-### How It Works
-
-Each `EnvConfig` subclass can override `get_env_processors()` to return benchmark-specific
-processor pipelines. The base class returns identity (no-op) processors by default.
+### Implementation in `envs/factory.py`

 ```python
-# In your EnvConfig subclass:
-def get_env_processors(self):
-    from lerobot.processor.pipeline import PolicyProcessorPipeline
-    return (
-        PolicyProcessorPipeline(steps=[MyProcessorStep()]),
-        PolicyProcessorPipeline(steps=[]),
-    )
-```
+def make_env_pre_post_processors(
+    env_cfg: EnvConfig,
+) -> tuple[
+    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
+    PolicyProcessorPipeline[dict[str, Any], dict[str, Any]],
+]:
+    """
+    Create preprocessor and postprocessor pipelines for environment observations.

-The factory function `make_env_pre_post_processors` simply delegates to this method,
-with a special case for `XVLAConfig` policies which override the env processors entirely.
+    Args:
+        env_cfg: The configuration of the environment.
+
+    Returns:
+        A tuple containing:
+            - preprocessor: Pipeline that processes environment observations
+            - postprocessor: Pipeline that processes environment outputs
+    """
+    # For LIBERO environments, add the LiberoProcessorStep to preprocessor
+    if isinstance(env_cfg, LiberoEnv) or "libero" in env_cfg.type:
+        preprocessor = PolicyProcessorPipeline(steps=[LiberoProcessorStep()])
+    else:
+        # For all other environments, return an identity preprocessor
+        preprocessor = PolicyProcessorPipeline(steps=[])
+
+    # Postprocessor is currently identity for all environments
+    # Future: Could add environment-specific action transformations
+    postprocessor = PolicyProcessorPipeline(steps=[])
+
+    return preprocessor, postprocessor
+```

 ### Integration in Evaluation

@@ -0,0 +1,162 @@
+# Evaluation
+
+`lerobot-eval` runs a trained policy on a simulation benchmark and reports success rate, reward, and (optionally) episode videos. It handles environment creation, batched rollouts, and metric aggregation automatically.
+
+## Quick start
+
+Evaluate a Hub-hosted policy on LIBERO:
+
+```bash
+lerobot-eval \
+    --policy.path=pepijn223/smolvla_libero \
+    --env.type=libero \
+    --env.task=libero_spatial \
+    --eval.n_episodes=10 \
+    --policy.device=cuda
+```
+
+Evaluate a local checkpoint:
+
+```bash
+lerobot-eval \
+    --policy.path=outputs/train/act_pusht/checkpoints/005000/pretrained_model \
+    --env.type=pusht \
+    --eval.n_episodes=10
+```
+
+`batch_size` defaults to **auto** (based on CPU cores). The script picks the right number of parallel environments for your machine.
+
+## Key flags
+
+| Flag                    | Default        | Description                                                                           |
+| ----------------------- | -------------- | ------------------------------------------------------------------------------------- |
+| `--policy.path`         | required       | Hub repo ID or local path to a pretrained model                                       |
+| `--env.type`            | required       | Benchmark name (`pusht`, `libero`, `metaworld`, etc.)                                 |
+| `--env.task`            | varies         | Task or suite name (e.g. `libero_spatial`, `libero_10`)                               |
+| `--eval.n_episodes`     | `50`           | Total episodes to run (across all tasks)                                              |
+| `--eval.batch_size`     | `0` (auto)     | Number of parallel environments. `0` = auto-tune from CPU cores                       |
+| `--eval.use_async_envs` | `true`         | Use `AsyncVectorEnv` (parallel stepping). Auto-downgrades to sync when `batch_size=1` |
+| `--policy.device`       | `cuda`         | Inference device                                                                      |
+| `--policy.use_amp`      | `false`        | Mixed-precision inference (saves VRAM, faster on Ampere+)                             |
+| `--seed`                | `1000`         | Random seed for reproducibility                                                       |
+| `--output_dir`          | auto-generated | Where to write results and videos                                                     |
+
+### Environment-specific flags
+
+Some benchmarks accept additional flags through `--env.*`:
+
+```bash
+# LIBERO: map simulator camera names to policy feature names
+--env.camera_name_mapping='{"agentview_image": "camera1", "robot0_eye_in_hand_image": "camera2"}'
+
+# Fill unused camera slots with zeros
+--policy.empty_cameras=1
+```
+
+See each benchmark's documentation ([LIBERO](libero), [Meta-World](metaworld)) for benchmark-specific flags.
+
+## How batch_size works
+
+`batch_size` controls how many environments run in parallel within a single `VectorEnv`:
+
+| `batch_size`  | Behavior                                                             |
+| ------------- | -------------------------------------------------------------------- |
+| `0` (default) | Auto-tune: `floor(cpu_cores × 0.7)`, capped by `n_episodes` and `64` |
+| `1`           | Single environment, synchronous. Useful for debugging                |
+| `N`           | N environments step in parallel via `AsyncVectorEnv`                 |
+
+When `batch_size > 1` and `use_async_envs=true`, each environment runs in its own subprocess via Gymnasium's `AsyncVectorEnv`. This parallelizes the simulation stepping (the main bottleneck), while the policy runs a single batched forward pass on GPU.
+
+**Example:** On a 16-core machine with `n_episodes=100`:
+
+- Auto batch_size = `floor(16 × 0.7)` = `11`
+- 11 environments step simultaneously → ~11× faster than sequential
+
+## Performance
+
+### AsyncVectorEnv (default)
+
+`AsyncVectorEnv` spawns one subprocess per environment. Each subprocess has its own simulator instance. While the policy computes actions on GPU, all environments step in parallel on CPU:
+
+```
+GPU:  [inference]....[inference]....[inference]....
+CPU:  [step × N]....................[step × N]......
+      ↑ parallel                   ↑ parallel
+```
+
+For GPU-based simulators (LIBERO, Meta-World), the environments use **lazy initialization**: the GPU/EGL context is created inside the worker subprocess on first `reset()`, not in the parent process. This avoids `EGL_BAD_CONTEXT` crashes from inheriting stale GPU handles across `fork()`.
+
+### Lazy task loading
+
+For multi-task benchmarks (e.g. LIBERO with 10 tasks), environments are wrapped in `_LazyAsyncVectorEnv` which defers worker creation until the task is actually evaluated. This keeps peak process count = `batch_size` instead of `n_tasks × batch_size`. After each task completes, workers are closed to free resources.
+
+### Tuning for speed
+
+| Situation                      | Recommendation                                        |
+| ------------------------------ | ----------------------------------------------------- |
+| Slow eval, low GPU utilization | Increase `batch_size` (or leave at auto)              |
+| Out of memory (system RAM)     | Decrease `batch_size`                                 |
+| Out of GPU memory              | Decrease `batch_size`, or use `--policy.use_amp=true` |
+| Debugging / single-stepping    | `--eval.batch_size=1 --eval.use_async_envs=false`     |
+
+## Output
+
+Results are written to `output_dir` (default: `outputs/eval/<date>/<time>_<job_name>/`):
+
+- `eval_info.json` — full metrics: per-episode, per-task, per-group, and overall aggregates
+- `videos/` — episode recordings (when `--eval.n_episodes_to_render > 0`)
+
+### Metrics
+
+| Metric           | Description                                                          |
+| ---------------- | -------------------------------------------------------------------- |
+| `pc_success`     | Success rate (%). Based on `info["is_success"]` from the environment |
+| `avg_sum_reward` | Mean cumulative reward per episode                                   |
+| `avg_max_reward` | Mean peak reward per episode                                         |
+| `n_episodes`     | Total episodes evaluated                                             |
+| `eval_s`         | Total wall-clock time                                                |
+| `eval_ep_s`      | Mean wall-clock time per episode                                     |
+
+## Multi-task evaluation
+
+For benchmarks with multiple tasks (LIBERO suites, Meta-World MT50), `lerobot-eval` automatically:
+
+1. Creates environments for all tasks in the selected suite(s)
+2. Evaluates each task sequentially (one task's workers at a time)
+3. Aggregates metrics per-task, per-group (suite), and overall
+
+```bash
+# Evaluate all 10 tasks in libero_spatial
+lerobot-eval \
+    --policy.path=pepijn223/smolvla_libero \
+    --env.type=libero \
+    --env.task=libero_spatial \
+    --eval.n_episodes=10
+
+# Evaluate multiple suites
+lerobot-eval \
+    --policy.path=pepijn223/smolvla_libero \
+    --env.type=libero \
+    --env.task="libero_spatial,libero_object" \
+    --eval.n_episodes=10
+```
+
+## API usage
+
+You can call the eval functions directly from Python:
+
+```python
+from lerobot.envs.factory import make_env
+from lerobot.policies.factory import make_policy
+from lerobot.scripts.lerobot_eval import eval_policy
+
+envs = make_env(env_cfg, n_envs=10)
+policy = make_policy(cfg=policy_cfg, env_cfg=env_cfg)
+
+metrics = eval_policy(
+    env=envs["libero_spatial"][0],
+    policy=policy,
+    n_episodes=10,
+)
+print(metrics["pc_success"])
+```
@@ -131,4 +131,4 @@ lerobot-record \

 ## License

-This model follows the **Apache 2.0 License**, consistent with the original [GR00T repository](https://github.com/NVIDIA/Isaac-GR00T).
+This model follows NVIDIA's proprietary license, consistent with the original [GR00T repository](https://github.com/NVIDIA/Isaac-GR00T). Future versions (starting from N1.7) will follow **Apache 2.0 License**.
@@ -1,6 +1,6 @@
 # Installation

-This guide uses `conda` (via miniforge) to manage environments (recommended). If you prefer another environment manager (e.g. `uv`, `venv`), ensure you have Python >=3.12 and `ffmpeg` installed with the `libsvtav1` encoder, then skip ahead to [Environment Setup](#step-2-environment-setup).
+This guide uses `conda` (via miniforge) to manage environments (recommended). If you prefer another environment manager (e.g. `uv`, `venv`), ensure you have Python >=3.12 and support PyTorch >= 2.10, then skip ahead to [Environment Setup](#step-2-environment-setup).

 ## Step 1 (`conda` only): Install [`miniforge`](https://conda-forge.org/download/)

@@ -20,7 +20,7 @@ Create a virtual environment with Python 3.12:
 conda create -y -n lerobot python=3.12
 ```
 </hfoption>
-<hfoption id="uv">
+<hfoption id="uv (PyTorch >= 2.10 only)">
 ```bash
 uv python install 3.12
 uv venv --python 3.12
@@ -32,48 +32,87 @@ uv venv --python 3.12
 Then activate your virtual environment, you have to do this each time you open a shell to use lerobot:

 <!-- prettier-ignore-start -->
+
 <hfoptions id="activate_venv">
-<hfoption id="conda">```bash
+<hfoption id="conda">
+```bash
 conda activate lerobot
-```</hfoption>
-<hfoption id="uv">
-```bash
-# Linux/macOSsource
-source .venv/bin/activate
-# Windows PowerShell
-source .venv\Scripts\Activate.ps1
 ```
-</hfoption>
-</hfoptions>
-<!-- prettier-ignore-end -->
-
-When using `conda`, install `ffmpeg` in your environment:
-
-```bash
-conda install ffmpeg -c conda-forge
-ffmpeg -version  # ffmpeg 8.X is not yet supported !
-```
-
-> [!TIP]
-> This usually installs `ffmpeg 7.X` for your platform compiled with the `libsvtav1` encoder. If `libsvtav1` is not supported (check supported encoders with `ffmpeg -encoders`), you can:
->
-> - _[On any platform]_ Explicitly install `ffmpeg 7.X` using:
->
-> ```bash
-> conda install ffmpeg=7.1.1 -c conda-forge
-> ```
->
-> - _[On Linux only]_ If you want to bring your own ffmpeg: Install [ffmpeg build dependencies](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu#GettheDependencies) and [compile ffmpeg from source with libsvtav1](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu#libsvtav1), and make sure you use the corresponding ffmpeg binary to your install with `which ffmpeg`.

 > [!NOTE]
-> When installing LeRobot inside WSL (Windows Subsystem for Linux), make sure to install `evdev` with the following command:
+> When installing LeRobot inside WSL (Windows Subsystem for Linux), make sure to also install `evdev`:
 >
 > ```bash
 > conda install evdev -c conda-forge
 > ```

+</hfoption>
+<hfoption id="uv (PyTorch >= 2.10 only)">
+```bash
+# Linux/macOS
+source .venv/bin/activate
+# Windows PowerShell
+.venv\Scripts\activate
+```
+
+> [!NOTE]
+> When installing LeRobot inside WSL (Windows Subsystem for Linux), make sure to also install `evdev`:
+>
+> ```bash
+> sudo apt install libevdev-dev
+> uv pip install evdev
+> ```
+
+</hfoption>
+</hfoptions>
+<!-- prettier-ignore-end -->
+
+### Install `ffmpeg` (for video decoding)
+
+LeRobot uses [TorchCodec](https://github.com/meta-pytorch/torchcodec) for video decoding by default, which requires `ffmpeg`.
+
+> [!NOTE]
+> **Platform support:** TorchCodec is **not available** on macOS Intel (x86_64), Linux ARM (aarch64, arm64, armv7l), or Windows with PyTorch < 2.8. On these platforms, LeRobot automatically falls back to `pyav` — so you do not need to install `ffmpeg` and can skip to Step 3.
+
+If your platform supports TorchCodec, install `ffmpeg` using one of the methods below:
+
+<!-- prettier-ignore-start -->
+
+<hfoptions id="install_ffmpeg">
+<hfoption id="conda (any PyTorch version)">
+
+Install `ffmpeg` in your conda environment. This works with **all PyTorch versions** and is **required for PyTorch < 2.10**:
+
+```bash
+conda install ffmpeg -c conda-forge
+```
+
+> [!TIP]
+> This usually installs `ffmpeg 8.X` with the `libsvtav1` encoder. If you run into issues (e.g. `libsvtav1` missing — check with `ffmpeg -encoders` — or a version mismatch with `torchcodec`), you can explicitly install `ffmpeg 7.1.1` using:
+>
+> ```bash
+> conda install ffmpeg=7.1.1 -c conda-forge
+> ```
+
+</hfoption>
+<hfoption id="uv (PyTorch >= 2.10 only)">
+
+Starting with **PyTorch >= 2.10** (TorchCodec ≥ 0.10), TorchCodec can dynamically link to a system-wide `ffmpeg` installation. This is useful when using `uv` or other non-`conda` environment managers:
+
+```bash
+# Ubuntu/Debian
+sudo apt install ffmpeg
+
+# macOS (Apple Silicon)
+brew install ffmpeg
+```
+
 > [!IMPORTANT]
-> If you are using `uv` you will have to install `ffmpeg` system-wide (outside of the virtual environment). You rely on `uv` and `torchcodec` ability to dynamically link to the system `ffmpeg`.
+> System-wide `ffmpeg` is **only supported with PyTorch >= 2.10** (TorchCodec ≥ 0.10). For older PyTorch versions, you **must** use `conda install ffmpeg -c conda-forge` instead.
+
+</hfoption>
+</hfoptions>
+<!-- prettier-ignore-end -->

 ## Step 3: Install LeRobot 🤗

@@ -331,6 +331,54 @@ lerobot-train \
  --wandb.project=multitask_dit
 ```

+## Libero Results
+
+```
+python -m lerobot.scripts.lerobot_train \
+  --dataset.repo_id=HuggingFaceVLA/libero \
+  --policy.type=multi_task_dit \
+  --policy.push_to_hub=false \
+  --output_dir="./outputs/multitask_dit_libero" \
+  --job_name="multitask-dit-libero" \
+  --wandb.enable=true \
+  --wandb.project=multitask_dit_libero \
+  --dataset.image_transforms.enable=true \
+  --dataset.image_transforms.max_num_transforms=4 \
+  --dataset.image_transforms.tfs='{"brightness":{"type":"ColorJitter","kwargs":{"brightness":[0.75,1.25]}},"contrast":{"type":"ColorJitter","kwargs":{"contrast":[0.6,1.4]}},"saturation":{"type":"ColorJitter","kwargs":{"saturation":[0.8,1.2]}},"hue":{"type":"ColorJitter","kwargs":{"hue":[-0.05,0.05]}},"sharpness":{"type":"SharpnessJitter","kwargs":{"sharpness":[0.6,1.4]}},"rotation":{"type":"RandomRotation","kwargs":{"degrees":[-5,5]}},"translation":{"type":"RandomAffine","kwargs":{"degrees":0,"translate":[0.1,0.1]}}}' \
+  --dataset.video_backend=torchcodec \
+  --policy.use_amp=true \
+  --policy.horizon=48 \
+  --policy.n_obs_steps=2 \
+  --policy.use_rope=true \
+  --policy.use_positional_encoding=false \
+  --policy.hidden_dim=768 \
+  --policy.num_layers=8 \
+  --policy.num_heads=12 \
+  --policy.dropout=0.1 \
+  --policy.timestep_embed_dim=256 \
+  --policy.objective=diffusion \
+  --policy.optimizer_lr=3e-4 \
+  --policy.optimizer_weight_decay=0 \
+  --policy.scheduler_warmup_steps=0 \
+  --policy.vision_encoder_name=openai/clip-vit-base-patch16 \
+  --policy.image_resize_shape=[256,256] \
+  --policy.image_crop_is_random=true \
+  --policy.text_encoder_name=openai/clip-vit-base-patch16 \
+  --policy.vision_encoder_lr_multiplier=0.1 \
+  --policy.device=cuda \
+  --num_workers=8 \
+  --save_freq=4000 \
+  --log_freq=100 \
+  --steps=100000 \
+  --batch_size=320
+```
+
+Results:
+
+| LIBERO Spatial | LIBERO Object | LIBERO Goal | LIBERO 10 | Average |
+| -------------- | ------------- | ----------- | --------- | ------- |
+| 87.0           | 98.2          | 93.8        | 83.2      | 90.6    |
+
 ## References

 For more details on the technical implementation and architecture, see:
@@ -0,0 +1,91 @@
+# π₀.₅ (pi05)
+
+This repository contains the Hugging Face port of **π₀.₅**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
+It is designed as a **Vision-Language-Action model with open-world generalization**.
+
+---
+
+## Model Overview
+
+| Feature              | π₀                                                     | π₀.₅                                      |
+| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
+| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
+| AdaRMS               | Not used                                               | Used in action expert                     |
+| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
+| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
+| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
+
+---
+
+## Relative Actions
+
+π₀.₅ supports training with **relative actions**, where the model learns relative offsets
+from the current robot state instead of absolute joint positions. This mirrors the
+relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
+
+### How it works
+
+1. **During preprocessing**, absolute actions are converted to relative offsets:
+   `relative = action - state` (for selected joints).
+2. The relative actions are normalized using statistics computed from the relative distribution.
+3. **During postprocessing**, predicted relative actions are converted back to absolute:
+   `absolute = relative + state`.
+
+Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
+
+### Configuration
+
+| Parameter                 | Type        | Default       | Description                                                      |
+| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
+| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
+| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
+| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
+
+### Training example
+
+```bash
+python -m lerobot.scripts.lerobot_train \
+  --policy.type=pi05 \
+  --dataset.repo_id=your_org/your_dataset \
+  --policy.use_relative_actions=true \
+  --policy.relative_exclude_joints='["gripper"]'
+```
+
+When `use_relative_actions=true`, the training script automatically:
+
+- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
+- Replaces the standard action stats with relative stats for normalization
+- Broadcasts these stats across all ranks in distributed training
+
+---
+
+## Citation
+
+If you use this work, please cite both **OpenPI** and the π₀.₅ paper:
+
+```bibtex
+@misc{openpi2024,
+  author       = {Physical Intelligence Lab},
+  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
+  year         = {2024},
+  publisher    = {GitHub},
+  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
+  license      = {Apache-2.0}
+}
+
+@misc{intelligence2025pi05visionlanguageactionmodelopenworld,
+  title        = {π₀.₅: a Vision-Language-Action Model with Open-World Generalization},
+  author       = {Physical Intelligence and Kevin Black and Noah Brown and James Darpinian and Karan Dhabalia and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Manuel Y. Galliker and Dibya Ghosh and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Devin LeBlanc and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Allen Z. Ren and Lucy Xiaoyang Shi and Laura Smith and Jost Tobias Springenberg and Kyle Stachowicz and James Tanner and Quan Vuong and Homer Walke and Anna Walling and Haohuan Wang and Lili Yu and Ury Zhilinsky},
+  year         = {2025},
+  eprint       = {2504.16054},
+  archivePrefix= {arXiv},
+  primaryClass = {cs.LG},
+  url          = {https://arxiv.org/abs/2504.16054},
+}
+```
+
+---
+
+## License
+
+This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1,108 @@
+# π₀ (pi0)
+
+This repository contains the Hugging Face port of **π₀**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
+It is designed as a **Vision-Language-Action model for general robot control**.
+
+---
+
+## Model Overview
+
+| Feature              | π₀                                                     | π₀.₅                                      |
+| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
+| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
+| AdaRMS               | Not used                                               | Used in action expert                     |
+| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
+| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
+| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
+
+---
+
+## Relative Actions
+
+π₀ supports training with **relative actions**, where the model learns relative offsets
+from the current robot state instead of absolute joint positions. This mirrors the
+relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
+
+### How it works
+
+1. **During preprocessing**, absolute actions are converted to relative offsets:
+   `relative = action - state` (for selected joints).
+2. The relative actions are normalized using statistics computed from the relative distribution.
+3. **During postprocessing**, predicted relative actions are converted back to absolute:
+   `absolute = relative + state`.
+
+Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
+
+### Configuration
+
+| Parameter                 | Type        | Default       | Description                                                      |
+| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
+| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
+| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
+| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
+
+### Training example
+
+```bash
+python -m lerobot.scripts.lerobot_train \
+  --policy.type=pi0 \
+  --dataset.repo_id=your_org/your_dataset \
+  --policy.use_relative_actions=true \
+  --policy.relative_exclude_joints='["gripper"]'
+```
+
+When `use_relative_actions=true`, the training script automatically:
+
+- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
+- Replaces the standard action stats with relative stats for normalization
+- Broadcasts these stats across all ranks in distributed training
+
+### Recomputing stats for an existing dataset
+
+If you want to precompute relative action stats offline, use `recompute_stats` from
+`lerobot.datasets.dataset_tools`:
+
+```python
+from lerobot.datasets.lerobot_dataset import LeRobotDataset
+from lerobot.datasets.dataset_tools import recompute_stats
+
+dataset = LeRobotDataset("your_org/your_dataset")
+dataset = recompute_stats(
+    dataset,
+    relative_action=True,
+    relative_exclude_joints=["gripper"],
+)
+```
+
+---
+
+## Citation
+
+If you use this work, please cite both **OpenPI** and the π₀ paper:
+
+```bibtex
+@misc{openpi2024,
+  author       = {Physical Intelligence Lab},
+  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
+  year         = {2024},
+  publisher    = {GitHub},
+  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
+  license      = {Apache-2.0}
+}
+
+@misc{black2024pi0visionlanguageactionflowmodel,
+  title        = {π₀: A Vision-Language-Action Flow Model for General Robot Control},
+  author       = {Kevin Black and Noah Brown and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Lucy Xiaoyang Shi and James Tanner and Quan Vuong and Anna Walling and Haohuan Wang and Ury Zhilinsky},
+  year         = {2024},
+  eprint       = {2410.24164},
+  archivePrefix= {arXiv},
+  primaryClass = {cs.LG},
+  url          = {https://arxiv.org/abs/2410.24164},
+}
+```
+
+---
+
+## License
+
+This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1,38 @@
+# Real-Time Chunking (RTC)
+
+This module contains the LeRobot implementation of **Real-Time Chunking (RTC)**, an inference-time technique for flow-matching based policies.
+
+**Note**: RTC is not a policy itself, but rather an inference enhancement that works with flow-matching based policies including [π₀](../pi0/), [π₀.₅](../pi05/), and [SmolVLA](../smolvla/).
+
+---
+
+## Citation
+
+If you use Real-Time Chunking in your work, please cite:
+
+```bibtex
+@misc{openpi2024,
+  author       = {Physical Intelligence Lab},
+  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
+  year         = {2024},
+  publisher    = {GitHub},
+  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
+  license      = {Apache-2.0}
+}
+
+@misc{black2025realtimeexecutionactionchunking,
+      title={Real-Time Execution of Action Chunking Flow Policies},
+      author={Kevin Black and Manuel Y. Galliker and Sergey Levine},
+      year={2025},
+      eprint={2506.07339},
+      archivePrefix={arXiv},
+      primaryClass={cs.RO},
+      url={https://arxiv.org/abs/2506.07339},
+}
+```
+
+---
+
+## License
+
+This implementation follows the **Apache 2.0 License**, consistent with the LeRobot project.
@@ -0,0 +1,14 @@
+## Paper
+
+https://arxiv.org/abs/2509.25358
+
+## Citation
+
+```bibtex
+@article{chen2025sarm,
+  title={SARM: Stage-Aware Reward Modeling for Long Horizon Robot Manipulation},
+  author={Chen, Qianzhong and Yu, Justin and Schwager, Mac and Abbeel, Pieter and Shentu, Yide and Wu, Philipp},
+  journal={arXiv preprint arXiv:2509.25358},
+  year={2025}
+}
+```
@@ -25,7 +25,7 @@ discord = "https://discord.gg/s3KuuzsPFb"

 [project]
 name = "lerobot"
-version = "0.5.1"
+version = "0.5.2"
 description = "🤗 LeRobot: State-of-the-art Machine Learning for Real-World Robotics in Pytorch"
 dynamic = ["readme"]
 license = { text = "Apache-2.0" }
@@ -71,9 +71,9 @@ dependencies = [
    "cmake>=3.29.0.1,<4.2.0",
    "packaging>=24.2,<26.0",

-    "torch>=2.2.1,<2.11.0",
-    "torchcodec>=0.2.1,<0.11.0; sys_platform != 'win32' and (sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')) and (sys_platform != 'darwin' or platform_machine != 'x86_64')",
-    "torchvision>=0.21.0,<0.26.0",
+    "torch>=2.7,<2.11.0",
+    "torchcodec>=0.3.0,<0.11.0; sys_platform != 'win32' and (sys_platform != 'linux' or (platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')) and (sys_platform != 'darwin' or platform_machine != 'x86_64')", # NOTE: Windows support starts at version 0.7 (needs torch==2.8), ffmpeg>=8 support starts at version 0.8.1 (needs torch==2.9), system-wide ffmpeg support starts at version 0.10 (needs torch==2.10).
+    "torchvision>=0.22.0,<0.26.0",

    "einops>=0.8.0,<0.9.0",
    "opencv-python-headless>=4.9.0,<4.14.0",
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Extract natural-language task descriptions for a benchmark suite.
+
+Runs inside the benchmark Docker container (where the env library is installed)
+immediately after lerobot-eval, writing a JSON file that parse_eval_metrics.py
+picks up and embeds in metrics.json.
+
+Output format: {"<suite>_<task_idx>": "<nl instruction>", ...}
+
+Usage:
+    python scripts/ci/extract_task_descriptions.py \\
+        --env libero --task libero_spatial \\
+        --output /tmp/eval-artifacts/task_descriptions.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def _libero_descriptions(task_suite: str) -> dict[str, str]:
+    from libero.libero import benchmark  # type: ignore[import-untyped]
+
+    suite_dict = benchmark.get_benchmark_dict()
+    if task_suite not in suite_dict:
+        print(
+            f"[extract_task_descriptions] Unknown LIBERO suite '{task_suite}'. "
+            f"Available: {list(suite_dict.keys())}",
+            file=sys.stderr,
+        )
+        return {}
+    suite = suite_dict[task_suite]()
+    return {f"{task_suite}_{i}": suite.get_task(i).language for i in range(suite.n_tasks)}
+
+
+def _metaworld_descriptions(task_name: str) -> dict[str, str]:
+    # MetaWorld tasks don't expose a separate NL description attribute;
+    # use a cleaned version of the task name as the description.
+    label = task_name.removeprefix("metaworld-").replace("-", " ").strip()
+    return {f"{task_name}_0": label}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--env", required=True, help="Environment family (libero, metaworld, ...)")
+    parser.add_argument("--task", required=True, help="Task/suite name (e.g. libero_spatial)")
+    parser.add_argument("--output", required=True, help="Path to write task_descriptions.json")
+    args = parser.parse_args()
+
+    descriptions: dict[str, str] = {}
+    try:
+        if args.env == "libero":
+            descriptions = _libero_descriptions(args.task)
+        elif args.env == "metaworld":
+            descriptions = _metaworld_descriptions(args.task)
+        else:
+            print(
+                f"[extract_task_descriptions] No description extractor for env '{args.env}'.",
+                file=sys.stderr,
+            )
+    except Exception as exc:
+        print(f"[extract_task_descriptions] Warning: {exc}", file=sys.stderr)
+
+    out_path = Path(args.output)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(descriptions, indent=2))
+    print(f"[extract_task_descriptions] {len(descriptions)} descriptions → {out_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Parse lerobot-eval output into a small metrics.json artifact.
+
+Reads eval_info.json written by lerobot-eval --output_dir and extracts the
+key metrics needed by the health dashboard. Handles both single-task and
+multi-task eval output formats.
+
+Usage:
+    python scripts/ci/parse_eval_metrics.py \\
+        --artifacts-dir /tmp/libero-artifacts \\
+        --env libero \\
+        --task libero_spatial \\
+        --policy pepijn223/smolvla_libero
+
+Writes <artifacts-dir>/metrics.json. The CI workflow then uploads this file
+as a GitHub Actions artifact named "<env>-metrics".
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+
+def _extract_metrics(info: dict) -> tuple[float | None, int | None, float | None, float | None]:
+    """Extract (pc_success, n_episodes, avg_sum_reward, eval_s) from eval_info.json.
+
+    Handles two output shapes:
+      - Single-task: {"aggregated": {"pc_success": 80.0, ...}}
+      - Multi-task:  {"overall": {"pc_success": 80.0, "n_episodes": 5, ...}}
+    """
+    for key in ("aggregated", "overall"):
+        if key not in info:
+            continue
+        agg = info[key]
+        pc = agg.get("pc_success")
+        n = agg.get("n_episodes")
+        reward = agg.get("avg_sum_reward")
+        eval_s = agg.get("eval_s")
+        if pc is not None and not math.isnan(pc):
+            return (
+                float(pc),
+                int(n) if n is not None else None,
+                float(reward) if reward is not None else None,
+                float(eval_s) if eval_s is not None else None,
+            )
+
+    return None, None, None, None
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--artifacts-dir", required=True, help="Path to the mounted artifacts volume")
+    parser.add_argument("--env", required=True, help="Environment name (e.g. libero)")
+    parser.add_argument("--task", required=True, help="Task name (e.g. libero_spatial)")
+    parser.add_argument("--policy", required=True, help="Policy hub path (e.g. pepijn223/smolvla_libero)")
+    args = parser.parse_args()
+
+    artifacts_dir = Path(args.artifacts_dir)
+    eval_info_path = artifacts_dir / "eval_info.json"
+
+    pc_success: float | None = None
+    n_episodes: int | None = None
+    avg_sum_reward: float | None = None
+    eval_s: float | None = None
+
+    if eval_info_path.exists():
+        try:
+            info = json.loads(eval_info_path.read_text())
+            pc_success, n_episodes, avg_sum_reward, eval_s = _extract_metrics(info)
+        except (json.JSONDecodeError, KeyError, TypeError) as exc:
+            print(f"[parse_eval_metrics] Warning: could not parse eval_info.json: {exc}", file=sys.stderr)
+    else:
+        print(
+            f"[parse_eval_metrics] Warning: {eval_info_path} not found — eval may have failed.",
+            file=sys.stderr,
+        )
+
+    task_descriptions: dict[str, str] = {}
+    task_desc_path = artifacts_dir / "task_descriptions.json"
+    if task_desc_path.exists():
+        try:
+            task_descriptions = json.loads(task_desc_path.read_text())
+        except json.JSONDecodeError as exc:
+            print(
+                f"[parse_eval_metrics] Warning: could not parse task_descriptions.json: {exc}",
+                file=sys.stderr,
+            )
+
+    metrics = {
+        "env": args.env,
+        "task": args.task,
+        "policy": args.policy,
+        "pc_success": pc_success,
+        "n_episodes": n_episodes,
+        "avg_sum_reward": avg_sum_reward,
+        "eval_s": eval_s,
+        "task_descriptions": task_descriptions,
+    }
+
+    out_path = artifacts_dir / "metrics.json"
+    out_path.write_text(json.dumps(metrics, indent=2))
+    print(f"[parse_eval_metrics] Written: {out_path}")
+    print(json.dumps(metrics, indent=2))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -65,20 +65,27 @@ class WandBConfig:
 class EvalConfig:
    n_episodes: int = 50
    # `batch_size` specifies the number of environments to use in a gym.vector.VectorEnv.
-    batch_size: int = 50
+    # Set to 0 for auto-tuning based on available CPU cores and n_episodes.
+    batch_size: int = 0
    # `use_async_envs` specifies whether to use asynchronous environments (multiprocessing).
-    use_async_envs: bool = False
+    # Defaults to True; automatically downgraded to SyncVectorEnv when batch_size=1.
+    use_async_envs: bool = True

    def __post_init__(self) -> None:
+        if self.batch_size == 0:
+            self.batch_size = self._auto_batch_size()
        if self.batch_size > self.n_episodes:
-            raise ValueError(
-                "The eval batch size is greater than the number of eval episodes "
-                f"({self.batch_size} > {self.n_episodes}). As a result, {self.batch_size} "
-                f"eval environments will be instantiated, but only {self.n_episodes} will be used. "
-                "This might significantly slow down evaluation. To fix this, you should update your command "
-                f"to increase the number of episodes to match the batch size (e.g. `eval.n_episodes={self.batch_size}`), "
-                f"or lower the batch size (e.g. `eval.batch_size={self.n_episodes}`)."
-            )
+            self.batch_size = self.n_episodes
+
+    def _auto_batch_size(self) -> int:
+        """Pick batch_size based on CPU cores, capped by n_episodes."""
+        import math
+        import os
+
+        cpu_cores = os.cpu_count() or 4
+        # Each async env worker needs ~1 core; leave headroom for main process + inference.
+        by_cpu = max(1, math.floor(cpu_cores * 0.7))
+        return min(by_cpu, self.n_episodes, 64)


@dataclass
@@ -151,9 +151,11 @@ class LeRobotDataset(torch.utils.data.Dataset):
                ``$HF_LEROBOT_HOME/hub``.
            episodes (list[int] | None, optional): If specified, this will only load episodes specified by
                their episode_index in this list. Defaults to None.
-            image_transforms (Callable | None, optional): You can pass standard v2 image transforms from
-                torchvision.transforms.v2 here which will be applied to visual modalities (whether they come
-                from videos or images). Defaults to None.
+            image_transforms (Callable | None, optional):
+                Transform applied to visual modalities inside `__getitem__` after image decoding / tensor
+                conversion. This works for both image-backed and video-backed observations and can later be
+                updated with `set_image_transforms()` or cleared with `clear_image_transforms()`.
+                Defaults to None.
            delta_timestamps (dict[list[float]] | None, optional): _description_. Defaults to None.
            tolerance_s (float, optional): Tolerance in seconds used to ensure data timestamps are actually in
                sync with the fps value. It is used at the init of the dataset to make sure that each
@@ -192,7 +194,8 @@ class LeRobotDataset(torch.utils.data.Dataset):
        super().__init__()
        self.repo_id = repo_id
        self._requested_root = Path(root) if root else None
-        self.image_transforms = image_transforms
+        self.reader = None
+        self.set_image_transforms(image_transforms)
        self.delta_timestamps = delta_timestamps
        self.episodes = episodes
        self.tolerance_s = tolerance_s
@@ -475,6 +478,18 @@ class LeRobotDataset(torch.utils.data.Dataset):
            f"}})"
        )

+    def set_image_transforms(self, image_transforms: Callable | None) -> None:
+        """Replace the transform applied to visual observations."""
+        if image_transforms is not None and not callable(image_transforms):
+            raise TypeError("image_transforms must be callable or None.")
+        self.image_transforms = image_transforms
+        if self.reader is not None:
+            self.reader._image_transforms = image_transforms
+
+    def clear_image_transforms(self) -> None:
+        """Remove the transform applied to visual observations."""
+        self.set_image_transforms(None)
+
    # ── Hub methods (stay on facade) ──────────────────────────────────

    def push_to_hub(
@@ -89,12 +89,24 @@ class MultiLeRobotDataset(torch.utils.data.Dataset):
                )
                self.disabled_features.update(extra_keys)

-        self.image_transforms = image_transforms
        self.delta_timestamps = delta_timestamps
        # TODO(rcadene, aliberts): We should not perform this aggregation for datasets
        # with multiple robots of different ranges. Instead we should have one normalization
        # per robot.
        self.stats = aggregate_stats([dataset.meta.stats for dataset in self._datasets])
+        self.set_image_transforms(image_transforms)
+
+    def set_image_transforms(self, image_transforms: Callable | None) -> None:
+        """Replace the transform for this dataset and its children."""
+        if image_transforms is not None and not callable(image_transforms):
+            raise TypeError("image_transforms must be callable or None.")
+        self.image_transforms = image_transforms
+        for dataset in getattr(self, "_datasets", []):
+            dataset.set_image_transforms(self.image_transforms)
+
+    def clear_image_transforms(self) -> None:
+        """Remove the transform from this dataset and its children."""
+        self.set_image_transforms(None)

    @property
    def repo_id_to_index(self):
@@ -44,6 +44,13 @@ from lerobot.utils.constants import (
 )


+def _make_vec_env_cls(use_async: bool, n_envs: int):
+    """Return the right VectorEnv constructor."""
+    if use_async and n_envs > 1:
+        return gym.vector.AsyncVectorEnv
+    return gym.vector.SyncVectorEnv
+
+
@dataclass
 class EnvConfig(draccus.ChoiceRegistry, abc.ABC):
    task: str | None = None
@@ -75,13 +82,14 @@ class EnvConfig(draccus.ChoiceRegistry, abc.ABC):
    def create_envs(
        self,
        n_envs: int,
-        use_async_envs: bool = False,
+        use_async_envs: bool = True,
    ) -> dict[str, dict[int, gym.vector.VectorEnv]]:
        """Create {suite: {task_id: VectorEnv}}.

        Default: single-task env via gym.make(). Multi-task benchmarks override.
+        AsyncVectorEnv is the default for n_envs > 1; auto-downgraded to Sync for n_envs=1.
        """
-        env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv
+        env_cls = gym.vector.AsyncVectorEnv if (use_async_envs and n_envs > 1) else gym.vector.SyncVectorEnv

        if self.gym_id not in gym_registry:
            print(f"gym id '{self.gym_id}' not found, attempting to import '{self.package_name}'...")
@@ -394,17 +402,22 @@ class LiberoEnv(EnvConfig):

    @property
    def gym_kwargs(self) -> dict:
-        kwargs: dict[str, Any] = {"obs_type": self.obs_type, "render_mode": self.render_mode}
+        kwargs: dict[str, Any] = {
+            "obs_type": self.obs_type,
+            "render_mode": self.render_mode,
+            "observation_height": self.observation_height,
+            "observation_width": self.observation_width,
+        }
        if self.task_ids is not None:
            kwargs["task_ids"] = self.task_ids
        return kwargs

-    def create_envs(self, n_envs: int, use_async_envs: bool = False):
+    def create_envs(self, n_envs: int, use_async_envs: bool = True):
        from lerobot.envs.libero import create_libero_envs

        if self.task is None:
            raise ValueError("LiberoEnv requires a task to be specified")
-        env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv
+        env_cls = _make_vec_env_cls(use_async_envs, n_envs)
        return create_libero_envs(
            task=self.task,
            n_envs=n_envs,
@@ -468,12 +481,12 @@ class MetaworldEnv(EnvConfig):
            "render_mode": self.render_mode,
        }

-    def create_envs(self, n_envs: int, use_async_envs: bool = False):
+    def create_envs(self, n_envs: int, use_async_envs: bool = True):
        from lerobot.envs.metaworld import create_metaworld_envs

        if self.task is None:
            raise ValueError("MetaWorld requires a task to be specified")
-        env_cls = gym.vector.AsyncVectorEnv if use_async_envs else gym.vector.SyncVectorEnv
+        env_cls = _make_vec_env_cls(use_async_envs, n_envs)
        return create_metaworld_envs(
            task=self.task,
            n_envs=n_envs,
@@ -58,7 +58,7 @@ def make_env_pre_post_processors(
 def make_env(
    cfg: EnvConfig | str,
    n_envs: int = 1,
-    use_async_envs: bool = False,
+    use_async_envs: bool = True,
    hub_cache_dir: str | None = None,
    trust_remote_code: bool = False,
 ) -> dict[str, dict[int, gym.vector.VectorEnv]]:
@@ -29,6 +29,7 @@ from gymnasium import spaces
 from libero.libero import benchmark, get_libero_path
 from libero.libero.envs import OffScreenRenderEnv

+from lerobot.envs.utils import _LazyAsyncVectorEnv
 from lerobot.types import RobotObservation


@@ -150,7 +151,17 @@ class LiberoEnv(gym.Env):

        self.init_state_id = self.episode_index  # tie each sub-env to a fixed init state

-        self._env = self._make_envs_task(task_suite, self.task_id)
+        # Extract task metadata without allocating GPU resources (safe before fork).
+        task = task_suite.get_task(task_id)
+        self.task = task.name
+        self.task_description = task.language
+        self._task_bddl_file = os.path.join(
+            get_libero_path("bddl_files"), task.problem_folder, task.bddl_file
+        )
+        self._env: OffScreenRenderEnv | None = (
+            None  # deferred — created on first reset() inside the worker subprocess
+        )
+
        default_steps = 500
        self._max_episode_steps = (
            TASK_SUITE_MAX_STEPS.get(task_suite_name, default_steps)
@@ -221,29 +232,33 @@ class LiberoEnv(gym.Env):
            low=ACTION_LOW, high=ACTION_HIGH, shape=(ACTION_DIM,), dtype=np.float32
        )

+    def _ensure_env(self) -> None:
+        """Create the underlying OffScreenRenderEnv on first use.
+
+        Called inside the worker subprocess after fork(), so each worker gets
+        its own clean EGL context rather than inheriting a stale one from the
+        parent process (which causes EGL_BAD_CONTEXT crashes with AsyncVectorEnv).
+        """
+        if self._env is not None:
+            return
+        env = OffScreenRenderEnv(
+            bddl_file_name=self._task_bddl_file,
+            camera_heights=self.observation_height,
+            camera_widths=self.observation_width,
+        )
+        env.reset()
+        self._env = env
+
    def render(self):
+        self._ensure_env()
        raw_obs = self._env.env._get_observations()
        pixels = self._format_raw_obs(raw_obs)["pixels"]
        image = next(iter(pixels.values()))
        image = image[::-1, ::-1]  # flip both H and W for visualization
        return image

-    def _make_envs_task(self, task_suite: Any, task_id: int = 0):
-        task = task_suite.get_task(task_id)
-        self.task = task.name
-        self.task_description = task.language
-        task_bddl_file = os.path.join(get_libero_path("bddl_files"), task.problem_folder, task.bddl_file)
-
-        env_args = {
-            "bddl_file_name": task_bddl_file,
-            "camera_heights": self.observation_height,
-            "camera_widths": self.observation_width,
-        }
-        env = OffScreenRenderEnv(**env_args)
-        env.reset()
-        return env
-
    def _format_raw_obs(self, raw_obs: RobotObservation) -> RobotObservation:
+        assert self._env is not None, "_format_raw_obs called before _ensure_env()"
        images = {}
        for camera_name in self.camera_name:
            image = raw_obs[camera_name]
@@ -295,6 +310,7 @@ class LiberoEnv(gym.Env):
        )

    def reset(self, seed=None, **kwargs):
+        self._ensure_env()
        super().reset(seed=seed)
        self._env.seed(seed)
        raw_obs = self._env.reset()
@@ -321,6 +337,8 @@ class LiberoEnv(gym.Env):
        return observation, info

    def step(self, action: np.ndarray) -> tuple[RobotObservation, float, bool, bool, dict[str, Any]]:
+        self._ensure_env()
+        assert self._env is not None
        if action.ndim != 1:
            raise ValueError(
                f"Expected action to be 1-D (shape (action_dim,)), "
@@ -345,7 +363,8 @@ class LiberoEnv(gym.Env):
        return observation, reward, terminated, truncated, info

    def close(self):
-        self._env.close()
+        if self._env is not None:
+            self._env.close()


 def _make_env_fns(
@@ -428,6 +447,8 @@ def create_libero_envs(
    if task_ids_filter is not None:
        print(f"Restricting to task_ids={task_ids_filter}")

+    is_async = env_cls is gym.vector.AsyncVectorEnv
+
    out: dict[str, dict[int, Any]] = defaultdict(dict)
    for suite_name in suite_names:
        suite = _get_suite(suite_name)
@@ -436,6 +457,11 @@ def create_libero_envs(
        if not selected:
            raise ValueError(f"No tasks selected for suite '{suite_name}' (available: {total}).")

+        # All tasks in a suite share identical observation/action spaces.
+        # Probe once and reuse to avoid creating a temp env per task.
+        cached_obs_space: spaces.Space | None = None
+        cached_act_space: spaces.Space | None = None
+
        for tid in selected:
            fns = _make_env_fns(
                suite=suite,
@@ -449,8 +475,14 @@ def create_libero_envs(
                control_mode=control_mode,
                camera_name_mapping=camera_name_mapping,
            )
-            out[suite_name][tid] = env_cls(fns)
+            if is_async:
+                lazy = _LazyAsyncVectorEnv(fns, cached_obs_space, cached_act_space)
+                if cached_obs_space is None:
+                    cached_obs_space = lazy.observation_space
+                    cached_act_space = lazy.action_space
+                out[suite_name][tid] = lazy
+            else:
+                out[suite_name][tid] = env_cls(fns)
            print(f"Built vec env | suite={suite_name} | task_id={tid} | n_envs={n_envs}")

-    # return plain dicts for predictability
    return {suite: dict(task_map) for suite, task_map in out.items()}
@@ -25,6 +25,7 @@ import metaworld.policies as policies
 import numpy as np
 from gymnasium import spaces

+from lerobot.envs.utils import _LazyAsyncVectorEnv
 from lerobot.types import RobotObservation

 # ---- Load configuration data from the external JSON file ----
@@ -97,8 +98,9 @@ class MetaworldEnv(gym.Env):
        self.visualization_height = visualization_height
        self.camera_name = camera_name

-        self._env = self._make_envs_task(self.task)
-        self._max_episode_steps = self._env.max_path_length
+        self._env_name = self.task  # already stripped of "metaworld-" prefix above
+        self._env = None  # deferred — created on first reset() inside the worker subprocess
+        self._max_episode_steps = 500  # MT1 environments always have max_path_length=500
        self.task_description = TASK_DESCRIPTIONS[self.task]

        self.expert_policy = TASK_POLICY_MAPPING[self.task]()
@@ -136,6 +138,24 @@ class MetaworldEnv(gym.Env):

        self.action_space = spaces.Box(low=-1, high=1, shape=(ACTION_DIM,), dtype=np.float32)

+    def _ensure_env(self) -> None:
+        """Create the underlying MetaWorld env on first use.
+
+        Called inside the worker subprocess after fork(), so each worker gets
+        its own clean rendering context rather than inheriting a stale one from
+        the parent process (which causes crashes with AsyncVectorEnv).
+        """
+        if self._env is not None:
+            return
+        mt1 = metaworld.MT1(self._env_name, seed=42)
+        env = mt1.train_classes[self._env_name](render_mode="rgb_array", camera_name=self.camera_name)
+        env.set_task(mt1.train_tasks[0])
+        if self.camera_name == "corner2":
+            env.model.cam_pos[2] = [0.75, 0.075, 0.7]
+        env.reset()
+        env._freeze_rand_vec = False  # otherwise no randomization
+        self._env = env
+
    def render(self) -> np.ndarray:
        """
        Render the current environment frame.
@@ -143,26 +163,13 @@ class MetaworldEnv(gym.Env):
        Returns:
            np.ndarray: The rendered RGB image from the environment.
        """
+        self._ensure_env()
        image = self._env.render()
        if self.camera_name == "corner2":
            # Images from this camera are flipped — correct them
            image = np.flip(image, (0, 1))
        return image

-    def _make_envs_task(self, env_name: str):
-        mt1 = metaworld.MT1(env_name, seed=42)
-        env = mt1.train_classes[env_name](render_mode="rgb_array", camera_name=self.camera_name)
-        env.set_task(mt1.train_tasks[0])
-        if self.camera_name == "corner2":
-            env.model.cam_pos[2] = [
-                0.75,
-                0.075,
-                0.7,
-            ]  # corner2 position, similar to https://arxiv.org/pdf/2206.14244
-        env.reset()
-        env._freeze_rand_vec = False  # otherwise no randomization
-        return env
-
    def _format_raw_obs(self, raw_obs: np.ndarray) -> RobotObservation:
        image = None
        if self._env is not None:
@@ -209,6 +216,7 @@ class MetaworldEnv(gym.Env):
            observation (RobotObservation): The initial formatted observation.
            info (Dict[str, Any]): Additional info about the reset state.
        """
+        self._ensure_env()
        super().reset(seed=seed)

        raw_obs, info = self._env.reset(seed=seed)
@@ -232,6 +240,7 @@ class MetaworldEnv(gym.Env):
            truncated (bool): Whether the episode was truncated due to a time limit.
            info (Dict[str, Any]): Additional environment info.
        """
+        self._ensure_env()
        if action.ndim != 1:
            raise ValueError(
                f"Expected action to be 1-D (shape (action_dim,)), "
@@ -263,7 +272,8 @@ class MetaworldEnv(gym.Env):
        return observation, reward, terminated, truncated, info

    def close(self):
-        self._env.close()
+        if self._env is not None:
+            self._env.close()


 # ---- Main API ----------------------------------------------------------------
@@ -297,6 +307,9 @@ def create_metaworld_envs(

    print(f"Creating Meta-World envs | task_groups={task_groups} | n_envs(per task)={n_envs}")

+    is_async = env_cls is gym.vector.AsyncVectorEnv
+    cached_obs_space = None
+    cached_act_space = None
    out: dict[str, dict[int, Any]] = defaultdict(dict)

    for group in task_groups:
@@ -309,7 +322,14 @@ def create_metaworld_envs(
            # build n_envs factories
            fns = [(lambda tn=task_name: MetaworldEnv(task=tn, **gym_kwargs)) for _ in range(n_envs)]

-            out[group][tid] = env_cls(fns)
+            if is_async:
+                lazy = _LazyAsyncVectorEnv(fns, cached_obs_space, cached_act_space)
+                if cached_obs_space is None:
+                    cached_obs_space = lazy.observation_space
+                    cached_act_space = lazy.action_space
+                out[group][tid] = lazy
+            else:
+                out[group][tid] = env_cls(fns)

    # return a plain dict for consistency
    return {group: dict(task_map) for group, task_map in out.items()}
@@ -16,7 +16,7 @@
 import importlib.util
 import os
 import warnings
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from functools import singledispatch
 from typing import Any

@@ -130,56 +130,99 @@ def env_to_policy_features(env_cfg: EnvConfig) -> dict[str, PolicyFeature]:
    return policy_features


-def are_all_envs_same_type(env: gym.vector.VectorEnv) -> bool:
-    first_type = type(env.envs[0])  # Get type of first env
-    return all(type(e) is first_type for e in env.envs)  # Fast type check
+def _sub_env_has_attr(env: gym.vector.VectorEnv, attr: str) -> bool:
+    try:
+        env.get_attr(attr)
+        return True
+    except (AttributeError, Exception):
+        return False
+
+
+class _LazyAsyncVectorEnv:
+    """Defers AsyncVectorEnv creation until first use.
+
+    Creating all tasks' AsyncVectorEnvs upfront spawns N_tasks × n_envs worker
+    processes, all of which allocate EGL/GPU resources immediately. Since tasks
+    are evaluated sequentially, only one task's workers need to be alive at a
+    time. This wrapper stores the factory functions and creates the real
+    AsyncVectorEnv on first reset()/step()/call(), keeping peak process count = n_envs.
+    """
+
+    def __init__(
+        self,
+        env_fns: list[Callable],
+        observation_space=None,
+        action_space=None,
+    ):
+        self._env_fns = env_fns
+        self._env: gym.vector.AsyncVectorEnv | None = None
+        self.num_envs = len(env_fns)
+        if observation_space is not None and action_space is not None:
+            self.observation_space = observation_space
+            self.action_space = action_space
+        else:
+            tmp = env_fns[0]()
+            self.observation_space = tmp.observation_space
+            self.action_space = tmp.action_space
+            tmp.close()
+        self.single_observation_space = self.observation_space
+        self.single_action_space = self.action_space
+
+    def _ensure(self) -> None:
+        if self._env is None:
+            self._env = gym.vector.AsyncVectorEnv(self._env_fns, context="forkserver", shared_memory=True)
+
+    def reset(self, **kwargs):
+        self._ensure()
+        return self._env.reset(**kwargs)
+
+    def step(self, actions):
+        self._ensure()
+        return self._env.step(actions)
+
+    def call(self, name, *args, **kwargs):
+        self._ensure()
+        return self._env.call(name, *args, **kwargs)
+
+    def get_attr(self, name):
+        self._ensure()
+        return self._env.get_attr(name)
+
+    def close(self) -> None:
+        if self._env is not None:
+            self._env.close()
+            self._env = None


 def check_env_attributes_and_types(env: gym.vector.VectorEnv) -> None:
    with warnings.catch_warnings():
-        warnings.simplefilter("once", UserWarning)  # Apply filter only in this function
+        warnings.simplefilter("once", UserWarning)

-        if not (hasattr(env.envs[0], "task_description") and hasattr(env.envs[0], "task")):
+        if not (_sub_env_has_attr(env, "task_description") and _sub_env_has_attr(env, "task")):
            warnings.warn(
                "The environment does not have 'task_description' and 'task'. Some policies require these features.",
                UserWarning,
                stacklevel=2,
            )
-        if not are_all_envs_same_type(env):
-            warnings.warn(
-                "The environments have different types. Make sure you infer the right task from each environment. Empty task will be passed instead.",
-                UserWarning,
-                stacklevel=2,
-            )


 def add_envs_task(env: gym.vector.VectorEnv, observation: RobotObservation) -> RobotObservation:
    """Adds task feature to the observation dict with respect to the first environment attribute."""
-    if hasattr(env.envs[0], "task_description"):
-        task_result = env.call("task_description")
+    if _sub_env_has_attr(env, "task_description"):
+        task_result = list(env.call("task_description"))

-        if isinstance(task_result, tuple):
-            task_result = list(task_result)
-
-        if not isinstance(task_result, list):
-            raise TypeError(f"Expected task_description to return a list, got {type(task_result)}")
        if not all(isinstance(item, str) for item in task_result):
            raise TypeError("All items in task_description result must be strings")

        observation["task"] = task_result
-    elif hasattr(env.envs[0], "task"):
-        task_result = env.call("task")
+    elif _sub_env_has_attr(env, "task"):
+        task_result = list(env.call("task"))

-        if isinstance(task_result, tuple):
-            task_result = list(task_result)
-
-        if not isinstance(task_result, list):
-            raise TypeError(f"Expected task to return a list, got {type(task_result)}")
        if not all(isinstance(item, str) for item in task_result):
            raise TypeError("All items in task result must be strings")

        observation["task"] = task_result
-    else:  #  For envs without language instructions, e.g. aloha transfer cube and etc.
+    else:
        num_envs = observation[list(observation.keys())[0]].shape[0]
        observation["task"] = ["" for _ in range(num_envs)]
    return observation
@@ -1,37 +0,0 @@
-# Multitask DiT Policy
-
-## Citation
-
-If you use this work, please cite the following works:
-
-```bibtex
-@misc{jones2025multitaskditpolicy,
-  author = {Bryson Jones},
-  title = {Dissecting and Open-Sourcing Multitask Diffusion Transformer Policy},
-  year = {2025},
-  url = {https://brysonkjones.substack.com/p/dissecting-and-open-sourcing-multitask-diffusion-transformer-policy},
-  note = {Blog post}
-}
-```
-
-```bibtex
-@misc{trilbmteam2025carefulexaminationlargebehaviormodels,
-  author       = {TRI LBM Team},
-  title        = {A Careful Examination of Large Behavior Models for Multitask Dexterous Manipulation},
-  year         = {2025},
-  eprint       = {arXiv:2507.05331},
-  archivePrefix = {arXiv},
-  primaryClass = {cs.RO},
-  url          = {https://arxiv.org/abs/2507.05331}
-}
-```
-
-```bibtex
-@misc{bostondynamics2025largebehaviormodelsatlas,
-  author       = {Boston Dynamics and TRI Research Team},
-  title        = {Large Behavior Models and Atlas Find New Footing},
-  year         = {2025},
-  url          = {https://bostondynamics.com/blog/large-behavior-models-atlas-find-new-footing/},
-  note         = {Blog post}
-}
-```
@@ -0,0 +1 @@
+../../../../docs/source/policy_multi_task_dit_README.md
@@ -1,108 +0,0 @@
-# π₀ (pi0)
-
-This repository contains the Hugging Face port of **π₀**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
-It is designed as a **Vision-Language-Action model for general robot control**.
-
---
-
-## Model Overview
-
-| Feature              | π₀                                                     | π₀.₅                                      |
-| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
-| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
-| AdaRMS               | Not used                                               | Used in action expert                     |
-| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
-| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
-| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
-
---
-
-## Relative Actions
-
-π₀ supports training with **relative actions**, where the model learns relative offsets
-from the current robot state instead of absolute joint positions. This mirrors the
-relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
-
-### How it works
-
-1. **During preprocessing**, absolute actions are converted to relative offsets:
-   `relative = action - state` (for selected joints).
-2. The relative actions are normalized using statistics computed from the relative distribution.
-3. **During postprocessing**, predicted relative actions are converted back to absolute:
-   `absolute = relative + state`.
-
-Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
-
-### Configuration
-
-| Parameter                 | Type        | Default       | Description                                                      |
-| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
-| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
-| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
-| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
-
-### Training example
-
-```bash
-python -m lerobot.scripts.lerobot_train \
-  --policy.type=pi0 \
-  --dataset.repo_id=your_org/your_dataset \
-  --policy.use_relative_actions=true \
-  --policy.relative_exclude_joints='["gripper"]'
-```
-
-When `use_relative_actions=true`, the training script automatically:
-
- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
- Replaces the standard action stats with relative stats for normalization
- Broadcasts these stats across all ranks in distributed training
-
-### Recomputing stats for an existing dataset
-
-If you want to precompute relative action stats offline, use `recompute_stats` from
-`lerobot.datasets.dataset_tools`:
-
-```python
-from lerobot.datasets.lerobot_dataset import LeRobotDataset
-from lerobot.datasets.dataset_tools import recompute_stats
-
-dataset = LeRobotDataset("your_org/your_dataset")
-dataset = recompute_stats(
-    dataset,
-    relative_action=True,
-    relative_exclude_joints=["gripper"],
-)
-```
-
---
-
-## Citation
-
-If you use this work, please cite both **OpenPI** and the π₀ paper:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{black2024pi0visionlanguageactionflowmodel,
-  title        = {π₀: A Vision-Language-Action Flow Model for General Robot Control},
-  author       = {Kevin Black and Noah Brown and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Lucy Xiaoyang Shi and James Tanner and Quan Vuong and Anna Walling and Haohuan Wang and Ury Zhilinsky},
-  year         = {2024},
-  eprint       = {2410.24164},
-  archivePrefix= {arXiv},
-  primaryClass = {cs.LG},
-  url          = {https://arxiv.org/abs/2410.24164},
-}
-```
-
---
-
-## License
-
-This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1 @@
+../../../../docs/source/policy_pi0_README.md
@@ -1,91 +0,0 @@
-# π₀.₅ (pi05)
-
-This repository contains the Hugging Face port of **π₀.₅**, adapted from [OpenPI](https://github.com/Physical-Intelligence/openpi) by the Physical Intelligence.
-It is designed as a **Vision-Language-Action model with open-world generalization**.
-
---
-
-## Model Overview
-
-| Feature              | π₀                                                     | π₀.₅                                      |
-| -------------------- | ------------------------------------------------------ | ----------------------------------------- |
-| Time Conditioning    | Concatenates time with actions via `action_time_mlp_*` | Uses `time_mlp_*` for AdaRMS conditioning |
-| AdaRMS               | Not used                                               | Used in action expert                     |
-| Tokenizer Length     | 48 tokens                                              | 200 tokens                                |
-| Discrete State Input | False (Uses `state_proj` layer)                        | True                                      |
-| Parameter Count      | Higher (includes state embedding)                      | Lower (no state embedding)                |
-
---
-
-## Relative Actions
-
-π₀.₅ supports training with **relative actions**, where the model learns relative offsets
-from the current robot state instead of absolute joint positions. This mirrors the
-relative-action transform in OpenPI (`DeltaActions`) and can improve performance.
-
-### How it works
-
-1. **During preprocessing**, absolute actions are converted to relative offsets:
-   `relative = action - state` (for selected joints).
-2. The relative actions are normalized using statistics computed from the relative distribution.
-3. **During postprocessing**, predicted relative actions are converted back to absolute:
-   `absolute = relative + state`.
-
-Joints listed in `relative_exclude_joints` (e.g., gripper) are kept absolute.
-
-### Configuration
-
-| Parameter                 | Type        | Default       | Description                                                      |
-| ------------------------- | ----------- | ------------- | ---------------------------------------------------------------- |
-| `use_relative_actions`    | `bool`      | `False`       | Enable relative-action training                                  |
-| `relative_exclude_joints` | `list[str]` | `["gripper"]` | Joint names to keep absolute (matched by substring)              |
-| `action_feature_names`    | `list[str]` | `None`        | Auto-populated from dataset metadata at runtime by `make_policy` |
-
-### Training example
-
-```bash
-python -m lerobot.scripts.lerobot_train \
-  --policy.type=pi05 \
-  --dataset.repo_id=your_org/your_dataset \
-  --policy.use_relative_actions=true \
-  --policy.relative_exclude_joints='["gripper"]'
-```
-
-When `use_relative_actions=true`, the training script automatically:
-
- Computes relative action statistics from the dataset (sampled chunk-level relative actions)
- Replaces the standard action stats with relative stats for normalization
- Broadcasts these stats across all ranks in distributed training
-
---
-
-## Citation
-
-If you use this work, please cite both **OpenPI** and the π₀.₅ paper:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{intelligence2025pi05visionlanguageactionmodelopenworld,
-  title        = {π₀.₅: a Vision-Language-Action Model with Open-World Generalization},
-  author       = {Physical Intelligence and Kevin Black and Noah Brown and James Darpinian and Karan Dhabalia and Danny Driess and Adnan Esmail and Michael Equi and Chelsea Finn and Niccolo Fusai and Manuel Y. Galliker and Dibya Ghosh and Lachy Groom and Karol Hausman and Brian Ichter and Szymon Jakubczak and Tim Jones and Liyiming Ke and Devin LeBlanc and Sergey Levine and Adrian Li-Bell and Mohith Mothukuri and Suraj Nair and Karl Pertsch and Allen Z. Ren and Lucy Xiaoyang Shi and Laura Smith and Jost Tobias Springenberg and Kyle Stachowicz and James Tanner and Quan Vuong and Homer Walke and Anna Walling and Haohuan Wang and Lili Yu and Ury Zhilinsky},
-  year         = {2025},
-  eprint       = {2504.16054},
-  archivePrefix= {arXiv},
-  primaryClass = {cs.LG},
-  url          = {https://arxiv.org/abs/2504.16054},
-}
-```
-
---
-
-## License
-
-This port follows the **Apache 2.0 License**, consistent with the original [OpenPI repository](https://github.com/Physical-Intelligence/openpi).
@@ -0,0 +1 @@
+../../../../docs/source/policy_pi05_README.md
@@ -1,38 +0,0 @@
-# Real-Time Chunking (RTC)
-
-This module contains the LeRobot implementation of **Real-Time Chunking (RTC)**, an inference-time technique for flow-matching based policies.
-
-**Note**: RTC is not a policy itself, but rather an inference enhancement that works with flow-matching based policies including [π₀](../pi0/), [π₀.₅](../pi05/), and [SmolVLA](../smolvla/).
-
---
-
-## Citation
-
-If you use Real-Time Chunking in your work, please cite:
-
-```bibtex
-@misc{openpi2024,
-  author       = {Physical Intelligence Lab},
-  title        = {OpenPI: PyTorch Implementation of π0 and π0.5 Policies},
-  year         = {2024},
-  publisher    = {GitHub},
-  howpublished = {\url{https://github.com/Physical-Intelligence/openpi}},
-  license      = {Apache-2.0}
-}
-
-@misc{black2025realtimeexecutionactionchunking,
-      title={Real-Time Execution of Action Chunking Flow Policies},
-      author={Kevin Black and Manuel Y. Galliker and Sergey Levine},
-      year={2025},
-      eprint={2506.07339},
-      archivePrefix={arXiv},
-      primaryClass={cs.RO},
-      url={https://arxiv.org/abs/2506.07339},
-}
-```
-
---
-
-## License
-
-This implementation follows the **Apache 2.0 License**, consistent with the LeRobot project.
@@ -0,0 +1 @@
+../../../../docs/source/policy_rtc_README.md
@@ -1,14 +0,0 @@
-## Paper
-
-https://arxiv.org/abs/2509.25358
-
-## Citation
-
-```bibtex
-@article{chen2025sarm,
-  title={SARM: Stage-Aware Reward Modeling for Long Horizon Robot Manipulation},
-  author={Chen, Qianzhong and Yu, Justin and Schwager, Mac and Abbeel, Pieter and Shentu, Yide and Wu, Philipp},
-  journal={arXiv preprint arXiv:2509.25358},
-  year={2025}
-}
-```
@@ -0,0 +1 @@
+../../../../docs/source/policy_sarm_README.md
@@ -136,8 +136,8 @@ class TokenizerProcessorStep(ObservationProcessorStep):
        # Standardize to a list of strings for the tokenizer
        if isinstance(task, str):
            return [task]
-        elif isinstance(task, list) and all(isinstance(t, str) for t in task):
-            return task
+        elif isinstance(task, (list, tuple)) and all(isinstance(t, str) for t in task):
+            return list(task)

        return None

@@ -73,7 +73,6 @@ from lerobot.configs import parser
 from lerobot.configs.eval import EvalPipelineConfig
 from lerobot.envs.factory import make_env, make_env_pre_post_processors
 from lerobot.envs.utils import (
-    add_envs_task,
    check_env_attributes_and_types,
    close_envs,
    preprocess_observation,
@@ -166,9 +165,15 @@ def rollout(
        if return_observations:
            all_observations.append(deepcopy(observation))

-        # Infer "task" from attributes of environments.
-        # TODO: works with SyncVectorEnv but not AsyncVectorEnv
-        observation = add_envs_task(env, observation)
+        # Infer "task" from sub-environments (prefer natural language description).
+        # env.call() works with both SyncVectorEnv and AsyncVectorEnv.
+        try:
+            observation["task"] = list(env.call("task_description"))
+        except Exception:
+            try:
+                observation["task"] = list(env.call("task"))
+            except Exception:
+                observation["task"] = [""] * env.num_envs

        # Apply environment-specific preprocessing (e.g., LiberoProcessorStep for LIBERO)
        observation = env_preprocessor(observation)
@@ -193,8 +198,14 @@ def rollout(

        # VectorEnv stores is_success in `info["final_info"][env_index]["is_success"]`. "final_info" isn't
        # available if none of the envs finished.
-        if "final_info" in info and isinstance(info["final_info"], dict):
-            successes = info["final_info"]["is_success"].tolist()
+        if "final_info" in info:
+            final_info = info["final_info"]
+            if not isinstance(final_info, dict):
+                raise RuntimeError(
+                    "Unsupported `final_info` format: expected dict (Gymnasium >= 1.0). "
+                    "You're likely using an older version of gymnasium (< 1.0). Please upgrade."
+                )
+            successes = final_info["is_success"].tolist()
        elif "is_success" in info:
            is_success = info["is_success"]
            successes = (
@@ -312,8 +323,9 @@ def eval_policy(
        n_to_render_now = min(max_episodes_rendered - n_episodes_rendered, env.num_envs)
        if isinstance(env, gym.vector.SyncVectorEnv):
            ep_frames.append(np.stack([env.envs[i].render() for i in range(n_to_render_now)]))  # noqa: B023
-        elif isinstance(env, gym.vector.AsyncVectorEnv):
+        elif hasattr(env, "call"):
            # Here we must render all frames and discard any we don't need.
+            # Covers AsyncVectorEnv and _LazyAsyncVectorEnv (which wraps one).
            ep_frames.append(np.stack(env.call("render")[:n_to_render_now]))

    if max_episodes_rendered > 0:
@@ -515,7 +527,7 @@ def eval_main(cfg: EvalPipelineConfig):

    logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}")

-    logging.info("Making environment.")
+    logging.info(f"Making environment (batch_size={cfg.eval.batch_size}, async={cfg.eval.use_async_envs}).")
    envs = make_env(
        cfg.env,
        n_envs=cfg.eval.batch_size,
@@ -749,23 +761,39 @@ def eval_policy_all(
    )

    if max_parallel_tasks <= 1:
-        # sequential path (single accumulator path on the main thread)
-        # NOTE: keeping a single-threaded accumulator avoids concurrent list appends or locks
-        for task_group, task_id, env in tasks:
-            tg, tid, metrics = task_runner(task_group, task_id, env)
-            _accumulate_to(tg, metrics)
-            per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
+        prefetch_thread: threading.Thread | None = None
+        for i, (task_group, task_id, env) in enumerate(tasks):
+            if prefetch_thread is not None:
+                prefetch_thread.join()
+                prefetch_thread = None
+
+            try:
+                tg, tid, metrics = task_runner(task_group, task_id, env)
+                _accumulate_to(tg, metrics)
+                per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
+            finally:
+                env.close()
+                # Prefetch next task's workers *after* closing current env to prevent
+                # GPU memory overlap between consecutive tasks.
+                if i + 1 < len(tasks):
+                    next_env = tasks[i + 1][2]
+                    if hasattr(next_env, "_ensure"):
+                        prefetch_thread = threading.Thread(target=next_env._ensure, daemon=True)
+                        prefetch_thread.start()
    else:
-        # threaded path: submit all tasks, consume completions on main thread and accumulate there
        with cf.ThreadPoolExecutor(max_workers=max_parallel_tasks) as executor:
            fut2meta = {}
            for task_group, task_id, env in tasks:
                fut = executor.submit(task_runner, task_group, task_id, env)
-                fut2meta[fut] = (task_group, task_id)
+                fut2meta[fut] = (task_group, task_id, env)
            for fut in cf.as_completed(fut2meta):
-                tg, tid, metrics = fut.result()
-                _accumulate_to(tg, metrics)
-                per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
+                tg, tid, env = fut2meta[fut]
+                try:
+                    tg, tid, metrics = fut.result()
+                    _accumulate_to(tg, metrics)
+                    per_task_infos.append({"task_group": tg, "task_id": tid, "metrics": metrics})
+                finally:
+                    env.close()

    # compute aggregated metrics helper (robust to lists/scalars)
    def _agg_from_list(xs):
@@ -421,6 +421,7 @@ def record_loop(
                act_processed_policy: RobotAction = make_robot_action(action_values, dataset.features)
                # Applies a pipeline to the action, default is IdentityProcessor
                robot_action_to_send = robot_action_processor((act_processed_policy, obs))
+                action_values = robot_action_to_send

        elif policy is None and isinstance(teleop, Teleoperator):
            act = teleop.get_action()
@@ -24,6 +24,7 @@ import torch
 from huggingface_hub import HfApi
 from PIL import Image
 from safetensors.torch import load_file
+from torchvision.transforms import v2

 import lerobot
 from lerobot.configs.default import DatasetConfig
@@ -34,6 +35,7 @@ from lerobot.datasets.image_writer import image_array_to_pil_image
 from lerobot.datasets.io_utils import hf_transform_to_torch
 from lerobot.datasets.lerobot_dataset import LeRobotDataset
 from lerobot.datasets.multi_dataset import MultiLeRobotDataset
+from lerobot.datasets.transforms import ImageTransforms, ImageTransformsConfig
 from lerobot.datasets.utils import (
    DEFAULT_CHUNK_SIZE,
    DEFAULT_DATA_FILE_SIZE_IN_MB,
@@ -355,6 +357,62 @@ def test_add_frame_image_pil(image_dataset):
    assert dataset[0]["image"].shape == torch.Size(DUMMY_CHW)


+def test_set_image_transforms_applies_transparently(image_dataset):
+    dataset = image_dataset
+    dataset.add_frame({"image": np.random.rand(*DUMMY_CHW), "task": "Dummy task"})
+    dataset.save_episode()
+    dataset.finalize()
+
+    dataset.set_image_transforms(v2.Resize((224, 224)))
+    assert dataset[0]["image"].shape == torch.Size((3, 224, 224))
+
+    dataset.set_image_transforms(v2.Resize((128, 128)))
+    assert dataset[0]["image"].shape == torch.Size((3, 128, 128))
+
+    dataset.clear_image_transforms()
+    assert dataset[0]["image"].shape == torch.Size(DUMMY_CHW)
+
+
+def test_set_image_transforms_supports_lerobot_image_transforms(image_dataset):
+    dataset = image_dataset
+    dataset.add_frame({"image": np.random.rand(*DUMMY_CHW), "task": "Dummy task"})
+    dataset.save_episode()
+    dataset.finalize()
+
+    image_transforms = ImageTransforms(ImageTransformsConfig(enable=False))
+    dataset.set_image_transforms(image_transforms)
+
+    assert dataset.image_transforms is image_transforms
+    assert dataset[0]["image"].shape == torch.Size(DUMMY_CHW)
+
+
+def test_set_image_transforms_supports_loaded_dataset(tmp_path, lerobot_dataset_factory):
+    dataset = lerobot_dataset_factory(root=tmp_path / "test", use_videos=False)
+    dataset.set_image_transforms(v2.Compose([v2.Resize((224, 224)), v2.Resize((112, 112))]))
+
+    camera_key = dataset.meta.camera_keys[0]
+    assert dataset[0][camera_key].shape == torch.Size((3, 112, 112))
+
+
+def test_multilerobot_dataset_set_image_transforms_propagates(tmp_path, lerobot_dataset_factory):
+    root = tmp_path / "multi"
+    repo_ids = ["lerobot/test_multi_a", "lerobot/test_multi_b"]
+
+    for repo_id in repo_ids:
+        lerobot_dataset_factory(root=root / repo_id, repo_id=repo_id, use_videos=False)
+
+    dataset = MultiLeRobotDataset(repo_ids, root=root, download_videos=False)
+    dataset.set_image_transforms(v2.Resize((96, 96)))
+
+    camera_key = dataset.camera_keys[0]
+    assert dataset[0][camera_key].shape == torch.Size((3, 96, 96))
+    assert all(child.image_transforms is dataset.image_transforms for child in dataset._datasets)
+
+    dataset.clear_image_transforms()
+    assert dataset.image_transforms is None
+    assert all(child.image_transforms is None for child in dataset._datasets)
+
+
 def test_image_array_to_pil_image_wrong_range_float_0_255():
    image = np.random.rand(*DUMMY_HWC) * 255
    with pytest.raises(ValueError):
@@ -90,7 +90,7 @@ def test_base_create_envs():
        envs = _Env().create_envs(n_envs=2)
        assert "_dispatch_base_test" in envs
        env = envs["_dispatch_base_test"][0]
-        assert isinstance(env, gym.vector.SyncVectorEnv)
+        assert isinstance(env, gym.vector.VectorEnv)
        assert env.num_envs == 2
        env.close()
    finally:
@@ -189,6 +189,30 @@ def test_list_of_strings_tokenization(mock_auto_tokenizer):
    assert attention_mask.shape == (2, 8)


+@require_package("transformers")
+@patch("lerobot.processor.tokenizer_processor.AutoTokenizer")
+def test_tuple_of_strings_tokenization(mock_auto_tokenizer):
+    """Test tokenization of a tuple of strings (returned by VectorEnv.call())."""
+    mock_tokenizer = MockTokenizer(vocab_size=100)
+    mock_auto_tokenizer.from_pretrained.return_value = mock_tokenizer
+
+    processor = TokenizerProcessorStep(tokenizer_name="test-tokenizer", max_length=8)
+
+    transition = create_transition(
+        observation={"state": torch.tensor([1.0, 2.0])},
+        action=torch.tensor([0.1, 0.2]),
+        complementary_data={"task": ("pick up cube", "place on table")},
+    )
+
+    result = processor(transition)
+
+    observation = result[TransitionKey.OBSERVATION]
+    tokens = observation[f"{OBS_LANGUAGE}.tokens"]
+    attention_mask = observation[f"{OBS_LANGUAGE}.attention_mask"]
+    assert tokens.shape == (2, 8)
+    assert attention_mask.shape == (2, 8)
+
+
@require_package("transformers")
@patch("lerobot.processor.tokenizer_processor.AutoTokenizer")
 def test_custom_keys(mock_auto_tokenizer):
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_multi_task_dit_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_pi0_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_pi05_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_rtc_README.md`
				`@@ -0,0 +1 @@`
				`../../../../docs/source/policy_sarm_README.md`