fix: integrate PR #3396 review feedback

- processor(vlabench): drop the no-op `VLABenchProcessorStep` and
  let `VLABenchEnv` fall back to the identity `get_env_processors`
  default. Removes the class, its export, and its override.
- envs(vlabench): hoist `import cv2` to module scope; propagate the
  gym `seed` into the inner dm_control env (re-seeding `task.random`
  and the env's `_random_state`) so layout sampling is reproducible.
- docs(vlabench): mirror the CI eval command across all four
  snippets (`--eval.use_async_envs=false`, `--policy.device=cuda`,
  and the `rename_map` for image/second_image/wrist_image →
  camera1/2/3). Intro now calls out that LeRobot exposes 43 of
  VLABench's 100 upstream task categories via `--env.task`.
- docker(vlabench): drop the `|| echo WARN` fallback on the gdown
  asset-download path so a genuine download failure surfaces
  instead of being masked.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

docker/Dockerfile.benchmark.vlabench

@@ -68,9 +68,8 @@ p = snapshot_download(repo_id='${VLABENCH_ASSETS_REPO}', repo_type='dataset', \
     local_dir='${ASSETS_DIR}', allow_patterns=['obj/**', 'scenes/**']); \
 print('snapshot_download returned:', p)"; \
     else \
-        echo "No VLABENCH_ASSETS_REPO set — falling back to gdown (best-effort)" && \
+        echo "No VLABENCH_ASSETS_REPO set — falling back to gdown" && \
-        python ~/VLABench/scripts/download_assets.py --choice all || \
+        python ~/VLABench/scripts/download_assets.py --choice all; \
-          echo "WARN: VLABench asset download failed (likely Google Drive quota)."; \
     fi && \
     python -c "\
 from pathlib import Path; \

docs/source/vlabench.mdx

@@ -1,6 +1,6 @@
 # VLABench
 
-[VLABench](https://github.com/OpenMOSS/VLABench) is a large-scale benchmark for **language-conditioned robotic manipulation with long-horizon reasoning**. It covers 100 task categories across 2,000+ objects and evaluates six dimensions of robot intelligence: mesh & texture understanding, spatial reasoning, world-knowledge transfer, semantic instruction comprehension, physical-law understanding, and long-horizon planning. Built on MuJoCo / dm_control with a Franka Panda 7-DOF arm.
+[VLABench](https://github.com/OpenMOSS/VLABench) is a large-scale benchmark for **language-conditioned robotic manipulation with long-horizon reasoning**. The upstream suite covers 100 task categories across 2,000+ objects and evaluates six dimensions of robot intelligence: mesh & texture understanding, spatial reasoning, world-knowledge transfer, semantic instruction comprehension, physical-law understanding, and long-horizon planning. Built on MuJoCo / dm_control with a Franka Panda 7-DOF arm. LeRobot exposes **43 of these tasks** through `--env.task` (21 primitives + 22 composites, see [Available tasks](#available-tasks) below).
 
 - Paper: [VLABench: A Large-Scale Benchmark for Language-Conditioned Robotics Manipulation with Long-Horizon Reasoning](https://arxiv.org/abs/2412.18194)
 - GitHub: [OpenMOSS/VLABench](https://github.com/OpenMOSS/VLABench)
@@ -53,6 +53,8 @@ export MUJOCO_GL=egl  # for headless servers (HPC, cloud)
 
 ## Evaluation
 
+All eval snippets below mirror the command CI runs (see `.github/workflows/benchmark_tests.yml`). The `--rename_map` argument maps VLABench's `image` / `second_image` / `wrist_image` camera keys onto the three-camera (`camera1` / `camera2` / `camera3`) input layout the released `smolvla_vlabench` policy was trained on.
+
 ### Single-task evaluation (recommended for quick iteration)
 
 ```bash
@@ -61,7 +63,10 @@ lerobot-eval \
   --env.type=vlabench \
   --env.task=select_fruit \
   --eval.batch_size=1 \
-  --eval.n_episodes=10
+  --eval.n_episodes=10 \
+  --eval.use_async_envs=false \
+  --policy.device=cuda \
+  '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}'
 ```
 
 ### Multi-task evaluation
@@ -74,7 +79,10 @@ lerobot-eval \
   --env.type=vlabench \
   --env.task=select_fruit,select_toy,add_condiment,heat_food \
   --eval.batch_size=1 \
-  --eval.n_episodes=10
+  --eval.n_episodes=10 \
+  --eval.use_async_envs=false \
+  --policy.device=cuda \
+  '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}'
 ```
 
 ### Suite-wide evaluation
@@ -88,7 +96,10 @@ lerobot-eval \
   --env.task=primitive \
   --eval.batch_size=1 \
   --eval.n_episodes=10 \
-  --env.max_parallel_tasks=1
+  --eval.use_async_envs=false \
+  --policy.device=cuda \
+  --env.max_parallel_tasks=1 \
+  '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}'
 ```
 
 Or both suites:
@@ -100,7 +111,10 @@ lerobot-eval \
   --env.task=primitive,composite \
   --eval.batch_size=1 \
   --eval.n_episodes=10 \
-  --env.max_parallel_tasks=1
+  --eval.use_async_envs=false \
+  --policy.device=cuda \
+  --env.max_parallel_tasks=1 \
+  '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}'
 ```
 
 ### Recommended evaluation episodes

src/lerobot/envs/configs.py

@@ -560,14 +560,6 @@ class VLABenchEnv(EnvConfig):
             env_cls=env_cls,
         )
 
-    def get_env_processors(self):
-        from lerobot.processor.env_processor import VLABenchProcessorStep
-
-        return (
-            PolicyProcessorPipeline(steps=[VLABenchProcessorStep()]),
-            PolicyProcessorPipeline(steps=[]),
-        )
-
 
 @EnvConfig.register_subclass("isaaclab_arena")
 @dataclass

src/lerobot/envs/vlabench.py

@@ -31,6 +31,7 @@ from collections.abc import Callable, Sequence
 from functools import partial
 from typing import Any
 
+import cv2
 import gymnasium as gym
 import numpy as np
 from gymnasium import spaces
@@ -246,8 +247,6 @@ class VLABenchEnv(gym.Env):
 
         def _to_hwc3(arr: np.ndarray) -> np.ndarray:
             """Coerce any camera array to the declared (h, w, 3) uint8 shape."""
-            import cv2
-
             a = np.asarray(arr)
             # Drop a leading singleton batch dim if present.
             while a.ndim > 3 and a.shape[0] == 1:
@@ -392,12 +391,28 @@ class VLABenchEnv(gym.Env):
         assert self._env is not None
         super().reset(seed=seed)
 
+        if seed is not None:
+            self._seed_inner_env(int(self.np_random.integers(0, 2**31 - 1)))
+
         self._env.reset()
 
         observation = self._get_obs()
         info = {"is_success": False}
         return observation, info
 
+    def _seed_inner_env(self, seed: int) -> None:
+        """Propagate `seed` to the inner dm_control env. `Environment.reset()`
+        doesn't accept a seed, so we re-seed the task and environment
+        `RandomState`s directly. Best-effort: silently skipped when the
+        expected attributes are absent on a given VLABench version.
+        """
+        for owner_attr, rng_attr in (("task", "random"), (None, "_random_state")):
+            owner = getattr(self._env, owner_attr) if owner_attr else self._env
+            rng = getattr(owner, rng_attr, None)
+            rng_seed = getattr(rng, "seed", None)
+            if callable(rng_seed):
+                rng_seed(seed)
+
     def step(self, action: np.ndarray) -> tuple[RobotObservation, float, bool, bool, dict[str, Any]]:
         from dm_control.rl.control import PhysicsError  # type: ignore[import-untyped]
 

src/lerobot/processor/__init__.py

@@ -40,7 +40,7 @@ from .converters import (
 )
 from .delta_action_processor import MapDeltaActionToRobotActionStep, MapTensorToDeltaActionDictStep
 from .device_processor import DeviceProcessorStep
-from .env_processor import IsaaclabArenaProcessorStep, LiberoProcessorStep, VLABenchProcessorStep
+from .env_processor import IsaaclabArenaProcessorStep, LiberoProcessorStep
 from .factory import (
     make_default_processors,
     make_default_robot_action_processor,
@@ -165,5 +165,4 @@ __all__ = [
     "to_relative_actions",
     "UnnormalizerProcessorStep",
     "VanillaObservationProcessorStep",
-    "VLABenchProcessorStep",
 ]

src/lerobot/processor/env_processor.py

@@ -153,35 +153,6 @@ class LiberoProcessorStep(ObservationProcessorStep):
         return result
 
 
-@dataclass
-@ProcessorStepRegistry.register(name="vlabench_processor")
-class VLABenchProcessorStep(ObservationProcessorStep):
-    """
-    Processes VLABench observations into the LeRobot format.
-
-    VLABench returns end-effector state as a 7-dim vector:
-    [pos_x, pos_y, pos_z, euler_x, euler_y, euler_z, gripper].
-    This is already in the format expected by the datasets, so we pass it
-    through as observation.state with no additional conversion.
-
-    Images are passed through without transformation (VLABench cameras
-    are already oriented correctly).
-    """
-
-    def _process_observation(self, observation):
-        """Processes observations from VLABench — minimal transform needed."""
-        return observation.copy()
-
-    def transform_features(
-        self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
-    ) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
-        """VLABench state is already in the expected format."""
-        return features
-
-    def observation(self, observation):
-        return self._process_observation(observation)
-
-
 @dataclass
 @ProcessorStepRegistry.register(name="isaaclab_arena_processor")
 class IsaaclabArenaProcessorStep(ObservationProcessorStep):