From 34588453ef5213176f81b757b5fc61a90742cc10 Mon Sep 17 00:00:00 2001 From: Pepijn Date: Mon, 20 Apr 2026 10:25:37 +0200 Subject: [PATCH] fix: integrate PR #3396 review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - processor(vlabench): drop the no-op `VLABenchProcessorStep` and let `VLABenchEnv` fall back to the identity `get_env_processors` default. Removes the class, its export, and its override. - envs(vlabench): hoist `import cv2` to module scope; propagate the gym `seed` into the inner dm_control env (re-seeding `task.random` and the env's `_random_state`) so layout sampling is reproducible. - docs(vlabench): mirror the CI eval command across all four snippets (`--eval.use_async_envs=false`, `--policy.device=cuda`, and the `rename_map` for image/second_image/wrist_image → camera1/2/3). Intro now calls out that LeRobot exposes 43 of VLABench's 100 upstream task categories via `--env.task`. - docker(vlabench): drop the `|| echo WARN` fallback on the gdown asset-download path so a genuine download failure surfaces instead of being masked. Co-Authored-By: Claude Opus 4.7 (1M context) --- docker/Dockerfile.benchmark.vlabench | 5 ++--- docs/source/vlabench.mdx | 24 ++++++++++++++++----- src/lerobot/envs/configs.py | 8 ------- src/lerobot/envs/vlabench.py | 19 +++++++++++++++-- src/lerobot/processor/__init__.py | 3 +-- src/lerobot/processor/env_processor.py | 29 -------------------------- 6 files changed, 39 insertions(+), 49 deletions(-) diff --git a/docker/Dockerfile.benchmark.vlabench b/docker/Dockerfile.benchmark.vlabench index 6f3093f71..bb2b7c6ea 100644 --- a/docker/Dockerfile.benchmark.vlabench +++ b/docker/Dockerfile.benchmark.vlabench @@ -68,9 +68,8 @@ p = snapshot_download(repo_id='${VLABENCH_ASSETS_REPO}', repo_type='dataset', \ local_dir='${ASSETS_DIR}', allow_patterns=['obj/**', 'scenes/**']); \ print('snapshot_download returned:', p)"; \ else \ - echo "No VLABENCH_ASSETS_REPO set — falling back to gdown (best-effort)" && \ - python ~/VLABench/scripts/download_assets.py --choice all || \ - echo "WARN: VLABench asset download failed (likely Google Drive quota)."; \ + echo "No VLABENCH_ASSETS_REPO set — falling back to gdown" && \ + python ~/VLABench/scripts/download_assets.py --choice all; \ fi && \ python -c "\ from pathlib import Path; \ diff --git a/docs/source/vlabench.mdx b/docs/source/vlabench.mdx index 3036cc933..cdb85eb84 100644 --- a/docs/source/vlabench.mdx +++ b/docs/source/vlabench.mdx @@ -1,6 +1,6 @@ # VLABench -[VLABench](https://github.com/OpenMOSS/VLABench) is a large-scale benchmark for **language-conditioned robotic manipulation with long-horizon reasoning**. It covers 100 task categories across 2,000+ objects and evaluates six dimensions of robot intelligence: mesh & texture understanding, spatial reasoning, world-knowledge transfer, semantic instruction comprehension, physical-law understanding, and long-horizon planning. Built on MuJoCo / dm_control with a Franka Panda 7-DOF arm. +[VLABench](https://github.com/OpenMOSS/VLABench) is a large-scale benchmark for **language-conditioned robotic manipulation with long-horizon reasoning**. The upstream suite covers 100 task categories across 2,000+ objects and evaluates six dimensions of robot intelligence: mesh & texture understanding, spatial reasoning, world-knowledge transfer, semantic instruction comprehension, physical-law understanding, and long-horizon planning. Built on MuJoCo / dm_control with a Franka Panda 7-DOF arm. LeRobot exposes **43 of these tasks** through `--env.task` (21 primitives + 22 composites, see [Available tasks](#available-tasks) below). - Paper: [VLABench: A Large-Scale Benchmark for Language-Conditioned Robotics Manipulation with Long-Horizon Reasoning](https://arxiv.org/abs/2412.18194) - GitHub: [OpenMOSS/VLABench](https://github.com/OpenMOSS/VLABench) @@ -53,6 +53,8 @@ export MUJOCO_GL=egl # for headless servers (HPC, cloud) ## Evaluation +All eval snippets below mirror the command CI runs (see `.github/workflows/benchmark_tests.yml`). The `--rename_map` argument maps VLABench's `image` / `second_image` / `wrist_image` camera keys onto the three-camera (`camera1` / `camera2` / `camera3`) input layout the released `smolvla_vlabench` policy was trained on. + ### Single-task evaluation (recommended for quick iteration) ```bash @@ -61,7 +63,10 @@ lerobot-eval \ --env.type=vlabench \ --env.task=select_fruit \ --eval.batch_size=1 \ - --eval.n_episodes=10 + --eval.n_episodes=10 \ + --eval.use_async_envs=false \ + --policy.device=cuda \ + '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}' ``` ### Multi-task evaluation @@ -74,7 +79,10 @@ lerobot-eval \ --env.type=vlabench \ --env.task=select_fruit,select_toy,add_condiment,heat_food \ --eval.batch_size=1 \ - --eval.n_episodes=10 + --eval.n_episodes=10 \ + --eval.use_async_envs=false \ + --policy.device=cuda \ + '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}' ``` ### Suite-wide evaluation @@ -88,7 +96,10 @@ lerobot-eval \ --env.task=primitive \ --eval.batch_size=1 \ --eval.n_episodes=10 \ - --env.max_parallel_tasks=1 + --eval.use_async_envs=false \ + --policy.device=cuda \ + --env.max_parallel_tasks=1 \ + '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}' ``` Or both suites: @@ -100,7 +111,10 @@ lerobot-eval \ --env.task=primitive,composite \ --eval.batch_size=1 \ --eval.n_episodes=10 \ - --env.max_parallel_tasks=1 + --eval.use_async_envs=false \ + --policy.device=cuda \ + --env.max_parallel_tasks=1 \ + '--rename_map={"observation.images.image": "observation.images.camera1", "observation.images.second_image": "observation.images.camera2", "observation.images.wrist_image": "observation.images.camera3"}' ``` ### Recommended evaluation episodes diff --git a/src/lerobot/envs/configs.py b/src/lerobot/envs/configs.py index 044d94bb9..d36f68993 100644 --- a/src/lerobot/envs/configs.py +++ b/src/lerobot/envs/configs.py @@ -560,14 +560,6 @@ class VLABenchEnv(EnvConfig): env_cls=env_cls, ) - def get_env_processors(self): - from lerobot.processor.env_processor import VLABenchProcessorStep - - return ( - PolicyProcessorPipeline(steps=[VLABenchProcessorStep()]), - PolicyProcessorPipeline(steps=[]), - ) - @EnvConfig.register_subclass("isaaclab_arena") @dataclass diff --git a/src/lerobot/envs/vlabench.py b/src/lerobot/envs/vlabench.py index 821b3dc22..4e1d660a4 100644 --- a/src/lerobot/envs/vlabench.py +++ b/src/lerobot/envs/vlabench.py @@ -31,6 +31,7 @@ from collections.abc import Callable, Sequence from functools import partial from typing import Any +import cv2 import gymnasium as gym import numpy as np from gymnasium import spaces @@ -246,8 +247,6 @@ class VLABenchEnv(gym.Env): def _to_hwc3(arr: np.ndarray) -> np.ndarray: """Coerce any camera array to the declared (h, w, 3) uint8 shape.""" - import cv2 - a = np.asarray(arr) # Drop a leading singleton batch dim if present. while a.ndim > 3 and a.shape[0] == 1: @@ -392,12 +391,28 @@ class VLABenchEnv(gym.Env): assert self._env is not None super().reset(seed=seed) + if seed is not None: + self._seed_inner_env(int(self.np_random.integers(0, 2**31 - 1))) + self._env.reset() observation = self._get_obs() info = {"is_success": False} return observation, info + def _seed_inner_env(self, seed: int) -> None: + """Propagate `seed` to the inner dm_control env. `Environment.reset()` + doesn't accept a seed, so we re-seed the task and environment + `RandomState`s directly. Best-effort: silently skipped when the + expected attributes are absent on a given VLABench version. + """ + for owner_attr, rng_attr in (("task", "random"), (None, "_random_state")): + owner = getattr(self._env, owner_attr) if owner_attr else self._env + rng = getattr(owner, rng_attr, None) + rng_seed = getattr(rng, "seed", None) + if callable(rng_seed): + rng_seed(seed) + def step(self, action: np.ndarray) -> tuple[RobotObservation, float, bool, bool, dict[str, Any]]: from dm_control.rl.control import PhysicsError # type: ignore[import-untyped] diff --git a/src/lerobot/processor/__init__.py b/src/lerobot/processor/__init__.py index 21acd8db2..3688a4b8c 100644 --- a/src/lerobot/processor/__init__.py +++ b/src/lerobot/processor/__init__.py @@ -40,7 +40,7 @@ from .converters import ( ) from .delta_action_processor import MapDeltaActionToRobotActionStep, MapTensorToDeltaActionDictStep from .device_processor import DeviceProcessorStep -from .env_processor import IsaaclabArenaProcessorStep, LiberoProcessorStep, VLABenchProcessorStep +from .env_processor import IsaaclabArenaProcessorStep, LiberoProcessorStep from .factory import ( make_default_processors, make_default_robot_action_processor, @@ -165,5 +165,4 @@ __all__ = [ "to_relative_actions", "UnnormalizerProcessorStep", "VanillaObservationProcessorStep", - "VLABenchProcessorStep", ] diff --git a/src/lerobot/processor/env_processor.py b/src/lerobot/processor/env_processor.py index 3c7645863..75cbb79de 100644 --- a/src/lerobot/processor/env_processor.py +++ b/src/lerobot/processor/env_processor.py @@ -153,35 +153,6 @@ class LiberoProcessorStep(ObservationProcessorStep): return result -@dataclass -@ProcessorStepRegistry.register(name="vlabench_processor") -class VLABenchProcessorStep(ObservationProcessorStep): - """ - Processes VLABench observations into the LeRobot format. - - VLABench returns end-effector state as a 7-dim vector: - [pos_x, pos_y, pos_z, euler_x, euler_y, euler_z, gripper]. - This is already in the format expected by the datasets, so we pass it - through as observation.state with no additional conversion. - - Images are passed through without transformation (VLABench cameras - are already oriented correctly). - """ - - def _process_observation(self, observation): - """Processes observations from VLABench — minimal transform needed.""" - return observation.copy() - - def transform_features( - self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]] - ) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]: - """VLABench state is already in the expected format.""" - return features - - def observation(self, observation): - return self._process_observation(observation) - - @dataclass @ProcessorStepRegistry.register(name="isaaclab_arena_processor") class IsaaclabArenaProcessorStep(ObservationProcessorStep):