diff --git a/.github/workflows/benchmark_tests.yml b/.github/workflows/benchmark_tests.yml index aea920485..36d9bee03 100644 --- a/.github/workflows/benchmark_tests.yml +++ b/.github/workflows/benchmark_tests.yml @@ -332,6 +332,12 @@ jobs: with: cache-binary: false + - name: Login to Docker Hub + uses: docker/login-action@v3 # zizmor: ignore[unpinned-uses] + with: + username: ${{ secrets.DOCKERHUB_LEROBOT_USERNAME }} + password: ${{ secrets.DOCKERHUB_LEROBOT_PASSWORD }} + - name: Build LIBERO-plus benchmark image uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] with: @@ -352,6 +358,7 @@ jobs: bash -c "hf auth login --token '$HF_USER_TOKEN' --add-to-git-credential && hf auth whoami" - name: Run LIBERO-plus smoke eval (1 episode) + if: env.HF_USER_TOKEN != '' run: | docker run --name libero-plus-eval --gpus all \ --shm-size=4g \ diff --git a/docker/Dockerfile.benchmark.libero_plus b/docker/Dockerfile.benchmark.libero_plus index 941260d4d..f39d644e5 100644 --- a/docker/Dockerfile.benchmark.libero_plus +++ b/docker/Dockerfile.benchmark.libero_plus @@ -44,7 +44,7 @@ RUN uv pip install --no-cache \ ENV LIBERO_PLUS_ROOT=/home/user_lerobot/libero-plus/libero/libero RUN git clone --depth=1 https://github.com/sylvestf/LIBERO-plus.git /home/user_lerobot/libero-plus \ && cd /home/user_lerobot/libero-plus && uv pip install --no-cache --no-deps -e "." \ - && uv pip uninstall hf-libero 2>/dev/null || true + && (uv pip uninstall hf-libero 2>/dev/null || true) ENV PYTHONPATH="/home/user_lerobot/libero-plus:${PYTHONPATH}" # Perturbation textures/scenes: bddl_base_domain.py resolves XMLs via @@ -57,8 +57,8 @@ from huggingface_hub import hf_hub_download; \ hf_hub_download(repo_id='Sylvest/LIBERO-plus', repo_type='dataset', \ filename='assets.zip', local_dir='/tmp/libero-plus-dl')" \ && unzip -q /tmp/libero-plus-dl/assets.zip -d /tmp/libero-plus-dl/extract \ - && mv /tmp/libero-plus-dl/extract/inspire/hdd/project/embodied-multimodality/public/syfei/libero_new/release/dataset/LIBERO-plus-0/assets \ - ${LIBERO_PLUS_ROOT}/assets \ + && ASSETS_DIR=$(find /tmp/libero-plus-dl/extract -type d -name assets | head -1) \ + && mv "${ASSETS_DIR}" ${LIBERO_PLUS_ROOT}/assets \ && rm -rf /tmp/libero-plus-dl # Point ~/.libero/config.yaml at the clone so LIBERO-plus's imports are diff --git a/docs/source/libero_plus.mdx b/docs/source/libero_plus.mdx index 620aad7ec..7f5e07abb 100644 --- a/docs/source/libero_plus.mdx +++ b/docs/source/libero_plus.mdx @@ -2,10 +2,12 @@ LIBERO-plus is a **robustness benchmark** for Vision-Language-Action (VLA) models built on top of [LIBERO](./libero). It systematically stress-tests policies by applying **seven independent perturbation dimensions** to the original LIBERO task set, exposing failure modes that standard benchmarks miss. -- Paper: [LIBERO-plus: A Robustness Benchmark for VLA Models](https://github.com/sylvestf/LIBERO-plus) +- Paper: [In-depth Robustness Analysis of Vision-Language-Action Models](https://arxiv.org/abs/2510.13626) - GitHub: [sylvestf/LIBERO-plus](https://github.com/sylvestf/LIBERO-plus) - Dataset: [lerobot/libero_plus](https://huggingface.co/datasets/lerobot/libero_plus) +![An overview of the LIBERO-plus benchmark perturbation dimensions](https://github.com/sylvestf/LIBERO-plus/raw/main/media/overview.png) + ## Perturbation dimensions LIBERO-plus creates ~10 000 task variants by perturbing each original LIBERO task along these axes: @@ -24,13 +26,20 @@ LIBERO-plus creates ~10 000 task variants by perturbing each original LIBERO tas LIBERO-plus covers the same five suites as LIBERO: -| Suite | CLI name | Tasks | Max steps | -| -------------- | ---------------- | ----- | --------- | -| LIBERO-Spatial | `libero_spatial` | 10 | 280 | -| LIBERO-Object | `libero_object` | 10 | 280 | -| LIBERO-Goal | `libero_goal` | 10 | 300 | -| LIBERO-90 | `libero_90` | 90 | 400 | -| LIBERO-Long | `libero_10` | 10 | 520 | +| Suite | CLI name | Tasks | Max steps | Description | +| -------------- | ---------------- | ----- | --------- | -------------------------------------------------- | +| LIBERO-Spatial | `libero_spatial` | 10 | 280 | Tasks requiring reasoning about spatial relations | +| LIBERO-Object | `libero_object` | 10 | 280 | Tasks centered on manipulating different objects | +| LIBERO-Goal | `libero_goal` | 10 | 300 | Goal-conditioned tasks with changing targets | +| LIBERO-90 | `libero_90` | 90 | 400 | Short-horizon tasks from the LIBERO-100 collection | +| LIBERO-Long | `libero_10` | 10 | 520 | Long-horizon tasks from the LIBERO-100 collection | + + + Installing LIBERO-plus **replaces** vanilla LIBERO — it uninstalls `hf-libero` + so that `import libero` resolves to the LIBERO-plus fork. You cannot have both + installed at the same time. To switch back to vanilla LIBERO, uninstall the + fork and reinstall with `pip install -e ".[libero]"`. + ## Installation @@ -72,72 +81,78 @@ python -c "import libero; print(libero.__file__)" ## Evaluation -### Minimal smoke-test (1 episode, no async) +### Default evaluation (recommended) + +Evaluate across the four standard suites (10 episodes per task): ```bash lerobot-eval \ - --policy.path=pepijn223/smolvla_libero_plus \ - --env.type=libero_plus \ - --env.task=libero_spatial \ - --eval.batch_size=1 \ - --eval.n_episodes=1 \ - --eval.use_async_envs=false \ - --policy.device=cuda \ - --env.camera_name_mapping='{"agentview_image": "camera1", "robot0_eye_in_hand_image": "camera2"}' \ - --policy.empty_cameras=1 + --policy.path="your-policy-id" \ + --env.type=libero_plus \ + --env.task=libero_spatial,libero_object,libero_goal,libero_10 \ + --eval.batch_size=1 \ + --eval.n_episodes=10 \ + --env.max_parallel_tasks=1 ``` -### Full robustness benchmark (recommended) +### Single-suite evaluation + +Evaluate on one LIBERO-plus suite: ```bash lerobot-eval \ - --policy.path= \ - --env.type=libero_plus \ - --env.task=libero_spatial,libero_object,libero_goal,libero_10 \ - --eval.batch_size=1 \ - --eval.n_episodes=10 \ - --env.max_parallel_tasks=1 + --policy.path="your-policy-id" \ + --env.type=libero_plus \ + --env.task=libero_spatial \ + --eval.batch_size=1 \ + --eval.n_episodes=10 ``` -### Key CLI flags +- `--env.task` picks the suite (`libero_spatial`, `libero_object`, etc.). +- `--env.task_ids` restricts to specific task indices (`[0]`, `[1,2,3]`, etc.). Omit to run all tasks in the suite. +- `--eval.batch_size` controls how many environments run in parallel. +- `--eval.n_episodes` sets how many episodes to run per task. -| Flag | Description | -| --------------------------- | ---------------------------------------------------------------- | -| `--env.type=libero_plus` | Selects LIBERO-plus environment (same gym interface as `libero`) | -| `--env.task` | Suite name(s), comma-separated | -| `--env.task_ids` | Restrict to specific task indices, e.g. `[0,1,2]` | -| `--env.camera_name_mapping` | JSON dict remapping raw camera names to policy input keys | -| `--env.control_mode` | `relative` (default) or `absolute` | -| `--eval.use_async_envs` | `true` for parallel rollouts (default), `false` for debugging | -| `--policy.empty_cameras` | Number of camera slots without observations (policy-specific) | +### Multi-suite evaluation -### Camera name mapping - -By default, LIBERO cameras are mapped as: - -| Raw camera name | LeRobot key | -| -------------------------- | --------------------------- | -| `agentview_image` | `observation.images.image` | -| `robot0_eye_in_hand_image` | `observation.images.image2` | - -If your policy was trained with different key names, pass a JSON remapping: +Benchmark a policy across multiple suites at once by passing a comma-separated list: ```bash ---env.camera_name_mapping='{"agentview_image": "camera1", "robot0_eye_in_hand_image": "camera2"}' +lerobot-eval \ + --policy.path="your-policy-id" \ + --env.type=libero_plus \ + --env.task=libero_spatial,libero_object \ + --eval.batch_size=1 \ + --eval.n_episodes=10 ``` -## Policy inputs and outputs +### Control mode -**Observations (after `LiberoProcessorStep`):** +LIBERO-plus supports two control modes — `relative` (default) and `absolute`. Different VLA checkpoints are trained with different action parameterizations, so make sure the mode matches your policy: -- `observation.state` — 8-dim proprioceptive vector: `[eef_pos(3), eef_axis_angle(3), gripper_qpos(2)]` -- `observation.images.` — camera image(s), flipped 180° to match VLA convention +```bash +--env.control_mode=relative # or "absolute" +``` + +### Policy inputs and outputs + +**Observations:** + +- `observation.state` — 8-dim proprioceptive features (eef position, axis-angle orientation, gripper qpos) +- `observation.images.image` — main camera view (`agentview_image`), HWC uint8 +- `observation.images.image2` — wrist camera view (`robot0_eye_in_hand_image`), HWC uint8 **Actions:** -- `Box(-1, 1, shape=(7,))` — 6D end-effector delta + 1D gripper +- Continuous control in `Box(-1, 1, shape=(7,))` — 6D end-effector delta + 1D gripper -## Dataset +### Recommended evaluation episodes + +For reproducible benchmarking, use **10 episodes per task** across all four standard suites (Spatial, Object, Goal, Long). This gives 400 total episodes and matches the protocol used for published results. + +## Training + +### Dataset A LeRobot-format training dataset for LIBERO-plus is available at: @@ -169,6 +184,5 @@ LIBERO-plus is a drop-in extension of LIBERO: - Same camera names and observation/action format - Same task suite names - Installs under the same `libero` Python package name (different GitHub repo) -- The only code difference in LeRobot is a try/except import fallback in `libero.py` that handles the slightly different package nesting in LIBERO-plus To use the original LIBERO benchmark, see [LIBERO](./libero) and use `--env.type=libero`. diff --git a/src/lerobot/envs/configs.py b/src/lerobot/envs/configs.py index 793197710..d5c8839af 100644 --- a/src/lerobot/envs/configs.py +++ b/src/lerobot/envs/configs.py @@ -331,6 +331,7 @@ class LiberoEnv(EnvConfig): camera_name_mapping: dict[str, str] | None = None observation_height: int = 360 observation_width: int = 360 + is_libero_plus: bool = False features: dict[str, PolicyFeature] = field( default_factory=lambda: { ACTION: PolicyFeature(type=FeatureType.ACTION, shape=(7,)), @@ -432,6 +433,7 @@ class LiberoEnv(EnvConfig): control_mode=self.control_mode, episode_length=self.episode_length, camera_name_mapping=self.camera_name_mapping, + is_libero_plus=self.is_libero_plus, ) def get_env_processors(self): @@ -597,3 +599,4 @@ class LiberoPlusEnv(LiberoEnv): """ task: str = "libero_spatial" + is_libero_plus: bool = True diff --git a/src/lerobot/envs/libero.py b/src/lerobot/envs/libero.py index f67be5b91..bfd5eb469 100644 --- a/src/lerobot/envs/libero.py +++ b/src/lerobot/envs/libero.py @@ -77,18 +77,24 @@ def _select_task_ids(total_tasks: int, task_ids: Iterable[int] | None) -> list[i _LIBERO_PERTURBATION_SUFFIX_RE = re.compile(r"_(?:language|view|light)_[^.]*|_(?:table|tb)_\d+") -def get_task_init_states(task_suite: Any, i: int) -> np.ndarray: +def get_task_init_states(task_suite: Any, i: int, is_libero_plus: bool = False) -> np.ndarray: task = task_suite.tasks[i] filename = Path(task.init_states_file) root = Path(get_libero_path("init_states")) - # `_add_` / `_level` variants store extra-object layouts under libero_newobj/ - # as a flat array that must be reshaped to (1, -1). + if not is_libero_plus: + init_states_path = root / task.problem_folder / filename.name + return torch.load(init_states_path, weights_only=False) # nosec B614 + + # LIBERO-plus: `_add_` / `_level` variants store extra-object layouts under + # libero_newobj/ as a flat array that must be reshaped to (1, -1). if "_add_" in filename.name or "_level" in filename.name: init_states_path = root / "libero_newobj" / task.problem_folder / filename.name init_states = torch.load(init_states_path, weights_only=False) # nosec B614 return init_states.reshape(1, -1) + # LIBERO-plus perturbation variants encode the perturbation in the filename + # but on disk only the base `.pruned_init` exists — strip the suffix to match. stripped = _LIBERO_PERTURBATION_SUFFIX_RE.sub("", filename.stem) + filename.suffix init_states_path = root / task.problem_folder / stripped return torch.load(init_states_path, weights_only=False) # nosec B614 @@ -133,9 +139,11 @@ class LiberoEnv(gym.Env): camera_name_mapping: dict[str, str] | None = None, num_steps_wait: int = 10, control_mode: str = "relative", + is_libero_plus: bool = False, ): super().__init__() self.task_id = task_id + self.is_libero_plus = is_libero_plus self.obs_type = obs_type self.render_mode = render_mode self.observation_width = observation_width @@ -162,7 +170,11 @@ class LiberoEnv(gym.Env): self.episode_index = episode_index self.episode_length = episode_length # Load once and keep - self._init_states = get_task_init_states(task_suite, self.task_id) if self.init_states else None + self._init_states = ( + get_task_init_states(task_suite, self.task_id, is_libero_plus=self.is_libero_plus) + if self.init_states + else None + ) self._reset_stride = n_envs # when performing a reset, append `_reset_stride` to `init_state_id`. self.init_state_id = self.episode_index # tie each sub-env to a fixed init state @@ -395,6 +407,7 @@ def _make_env_fns( gym_kwargs: Mapping[str, Any], control_mode: str, camera_name_mapping: dict[str, str] | None = None, + is_libero_plus: bool = False, ) -> list[Callable[[], LiberoEnv]]: """Build n_envs factory callables for a single (suite, task_id).""" @@ -411,6 +424,7 @@ def _make_env_fns( n_envs=n_envs, control_mode=control_mode, camera_name_mapping=camera_name_mapping, + is_libero_plus=is_libero_plus, **local_kwargs, ) @@ -433,6 +447,7 @@ def create_libero_envs( control_mode: str = "relative", episode_length: int | None = None, camera_name_mapping: dict[str, str] | None = None, + is_libero_plus: bool = False, ) -> dict[str, dict[int, Any]]: """ Create vectorized LIBERO environments with a consistent return shape. @@ -490,6 +505,7 @@ def create_libero_envs( gym_kwargs=gym_kwargs, control_mode=control_mode, camera_name_mapping=camera_name_mapping, + is_libero_plus=is_libero_plus, ) if is_async: lazy = _LazyAsyncVectorEnv(fns, cached_obs_space, cached_act_space)