diff --git a/docs/source/evo1.mdx b/docs/source/evo1.mdx index 33d8d8ce1..d768103f6 100644 --- a/docs/source/evo1.mdx +++ b/docs/source/evo1.mdx @@ -159,14 +159,18 @@ pixel embeddings, VLM fused tokens, normalized actions, and denormalized actions (`max_abs_diff=0.0`). The published checkpoint expects the raw LIBERO camera feature names -`observation.images.agentview_image` and `observation.images.robot0_eye_in_hand_image`. To run the converted -checkpoint with LeRobot LIBERO evaluation for the same one-episode-per-task setting, keep those camera names -instead of the default `image`/`image2` mapping: +`observation.images.agentview_image` and `observation.images.robot0_eye_in_hand_image`. The official EVO1 LIBERO +rollout protocol also replans every 14 actions and binarizes the gripper command before stepping the simulator. +The LIBERO environment postprocessor applies the gripper binarization automatically for EVO1 policies. To run the +converted checkpoint with LeRobot LIBERO evaluation for the same one-episode-per-task setting, keep the raw camera +names instead of the default `image`/`image2` mapping and override `policy.n_action_steps` to 14: ```bash lerobot-eval \ --policy.path=javadcc/evo1-libero-lerobot \ + --policy.vlm_model_name=OpenGVLab/InternVL3-1B \ --policy.device=cuda \ + --policy.n_action_steps=14 \ --env.type=libero \ --env.task=libero_object \ --env.camera_name_mapping="{agentview_image: agentview_image, robot0_eye_in_hand_image: robot0_eye_in_hand_image}" \ diff --git a/src/lerobot/envs/configs.py b/src/lerobot/envs/configs.py index 2e4de8c51..0807641f4 100644 --- a/src/lerobot/envs/configs.py +++ b/src/lerobot/envs/configs.py @@ -357,6 +357,7 @@ class LiberoEnv(EnvConfig): } ) control_mode: str = "relative" # or "absolute" + binarize_gripper: bool | None = None def __post_init__(self): if self.obs_type == "pixels": @@ -442,12 +443,21 @@ class LiberoEnv(EnvConfig): ) def get_env_processors(self, policy_cfg: Any | None = None): - max_state_dim = getattr(policy_cfg, "max_state_dim", None) if getattr(policy_cfg, "type", None) == "evo1" else None + is_evo1 = getattr(policy_cfg, "type", None) == "evo1" + max_state_dim = getattr(policy_cfg, "max_state_dim", None) if is_evo1 else None action_feature = self.features.get(ACTION) action_dim = int(action_feature.shape[0]) if action_feature is not None else 7 + binarize_gripper = is_evo1 if self.binarize_gripper is None else self.binarize_gripper return ( PolicyProcessorPipeline(steps=[LiberoProcessorStep(max_state_dim=max_state_dim)]), - PolicyProcessorPipeline(steps=[LiberoActionProcessorStep(action_dim=action_dim)]), + PolicyProcessorPipeline( + steps=[ + LiberoActionProcessorStep( + action_dim=action_dim, + binarize_gripper=binarize_gripper, + ) + ] + ), ) diff --git a/src/lerobot/processor/env_processor.py b/src/lerobot/processor/env_processor.py index c8287b1aa..4369a81af 100644 --- a/src/lerobot/processor/env_processor.py +++ b/src/lerobot/processor/env_processor.py @@ -173,13 +173,42 @@ class LiberoActionProcessorStep(ActionProcessorStep): """Slices padded policy actions back to the executable LIBERO action space.""" action_dim: int = 7 + binarize_gripper: bool = False + gripper_index: int = 6 + gripper_threshold: float = 0.5 + gripper_below_threshold_value: float = 1.0 + gripper_above_threshold_value: float = -1.0 def action(self, action): if action.shape[-1] < self.action_dim: raise ValueError( f"LIBERO action has {action.shape[-1]} dims, which is smaller than action_dim={self.action_dim}." ) - return action[..., : self.action_dim] + action = action[..., : self.action_dim] + if not self.binarize_gripper: + return action + + if not 0 <= self.gripper_index < self.action_dim: + raise ValueError( + f"gripper_index={self.gripper_index} must be within sliced action_dim={self.action_dim}." + ) + action = action.clone() + below = torch.as_tensor( + self.gripper_below_threshold_value, + dtype=action.dtype, + device=action.device, + ) + above = torch.as_tensor( + self.gripper_above_threshold_value, + dtype=action.dtype, + device=action.device, + ) + action[..., self.gripper_index] = torch.where( + action[..., self.gripper_index] > self.gripper_threshold, + above, + below, + ) + return action def transform_features( self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]] @@ -190,7 +219,14 @@ class LiberoActionProcessorStep(ActionProcessorStep): return new_features def get_config(self) -> dict: - return {"action_dim": self.action_dim} + return { + "action_dim": self.action_dim, + "binarize_gripper": self.binarize_gripper, + "gripper_index": self.gripper_index, + "gripper_threshold": self.gripper_threshold, + "gripper_below_threshold_value": self.gripper_below_threshold_value, + "gripper_above_threshold_value": self.gripper_above_threshold_value, + } @dataclass diff --git a/tests/envs/test_dispatch.py b/tests/envs/test_dispatch.py index e26f34690..eeed9d1aa 100644 --- a/tests/envs/test_dispatch.py +++ b/tests/envs/test_dispatch.py @@ -99,12 +99,18 @@ def test_libero_evo1_processors_use_padded_state_and_env_action_dim(): assert pre.steps[0].max_state_dim == 24 assert isinstance(post.steps[0], LiberoActionProcessorStep) assert post.steps[0].action_dim == cfg.features["action"].shape[0] == 7 + assert post.steps[0].binarize_gripper is True class _OtherConfig: type = "other" - pre_other, _ = make_env_pre_post_processors(cfg, policy_cfg=_OtherConfig()) + pre_other, post_other = make_env_pre_post_processors(cfg, policy_cfg=_OtherConfig()) assert pre_other.steps[0].max_state_dim is None + assert post_other.steps[0].binarize_gripper is False + + cfg.binarize_gripper = False + _, post_disabled = make_env_pre_post_processors(cfg, policy_cfg=_Evo1Config()) + assert post_disabled.steps[0].binarize_gripper is False def test_libero_processor_pads_state_to_max_dim(): @@ -138,6 +144,24 @@ def test_libero_action_processor_slices_padded_action(): step.action(torch.zeros(2, 6)) +def test_libero_action_processor_can_binarize_gripper(): + step = LiberoActionProcessorStep(action_dim=7, binarize_gripper=True) + action = torch.tensor( + [ + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 0.5, 7.0], + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 0.6, 7.0], + ], + dtype=torch.float32, + ) + + processed = step.action(action) + + assert processed.shape == (2, 7) + assert torch.equal(processed[:, :6], action[:, :6]) + assert torch.equal(processed[:, 6], torch.tensor([1.0, -1.0])) + assert torch.equal(action[:, 6], torch.tensor([0.5, 0.6])) + + def test_base_create_envs(): """Base class create_envs() should build a single-task VectorEnv via gym.make().""" gym_id = "_dispatch_test/CartPole-v99"