From bf03414b38c5892a99e1d8dc4bd38d85a5cee655 Mon Sep 17 00:00:00 2001 From: Pepijn Date: Mon, 2 Mar 2026 22:07:13 +0100 Subject: [PATCH] also have pipeline for feedback_features and action_features --- .../bi_openarm_follower.py | 2 +- .../robots/bi_so_follower/bi_so_follower.py | 2 +- .../robot_earthrover_mini_plus.py | 2 +- src/lerobot/robots/hope_jr/hope_jr_arm.py | 2 +- src/lerobot/robots/hope_jr/hope_jr_hand.py | 2 +- .../robots/koch_follower/koch_follower.py | 2 +- src/lerobot/robots/lekiwi/lekiwi.py | 2 +- src/lerobot/robots/lekiwi/lekiwi_client.py | 2 +- .../robots/omx_follower/omx_follower.py | 2 +- .../openarm_follower/openarm_follower.py | 2 +- src/lerobot/robots/reachy2/robot_reachy2.py | 2 +- src/lerobot/robots/robot.py | 35 ++++++++++++++----- src/lerobot/robots/so_follower/so_follower.py | 2 +- src/lerobot/robots/unitree_g1/unitree_g1.py | 2 +- .../bi_openarm_leader/bi_openarm_leader.py | 2 +- .../bi_so_leader/bi_so_leader.py | 2 +- .../teleoperators/gamepad/teleop_gamepad.py | 2 +- .../homunculus/homunculus_arm.py | 2 +- .../homunculus/homunculus_glove.py | 2 +- .../teleoperators/keyboard/teleop_keyboard.py | 2 +- .../teleoperators/koch_leader/koch_leader.py | 2 +- .../teleoperators/omx_leader/omx_leader.py | 2 +- .../openarm_leader/openarm_leader.py | 2 +- .../openarm_mini/openarm_mini.py | 2 +- .../teleoperators/phone/teleop_phone.py | 8 ++--- .../reachy2_teleoperator.py | 2 +- .../teleoperators/so_leader/so_leader.py | 2 +- src/lerobot/teleoperators/teleoperator.py | 34 +++++++++++++++--- .../teleoperators/unitree_g1/unitree_g1.py | 2 +- tests/test_robot_pipeline.py | 34 ++++++++++++++++-- 30 files changed, 117 insertions(+), 46 deletions(-) diff --git a/src/lerobot/robots/bi_openarm_follower/bi_openarm_follower.py b/src/lerobot/robots/bi_openarm_follower/bi_openarm_follower.py index ba0ae4663..17f18392e 100644 --- a/src/lerobot/robots/bi_openarm_follower/bi_openarm_follower.py +++ b/src/lerobot/robots/bi_openarm_follower/bi_openarm_follower.py @@ -106,7 +106,7 @@ class BiOpenArmFollower(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/bi_so_follower/bi_so_follower.py b/src/lerobot/robots/bi_so_follower/bi_so_follower.py index 56cc2f831..d1d57bbf5 100644 --- a/src/lerobot/robots/bi_so_follower/bi_so_follower.py +++ b/src/lerobot/robots/bi_so_follower/bi_so_follower.py @@ -90,7 +90,7 @@ class BiSOFollower(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/earthrover_mini_plus/robot_earthrover_mini_plus.py b/src/lerobot/robots/earthrover_mini_plus/robot_earthrover_mini_plus.py index fd0bdf143..b810fb331 100644 --- a/src/lerobot/robots/earthrover_mini_plus/robot_earthrover_mini_plus.py +++ b/src/lerobot/robots/earthrover_mini_plus/robot_earthrover_mini_plus.py @@ -184,7 +184,7 @@ class EarthRoverMiniPlus(Robot): } @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: """Define the action space. Returns: diff --git a/src/lerobot/robots/hope_jr/hope_jr_arm.py b/src/lerobot/robots/hope_jr/hope_jr_arm.py index d56c8c652..883d30b32 100644 --- a/src/lerobot/robots/hope_jr/hope_jr_arm.py +++ b/src/lerobot/robots/hope_jr/hope_jr_arm.py @@ -75,7 +75,7 @@ class HopeJrArm(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/hope_jr/hope_jr_hand.py b/src/lerobot/robots/hope_jr/hope_jr_hand.py index 2a1485754..26ee7f680 100644 --- a/src/lerobot/robots/hope_jr/hope_jr_hand.py +++ b/src/lerobot/robots/hope_jr/hope_jr_hand.py @@ -111,7 +111,7 @@ class HopeJrHand(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/koch_follower/koch_follower.py b/src/lerobot/robots/koch_follower/koch_follower.py index 453016e45..43fb5be8a 100644 --- a/src/lerobot/robots/koch_follower/koch_follower.py +++ b/src/lerobot/robots/koch_follower/koch_follower.py @@ -77,7 +77,7 @@ class KochFollower(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/lekiwi/lekiwi.py b/src/lerobot/robots/lekiwi/lekiwi.py index 39444a05b..09ed57445 100644 --- a/src/lerobot/robots/lekiwi/lekiwi.py +++ b/src/lerobot/robots/lekiwi/lekiwi.py @@ -102,7 +102,7 @@ class LeKiwi(Robot): return {**self._state_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._state_ft @property diff --git a/src/lerobot/robots/lekiwi/lekiwi_client.py b/src/lerobot/robots/lekiwi/lekiwi_client.py index c2055963c..ebc578b4e 100644 --- a/src/lerobot/robots/lekiwi/lekiwi_client.py +++ b/src/lerobot/robots/lekiwi/lekiwi_client.py @@ -102,7 +102,7 @@ class LeKiwiClient(Robot): return {**self._state_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._state_ft @property diff --git a/src/lerobot/robots/omx_follower/omx_follower.py b/src/lerobot/robots/omx_follower/omx_follower.py index f8030ef46..1fa740183 100644 --- a/src/lerobot/robots/omx_follower/omx_follower.py +++ b/src/lerobot/robots/omx_follower/omx_follower.py @@ -77,7 +77,7 @@ class OmxFollower(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/openarm_follower/openarm_follower.py b/src/lerobot/robots/openarm_follower/openarm_follower.py index 405dd92da..83a61e670 100644 --- a/src/lerobot/robots/openarm_follower/openarm_follower.py +++ b/src/lerobot/robots/openarm_follower/openarm_follower.py @@ -110,7 +110,7 @@ class OpenArmFollower(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: """Action features.""" return self._motors_ft diff --git a/src/lerobot/robots/reachy2/robot_reachy2.py b/src/lerobot/robots/reachy2/robot_reachy2.py index 0886de18b..b3bfe2f7a 100644 --- a/src/lerobot/robots/reachy2/robot_reachy2.py +++ b/src/lerobot/robots/reachy2/robot_reachy2.py @@ -99,7 +99,7 @@ class Reachy2Robot(Robot): return {**self.motors_features, **self.camera_features} @property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self.motors_features @property diff --git a/src/lerobot/robots/robot.py b/src/lerobot/robots/robot.py index 725a44e43..9e04b6af7 100644 --- a/src/lerobot/robots/robot.py +++ b/src/lerobot/robots/robot.py @@ -168,23 +168,40 @@ class Robot(abc.ABC): @property @abc.abstractmethod - def action_features(self) -> dict: + def raw_action_features(self) -> dict: """ - A dictionary describing the structure and types of the actions expected by the robot. - Its structure (keys) should match the structure of what is passed to - :pymeth:`send_action`. Values for the dict should be the type of the value if it's - a simple value, e.g. ``float`` for single proprioceptive value - (a joint's goal position/velocity). + Hardware-level action features (before any pipeline transformation). - For simple robots (no input pipeline), this returns motor-level features. - For robots with an IK pipeline, override this to return the pipeline's input spec - (e.g., EE features) so that compatibility checks work correctly. + A dictionary describing the structure and types of the actions accepted directly + by the robot hardware (i.e. what :pymeth:`_send_action` receives). Its structure + (keys) should match the structure of what is expected by :pymeth:`_send_action`. + Values should be the type of the value if it's a simple value, e.g. ``float`` for + single proprioceptive value (a joint's goal position/velocity). Note: this property should be able to be called regardless of whether the robot is connected or not. """ pass + @property + def action_features(self) -> dict: + """ + Pipeline-transformed action features. + + Applies input_pipeline().transform_features() to raw_action_features so the + returned dict reflects what the input pipeline outputs to hardware. + + Use raw_action_features to inspect hardware-level action feature shapes. + + Note: this property should be able to be called regardless of whether the robot + is connected or not. + """ + from lerobot.datasets.pipeline_features import create_initial_features # lazy import + + initial = create_initial_features(action=self.raw_action_features) + transformed = self.input_pipeline().transform_features(initial) + return transformed.get(PipelineFeatureType.ACTION, {}) + @property @abc.abstractmethod def is_connected(self) -> bool: diff --git a/src/lerobot/robots/so_follower/so_follower.py b/src/lerobot/robots/so_follower/so_follower.py index e0ad623ab..52b6a65b2 100644 --- a/src/lerobot/robots/so_follower/so_follower.py +++ b/src/lerobot/robots/so_follower/so_follower.py @@ -78,7 +78,7 @@ class SOFollower(Robot): return {**self._motors_ft, **self._cameras_ft} @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return self._motors_ft @property diff --git a/src/lerobot/robots/unitree_g1/unitree_g1.py b/src/lerobot/robots/unitree_g1/unitree_g1.py index 887c444e8..8cde2c2c2 100644 --- a/src/lerobot/robots/unitree_g1/unitree_g1.py +++ b/src/lerobot/robots/unitree_g1/unitree_g1.py @@ -170,7 +170,7 @@ class UnitreeG1(Robot): time.sleep(sleep_time) @cached_property - def action_features(self) -> dict[str, type]: + def raw_action_features(self) -> dict[str, type]: return {f"{G1_29_JointIndex(motor).name}.q": float for motor in G1_29_JointIndex} def calibrate(self) -> None: # robot is already calibrated diff --git a/src/lerobot/teleoperators/bi_openarm_leader/bi_openarm_leader.py b/src/lerobot/teleoperators/bi_openarm_leader/bi_openarm_leader.py index 539a433e4..875437f8c 100644 --- a/src/lerobot/teleoperators/bi_openarm_leader/bi_openarm_leader.py +++ b/src/lerobot/teleoperators/bi_openarm_leader/bi_openarm_leader.py @@ -82,7 +82,7 @@ class BiOpenArmLeader(Teleoperator): } @cached_property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/bi_so_leader/bi_so_leader.py b/src/lerobot/teleoperators/bi_so_leader/bi_so_leader.py index 9c72e2674..fb0a23453 100644 --- a/src/lerobot/teleoperators/bi_so_leader/bi_so_leader.py +++ b/src/lerobot/teleoperators/bi_so_leader/bi_so_leader.py @@ -65,7 +65,7 @@ class BiSOLeader(Teleoperator): } @cached_property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/gamepad/teleop_gamepad.py b/src/lerobot/teleoperators/gamepad/teleop_gamepad.py index 3b57ed54b..9efd32330 100644 --- a/src/lerobot/teleoperators/gamepad/teleop_gamepad.py +++ b/src/lerobot/teleoperators/gamepad/teleop_gamepad.py @@ -72,7 +72,7 @@ class GamepadTeleop(Teleoperator): } @property - def feedback_features(self) -> dict: + def raw_feedback_features(self) -> dict: return {} def connect(self) -> None: diff --git a/src/lerobot/teleoperators/homunculus/homunculus_arm.py b/src/lerobot/teleoperators/homunculus/homunculus_arm.py index 82a9aa6e7..412494506 100644 --- a/src/lerobot/teleoperators/homunculus/homunculus_arm.py +++ b/src/lerobot/teleoperators/homunculus/homunculus_arm.py @@ -85,7 +85,7 @@ class HomunculusArm(Teleoperator): return {f"{joint}.pos": float for joint in self.joints} @property - def feedback_features(self) -> dict: + def raw_feedback_features(self) -> dict: return {} @property diff --git a/src/lerobot/teleoperators/homunculus/homunculus_glove.py b/src/lerobot/teleoperators/homunculus/homunculus_glove.py index c42acb3e4..b8a13cbf1 100644 --- a/src/lerobot/teleoperators/homunculus/homunculus_glove.py +++ b/src/lerobot/teleoperators/homunculus/homunculus_glove.py @@ -111,7 +111,7 @@ class HomunculusGlove(Teleoperator): return {f"{joint}.pos": float for joint in self.joints} @property - def feedback_features(self) -> dict: + def raw_feedback_features(self) -> dict: return {} @property diff --git a/src/lerobot/teleoperators/keyboard/teleop_keyboard.py b/src/lerobot/teleoperators/keyboard/teleop_keyboard.py index 820d94e94..b5cf23665 100644 --- a/src/lerobot/teleoperators/keyboard/teleop_keyboard.py +++ b/src/lerobot/teleoperators/keyboard/teleop_keyboard.py @@ -75,7 +75,7 @@ class KeyboardTeleop(Teleoperator): } @property - def feedback_features(self) -> dict: + def raw_feedback_features(self) -> dict: return {} @property diff --git a/src/lerobot/teleoperators/koch_leader/koch_leader.py b/src/lerobot/teleoperators/koch_leader/koch_leader.py index 49351cbd3..185b09b45 100644 --- a/src/lerobot/teleoperators/koch_leader/koch_leader.py +++ b/src/lerobot/teleoperators/koch_leader/koch_leader.py @@ -62,7 +62,7 @@ class KochLeader(Teleoperator): return {f"{motor}.pos": float for motor in self.bus.motors} @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/omx_leader/omx_leader.py b/src/lerobot/teleoperators/omx_leader/omx_leader.py index a25427597..f7539a134 100644 --- a/src/lerobot/teleoperators/omx_leader/omx_leader.py +++ b/src/lerobot/teleoperators/omx_leader/omx_leader.py @@ -61,7 +61,7 @@ class OmxLeader(Teleoperator): return {f"{motor}.pos": float for motor in self.bus.motors} @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/openarm_leader/openarm_leader.py b/src/lerobot/teleoperators/openarm_leader/openarm_leader.py index 0e2879a25..4179574d3 100644 --- a/src/lerobot/teleoperators/openarm_leader/openarm_leader.py +++ b/src/lerobot/teleoperators/openarm_leader/openarm_leader.py @@ -75,7 +75,7 @@ class OpenArmLeader(Teleoperator): return features @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: """Feedback features (not implemented for OpenArms).""" return {} diff --git a/src/lerobot/teleoperators/openarm_mini/openarm_mini.py b/src/lerobot/teleoperators/openarm_mini/openarm_mini.py index 28c2cd1fb..df7f1d47c 100644 --- a/src/lerobot/teleoperators/openarm_mini/openarm_mini.py +++ b/src/lerobot/teleoperators/openarm_mini/openarm_mini.py @@ -131,7 +131,7 @@ class OpenArmMini(Teleoperator): return features @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/phone/teleop_phone.py b/src/lerobot/teleoperators/phone/teleop_phone.py index c9cbf2888..25b2c80be 100644 --- a/src/lerobot/teleoperators/phone/teleop_phone.py +++ b/src/lerobot/teleoperators/phone/teleop_phone.py @@ -56,9 +56,9 @@ class BasePhone: } @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: # No haptic or other feedback implemented yet - pass + return {} def configure(self) -> None: # No additional configuration required for phone teleop @@ -399,8 +399,8 @@ class Phone(Teleoperator): return self._phone_impl.raw_action_features @property - def feedback_features(self) -> dict[str, type]: - return self._phone_impl.feedback_features + def raw_feedback_features(self) -> dict[str, type]: + return self._phone_impl.raw_feedback_features def configure(self) -> None: return self._phone_impl.configure() diff --git a/src/lerobot/teleoperators/reachy2_teleoperator/reachy2_teleoperator.py b/src/lerobot/teleoperators/reachy2_teleoperator/reachy2_teleoperator.py index c2a94a216..fb0cfd5bf 100644 --- a/src/lerobot/teleoperators/reachy2_teleoperator/reachy2_teleoperator.py +++ b/src/lerobot/teleoperators/reachy2_teleoperator/reachy2_teleoperator.py @@ -120,7 +120,7 @@ class Reachy2Teleoperator(Teleoperator): return dict.fromkeys(self.joints_dict.keys(), float) @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/so_leader/so_leader.py b/src/lerobot/teleoperators/so_leader/so_leader.py index f5c861878..496d381d5 100644 --- a/src/lerobot/teleoperators/so_leader/so_leader.py +++ b/src/lerobot/teleoperators/so_leader/so_leader.py @@ -59,7 +59,7 @@ class SOLeader(Teleoperator): return {f"{motor}.pos": float for motor in self.bus.motors} @property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/src/lerobot/teleoperators/teleoperator.py b/src/lerobot/teleoperators/teleoperator.py index 3569acc2f..e2e85ba61 100644 --- a/src/lerobot/teleoperators/teleoperator.py +++ b/src/lerobot/teleoperators/teleoperator.py @@ -168,18 +168,42 @@ class Teleoperator(abc.ABC): @property @abc.abstractmethod - def feedback_features(self) -> dict: + def raw_feedback_features(self) -> dict: """ - A dictionary describing the structure and types of the feedback actions expected - by the teleoperator. Its structure (keys) should match the structure of what is - passed to :pymeth:`send_feedback`. Values should be the type of the value if it's - a simple value, e.g. ``float`` for single proprioceptive value. + Hardware-level feedback features (before any pipeline transformation). + + A dictionary describing the structure and types of the feedback accepted directly + by the teleoperator hardware (i.e. what :pymeth:`_send_feedback` receives). Its + structure (keys) should match the structure of what is expected by + :pymeth:`_send_feedback`. Values should be the type of the value if it's a simple + value, e.g. ``float`` for single proprioceptive value. + + Return an empty dict if this teleoperator does not support feedback. Note: this property should be able to be called regardless of whether the teleoperator is connected or not. """ pass + @property + def feedback_features(self) -> dict: + """ + Pipeline-transformed feedback features. + + Applies input_pipeline().transform_features() to raw_feedback_features so the + returned dict reflects what the input pipeline outputs to the teleoperator hardware. + + Use raw_feedback_features to inspect hardware-level feedback feature shapes. + + Note: this property should be able to be called regardless of whether the + teleoperator is connected or not. + """ + from lerobot.datasets.pipeline_features import create_initial_features # lazy import + + initial = create_initial_features(observation=self.raw_feedback_features) + transformed = self.input_pipeline().transform_features(initial) + return transformed.get(PipelineFeatureType.OBSERVATION, {}) + @property @abc.abstractmethod def is_connected(self) -> bool: diff --git a/src/lerobot/teleoperators/unitree_g1/unitree_g1.py b/src/lerobot/teleoperators/unitree_g1/unitree_g1.py index bf2941b6f..ec71c0138 100644 --- a/src/lerobot/teleoperators/unitree_g1/unitree_g1.py +++ b/src/lerobot/teleoperators/unitree_g1/unitree_g1.py @@ -76,7 +76,7 @@ class UnitreeG1Teleoperator(Teleoperator): return {f"{name}.q": float for name in self._g1_joint_names} @cached_property - def feedback_features(self) -> dict[str, type]: + def raw_feedback_features(self) -> dict[str, type]: return {} @property diff --git a/tests/test_robot_pipeline.py b/tests/test_robot_pipeline.py index 2f50b937f..345d8ece0 100644 --- a/tests/test_robot_pipeline.py +++ b/tests/test_robot_pipeline.py @@ -100,7 +100,7 @@ class MockRobot(Robot): return {**_JOINT_FEATURES, "camera": (480, 640, 3)} @property - def action_features(self) -> dict: + def raw_action_features(self) -> dict: return _JOINT_FEATURES @property @@ -147,7 +147,7 @@ class MockTeleop(Teleoperator): return _JOINT_FEATURES @property - def feedback_features(self) -> dict: + def raw_feedback_features(self) -> dict: return {} @property @@ -268,6 +268,24 @@ def test_robot_raw_observation_features_unchanged_after_pipeline(): assert "camera" in raw +def test_robot_action_features_identity_matches_raw(): + """action_features equals raw_action_features with identity input pipeline.""" + robot = MockRobot() + assert robot.action_features == robot.raw_action_features + + +def test_robot_raw_action_features_unchanged_after_pipeline(): + """raw_action_features is unaffected by any pipeline.""" + robot = MockRobot() + double_pipeline = RobotProcessorPipeline[tuple[RobotAction, RobotObservation], RobotAction]( + steps=[DoubleActionStep()], + to_transition=robot_action_observation_to_transition, + to_output=transition_to_robot_action, + ) + robot.set_input_pipeline(double_pipeline) + assert robot.raw_action_features == _JOINT_FEATURES + + def test_robot_set_output_pipeline_replaces_identity(): """set_output_pipeline replaces the default identity.""" robot = MockRobot() @@ -300,6 +318,18 @@ def test_teleop_action_features_identity_matches_raw(): assert teleop.action_features == teleop.raw_action_features +def test_teleop_feedback_features_identity_matches_raw(): + """feedback_features equals raw_feedback_features with identity input pipeline.""" + teleop = MockTeleop() + assert teleop.feedback_features == teleop.raw_feedback_features + + +def test_teleop_feedback_features_empty_when_raw_empty(): + """feedback_features returns empty dict when raw_feedback_features is empty.""" + teleop = MockTeleop() + assert teleop.feedback_features == {} + + def test_teleop_set_output_pipeline(): teleop = MockTeleop() p = _make_identity_teleop_action_pipeline()