mirror of
https://github.com/huggingface/lerobot.git
synced 2026-08-08 17:39:44 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c073482202 |
@@ -28,231 +28,3 @@ See [Phone teleoperation](../phone_teleop) and [Isaac Teleop](../isaac_teleop) f
|
||||
## make_teleoperator_from_config
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.make_teleoperator_from_config
|
||||
|
||||
## SO-100 and SO-101 leaders
|
||||
|
||||
`SO100Leader` and `SO101Leader` are aliases of the same `SOLeader` class; the two arms differ in their
|
||||
configuration, not their control code. `SO100LeaderConfig` and `SO101LeaderConfig` are likewise aliases of
|
||||
`SOLeaderTeleopConfig`.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.so_leader.SOLeader
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.so_leader.SOLeaderTeleopConfig
|
||||
|
||||
## KochLeader
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.koch_leader.KochLeader
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.koch_leader.KochLeaderConfig
|
||||
|
||||
## OmxLeader
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.omx_leader.OmxLeader
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.omx_leader.OmxLeaderConfig
|
||||
|
||||
## OpenArmLeader
|
||||
|
||||
CAN-based leader arm using Damiao motors.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.openarm_leader.OpenArmLeader
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.openarm_leader.OpenArmLeaderConfig
|
||||
|
||||
## BiOpenArmLeader
|
||||
|
||||
A bimanual pair of `OpenArmLeader` arms.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_openarm_leader.BiOpenArmLeader
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_openarm_leader.BiOpenArmLeaderConfig
|
||||
|
||||
## OpenArmMini
|
||||
|
||||
CAN-based leader arm using Damiao motors, a smaller/simpler OpenArm variant.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.openarm_mini.OpenArmMini
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.openarm_mini.OpenArmMiniConfig
|
||||
|
||||
## BiOpenArmMini
|
||||
|
||||
A bimanual pair of `OpenArmMini` arms.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_openarm_mini.BiOpenArmMini
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_openarm_mini.BiOpenArmMiniConfig
|
||||
|
||||
## HomunculusArm
|
||||
|
||||
A wearable exoskeleton arm read over a serial link.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.homunculus.HomunculusArm
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.homunculus.HomunculusArmConfig
|
||||
|
||||
## HomunculusGlove
|
||||
|
||||
A wearable exoskeleton glove read over a serial link, remapped to HopeJR hand joints via
|
||||
`homunculus_glove_to_hope_jr_hand`.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.homunculus.HomunculusGlove
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.homunculus.HomunculusGloveConfig
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.homunculus.homunculus_glove_to_hope_jr_hand
|
||||
|
||||
## RebotArm102Leader
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.rebot_102_leader.RebotArm102Leader
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.rebot_102_leader.RebotArm102LeaderTeleopConfig
|
||||
|
||||
## BiRebot102Leader
|
||||
|
||||
A bimanual pair of `RebotArm102Leader` arms.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_rebot_102_leader.BiRebot102Leader
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_rebot_102_leader.BiRebot102LeaderConfig
|
||||
|
||||
## BiSOLeader
|
||||
|
||||
A bimanual pair of `SOLeader` arms.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_so_leader.BiSOLeader
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.bi_so_leader.BiSOLeaderConfig
|
||||
|
||||
## Phone
|
||||
|
||||
Reads pose and touch input from a phone app (iOS or Android).
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.phone.Phone
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.phone.PhoneConfig
|
||||
|
||||
## Keyboard
|
||||
|
||||
`KeyboardTeleop`, `KeyboardEndEffectorTeleop`, and `KeyboardRoverTeleop` read key-press events for manual
|
||||
control, targeting joint-space, end-effector, or mobile-base actions respectively.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.keyboard.KeyboardTeleop
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.keyboard.KeyboardTeleopConfig
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.keyboard.KeyboardEndEffectorTeleop
|
||||
- all
|
||||
- action_features
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.keyboard.KeyboardEndEffectorTeleopConfig
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.keyboard.KeyboardRoverTeleop
|
||||
- all
|
||||
- action_features
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.keyboard.KeyboardRoverTeleopConfig
|
||||
|
||||
## GamepadTeleop
|
||||
|
||||
Reads joystick/button input from a gamepad via pygame.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.gamepad.GamepadTeleop
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.gamepad.GamepadTeleopConfig
|
||||
|
||||
## UnitreeG1Teleoperator
|
||||
|
||||
A wearable exoskeleton for teleoperating the Unitree G1 humanoid's arms, mapping exoskeleton joint angles to
|
||||
G1 end-effector poses via forward/inverse kinematics.
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.UnitreeG1Teleoperator
|
||||
- all
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.UnitreeG1TeleoperatorConfig
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.ExoskeletonArm
|
||||
- all
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.ExoskeletonArmPortConfig
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.ExoskeletonIKHelper
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.ExoskeletonCalibration
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.unitree_g1.ExoskeletonJointCalibration
|
||||
|
||||
## Reachy2Teleoperator
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.reachy2_teleoperator.Reachy2Teleoperator
|
||||
- all
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.reachy2_teleoperator.Reachy2TeleoperatorConfig
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Image Transforms
|
||||
|
||||
Data-augmentation transforms applied to camera observations during training. [`~transforms.ImageTransforms`]
|
||||
composes a random subset of them, configured via [`~transforms.ImageTransformsConfig`].
|
||||
|
||||
## ImageTransforms
|
||||
|
||||
[[autodoc]] lerobot.transforms.ImageTransforms
|
||||
- all
|
||||
|
||||
## ImageTransformsConfig
|
||||
|
||||
[[autodoc]] lerobot.transforms.ImageTransformsConfig
|
||||
|
||||
## ImageTransformConfig
|
||||
|
||||
[[autodoc]] lerobot.transforms.ImageTransformConfig
|
||||
|
||||
## make_transform_from_config
|
||||
|
||||
[[autodoc]] lerobot.transforms.make_transform_from_config
|
||||
|
||||
## RandomSubsetApply
|
||||
|
||||
[[autodoc]] lerobot.transforms.RandomSubsetApply
|
||||
- all
|
||||
|
||||
## SharpnessJitter
|
||||
|
||||
[[autodoc]] lerobot.transforms.SharpnessJitter
|
||||
- all
|
||||
|
||||
## GaussianNoise
|
||||
|
||||
[[autodoc]] lerobot.transforms.GaussianNoise
|
||||
- all
|
||||
|
||||
## MotionBlur
|
||||
|
||||
[[autodoc]] lerobot.transforms.MotionBlur
|
||||
- all
|
||||
|
||||
## JPEGCompression
|
||||
|
||||
[[autodoc]] lerobot.transforms.JPEGCompression
|
||||
- all
|
||||
|
||||
## GaussianPatchBrightness
|
||||
|
||||
[[autodoc]] lerobot.transforms.GaussianPatchBrightness
|
||||
- all
|
||||
|
||||
## RandomShadow
|
||||
|
||||
[[autodoc]] lerobot.transforms.RandomShadow
|
||||
- all
|
||||
|
||||
## CoarseDropout
|
||||
|
||||
[[autodoc]] lerobot.transforms.CoarseDropout
|
||||
- all
|
||||
|
||||
## GammaCorrection
|
||||
|
||||
[[autodoc]] lerobot.transforms.GammaCorrection
|
||||
- all
|
||||
|
||||
## PlanckianJitter
|
||||
|
||||
[[autodoc]] lerobot.transforms.PlanckianJitter
|
||||
- all
|
||||
+1
-2
@@ -454,8 +454,7 @@ ignore = [
|
||||
"src/lerobot/rl/**" = ["D"]
|
||||
"src/lerobot/rollout/**" = ["D"]
|
||||
"src/lerobot/scripts/**" = ["D"]
|
||||
"src/lerobot/teleoperators/teleoperator.py" = ["D"]
|
||||
"src/lerobot/transforms/**" = ["D"]
|
||||
"src/lerobot/teleoperators/**" = ["D"]
|
||||
"src/lerobot/transport/**" = ["D"]
|
||||
"src/lerobot/utils/**" = ["D"]
|
||||
"src/lerobot/lerobot_types.py" = ["D"]
|
||||
|
||||
@@ -29,19 +29,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BiOpenArmLeader(BimanualMixin, Teleoperator):
|
||||
"""A bimanual pair of [`~teleoperators.openarm_leader.OpenArmLeader`] arms."""
|
||||
"""
|
||||
Bimanual OpenArm Leader Arms
|
||||
"""
|
||||
|
||||
config_class = BiOpenArmLeaderConfig
|
||||
name = "bi_openarm_leader"
|
||||
|
||||
def __init__(self, config: BiOpenArmLeaderConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`BiOpenArmLeaderConfig`):
|
||||
The teleoperator's configuration. Its `left_arm_config` and `right_arm_config` determine
|
||||
what is connected on each side.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -80,10 +75,6 @@ class BiOpenArmLeader(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Merges both arms' features, each key prefixed with `left_` or `right_`.
|
||||
"""
|
||||
left_arm_features = self.left_arm.action_features
|
||||
right_arm_features = self.right_arm.action_features
|
||||
|
||||
@@ -94,31 +85,15 @@ class BiOpenArmLeader(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
Always empty: feedback is not implemented for the OpenArm leader.
|
||||
"""
|
||||
return {}
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Not supported: raises `NotImplementedError`.
|
||||
|
||||
Motor ID configuration for CAN motors is typically done via manufacturer tools rather than through
|
||||
LeRobot.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Motor ID configuration is typically done via manufacturer tools for CAN motors."
|
||||
)
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""See [`~teleoperators.Teleoperator.get_action`].
|
||||
|
||||
Merges both arms' actions, each key prefixed with `left_` or `right_`.
|
||||
"""
|
||||
action_dict = {}
|
||||
|
||||
# Add "left_" prefix
|
||||
@@ -132,14 +107,5 @@ class BiOpenArmLeader(BimanualMixin, Teleoperator):
|
||||
return action_dict
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported: raises `NotImplementedError`.
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, float]`):
|
||||
Unused.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
# TODO: Implement force feedback
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -23,23 +23,7 @@ from ..openarm_leader import OpenArmLeaderConfigBase
|
||||
@TeleoperatorConfig.register_subclass("bi_openarm_leader")
|
||||
@dataclass
|
||||
class BiOpenArmLeaderConfig(TeleoperatorConfig):
|
||||
"""Configuration for a bimanual pair of OpenArm leader arms.
|
||||
|
||||
The two arms are configured independently, then driven as one teleoperator: action keys from each arm
|
||||
are prefixed with `left_` and `right_`.
|
||||
|
||||
Calibration is per arm, taken from each arm config's own `id` and `calibration_dir`.
|
||||
|
||||
Args:
|
||||
left_arm_config (`OpenArmLeaderConfigBase`):
|
||||
Configuration for the left arm, including its own `port` and `motor_config`.
|
||||
right_arm_config (`OpenArmLeaderConfigBase`):
|
||||
Configuration for the right arm, including its own `port` and `motor_config`.
|
||||
id (`str`, *optional*):
|
||||
Identifier for the pair as a whole.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Unused at this level; each arm calibrates through its own config.
|
||||
"""
|
||||
"""Configuration class for Bi OpenArm Leader teleoperators."""
|
||||
|
||||
left_arm_config: OpenArmLeaderConfigBase
|
||||
right_arm_config: OpenArmLeaderConfigBase
|
||||
|
||||
@@ -40,14 +40,6 @@ class BiOpenArmMini(BimanualMixin, Teleoperator):
|
||||
name = "bi_openarm_mini"
|
||||
|
||||
def __init__(self, config: BiOpenArmMiniConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`BiOpenArmMiniConfig`):
|
||||
The teleoperator's configuration. Its `left_arm_config` and `right_arm_config` determine
|
||||
what is connected on each side; each arm's `side` is forced to `"left"`/`"right"`
|
||||
regardless of what was set on the per-arm config.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -74,10 +66,6 @@ class BiOpenArmMini(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Merges both arms' features, each key prefixed with `left_` or `right_`.
|
||||
"""
|
||||
return {
|
||||
**{f"left_{k}": v for k, v in self.left_arm.action_features.items()},
|
||||
**{f"right_{k}": v for k, v in self.right_arm.action_features.items()},
|
||||
@@ -85,30 +73,17 @@ class BiOpenArmMini(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
Merges both arms' features, each key prefixed with `left_` or `right_`.
|
||||
"""
|
||||
return {
|
||||
**{f"left_{k}": v for k, v in self.left_arm.feedback_features.items()},
|
||||
**{f"right_{k}": v for k, v in self.right_arm.feedback_features.items()},
|
||||
}
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one arm at a time.
|
||||
|
||||
Run this once when building the teleoperator. Interactive: prompts you to connect the controller
|
||||
board to a single motor at a time, left arm first.
|
||||
"""
|
||||
self.left_arm.setup_motors()
|
||||
self.right_arm.setup_motors()
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""See [`~teleoperators.Teleoperator.get_action`].
|
||||
|
||||
Merges both arms' actions, each key prefixed with `left_` or `right_`.
|
||||
"""
|
||||
action: RobotAction = {}
|
||||
for k, v in self.left_arm.get_action().items():
|
||||
action[f"left_{k}"] = v
|
||||
@@ -118,14 +93,6 @@ class BiOpenArmMini(BimanualMixin, Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.send_feedback`].
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, float]`):
|
||||
Feedback values keyed with `left_`/`right_` prefixes, as produced by
|
||||
[`~teleoperators.bi_openarm_mini.BiOpenArmMini.get_action`]. Each arm only receives the entries for its
|
||||
own side.
|
||||
"""
|
||||
left_fb = {k.removeprefix("left_"): v for k, v in feedback.items() if k.startswith("left_")}
|
||||
right_fb = {k.removeprefix("right_"): v for k, v in feedback.items() if k.startswith("right_")}
|
||||
if left_fb:
|
||||
|
||||
@@ -23,18 +23,7 @@ from ..openarm_mini import OpenArmMiniConfigBase
|
||||
@TeleoperatorConfig.register_subclass("bi_openarm_mini")
|
||||
@dataclass
|
||||
class BiOpenArmMiniConfig(TeleoperatorConfig):
|
||||
"""Configuration for a bimanual pair of OpenArm Mini leader arms.
|
||||
|
||||
Args:
|
||||
left_arm_config (`OpenArmMiniConfigBase`):
|
||||
Configuration for the left arm.
|
||||
right_arm_config (`OpenArmMiniConfigBase`):
|
||||
Configuration for the right arm.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
"""
|
||||
"""Configuration class for Bi OpenArm Mini teleoperators."""
|
||||
|
||||
left_arm_config: OpenArmMiniConfigBase
|
||||
right_arm_config: OpenArmMiniConfigBase
|
||||
|
||||
@@ -40,12 +40,6 @@ class BiRebot102Leader(BimanualMixin, Teleoperator):
|
||||
name = "bi_rebot_102_leader"
|
||||
|
||||
def __init__(self, config: BiRebot102LeaderConfig):
|
||||
"""Build the two underlying [`~teleoperators.rebot_102_leader.RebotArm102Leader`] arms.
|
||||
|
||||
Args:
|
||||
config (`BiRebot102LeaderConfig`):
|
||||
The teleoperator's configuration.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -74,11 +68,6 @@ class BiRebot102Leader(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The union of both arms' action features, each key prefixed `left_` / `right_`.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: See [`~teleoperators.rebot_102_leader.RebotArm102Leader.action_features`].
|
||||
"""
|
||||
return {
|
||||
**{f"left_{k}": v for k, v in self.left_arm.action_features.items()},
|
||||
**{f"right_{k}": v for k, v in self.right_arm.action_features.items()},
|
||||
@@ -86,29 +75,14 @@ class BiRebot102Leader(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""Neither arm accepts feedback.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Always empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Read both arms' actions and merge them under `left_` / `right_` prefixed keys.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: See [`~teleoperators.rebot_102_leader.RebotArm102Leader.get_action`].
|
||||
"""
|
||||
action_dict = {}
|
||||
action_dict.update({f"left_{k}": v for k, v in self.left_arm.get_action().items()})
|
||||
action_dict.update({f"right_{k}": v for k, v in self.right_arm.get_action().items()})
|
||||
return action_dict
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported: neither arm has actuators to receive feedback.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError("Feedback is not implemented for the reBot Arm 102 leader.")
|
||||
|
||||
@@ -23,23 +23,7 @@ from ..rebot_102_leader import RebotArm102LeaderConfig
|
||||
@TeleoperatorConfig.register_subclass("bi_rebot_102_leader")
|
||||
@dataclass
|
||||
class BiRebot102LeaderConfig(TeleoperatorConfig):
|
||||
"""Configuration class for the bimanual reBot Arm 102 leader teleoperator.
|
||||
|
||||
Args:
|
||||
left_arm_config (`RebotArm102LeaderConfig`):
|
||||
Configuration of the left [`~teleoperators.rebot_102_leader.RebotArm102Leader`] arm. Its
|
||||
`id` and `calibration_dir` are ignored; the bimanual `id` and `calibration_dir` below are
|
||||
used for both arms instead.
|
||||
right_arm_config (`RebotArm102LeaderConfig`):
|
||||
Configuration of the right [`~teleoperators.rebot_102_leader.RebotArm102Leader`] arm. Same
|
||||
caveat as `left_arm_config`.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit; also names the calibration files for both arms
|
||||
(suffixed `_left` / `_right`).
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write both arms' calibration files. Defaults to the LeRobot calibration
|
||||
home.
|
||||
"""
|
||||
"""Configuration class for the bimanual reBot Arm 102 leader teleoperator."""
|
||||
|
||||
left_arm_config: RebotArm102LeaderConfig
|
||||
right_arm_config: RebotArm102LeaderConfig
|
||||
|
||||
@@ -29,19 +29,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BiSOLeader(BimanualMixin, Teleoperator):
|
||||
"""A bimanual pair of [SO leader arms](https://github.com/TheRobotStudio/SO-ARM100) by TheRobotStudio."""
|
||||
"""
|
||||
[Bimanual SO Leader Arms](https://github.com/TheRobotStudio/SO-ARM100) designed by TheRobotStudio
|
||||
"""
|
||||
|
||||
config_class = BiSOLeaderConfig
|
||||
name = "bi_so_leader"
|
||||
|
||||
def __init__(self, config: BiSOLeaderConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`BiSOLeaderConfig`):
|
||||
The teleoperator's configuration. Its `left_arm_config` and `right_arm_config` determine
|
||||
what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -66,12 +61,6 @@ class BiSOLeader(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The values this teleoperator produces, and their types.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Each arm's [`~teleoperators.so_leader.SOLeader.action_features`] keys,
|
||||
prefixed with `left_` or `right_`.
|
||||
"""
|
||||
left_arm_features = self.left_arm.action_features
|
||||
right_arm_features = self.right_arm.action_features
|
||||
|
||||
@@ -82,12 +71,6 @@ class BiSOLeader(BimanualMixin, Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""The values this teleoperator accepts as feedback, and their types.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Each arm's [`~teleoperators.so_leader.SOLeader.feedback_features`] keys,
|
||||
prefixed with `left_` or `right_`.
|
||||
"""
|
||||
# Bimanual teleop has feedback (can be actuated for handover).
|
||||
# Return the same structure as action_features for consistency with left/right arms.
|
||||
left_arm_features = self.left_arm.feedback_features
|
||||
@@ -99,25 +82,11 @@ class BiSOLeader(BimanualMixin, Teleoperator):
|
||||
}
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID on both arms, one at a time.
|
||||
|
||||
Run this once when building the teleoperator. Interactive: prompts you to connect the controller
|
||||
board to a single motor at a time, left arm first.
|
||||
"""
|
||||
self.left_arm.setup_motors()
|
||||
self.right_arm.setup_motors()
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Retrieve the current action from both leader arms.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: Each arm's action, keyed as described by
|
||||
[`~teleoperators.bi_so_leader.BiSOLeader.action_features`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If [`~teleoperators.Teleoperator.connect`] has not been called.
|
||||
"""
|
||||
action_dict = {}
|
||||
|
||||
# Add "left_" prefix
|
||||
|
||||
@@ -23,18 +23,7 @@ from ..so_leader import SOLeaderConfig
|
||||
@TeleoperatorConfig.register_subclass("bi_so_leader")
|
||||
@dataclass
|
||||
class BiSOLeaderConfig(TeleoperatorConfig):
|
||||
"""Configuration for a bimanual pair of SO-family leader arms.
|
||||
|
||||
Args:
|
||||
left_arm_config (`SOLeaderConfig`):
|
||||
Configuration for the left arm.
|
||||
right_arm_config (`SOLeaderConfig`):
|
||||
Configuration for the right arm.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
"""
|
||||
"""Configuration class for Bi SO Leader teleoperators."""
|
||||
|
||||
left_arm_config: SOLeaderConfig
|
||||
right_arm_config: SOLeaderConfig
|
||||
|
||||
@@ -21,21 +21,6 @@ import draccus
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class TeleoperatorConfig(draccus.ChoiceRegistry, abc.ABC):
|
||||
"""Base configuration shared by every teleoperator.
|
||||
|
||||
Concrete teleoperators subclass this and register themselves with
|
||||
`@TeleoperatorConfig.register_subclass("name")`, which is what makes `--teleop.type=name` work on the
|
||||
command line. Subclasses inherit the two fields below and must document them alongside their own.
|
||||
|
||||
Args:
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
# Allows to distinguish between different teleoperators of the same type
|
||||
id: str | None = None
|
||||
# Directory to store calibration file
|
||||
@@ -43,9 +28,4 @@ class TeleoperatorConfig(draccus.ChoiceRegistry, abc.ABC):
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
"""Return the registered name this config was registered under.
|
||||
|
||||
Returns:
|
||||
`str`: The name passed to `@TeleoperatorConfig.register_subclass`, e.g. `"so101_leader"`.
|
||||
"""
|
||||
return self.get_choice_name(self.__class__)
|
||||
|
||||
@@ -22,22 +22,6 @@ from ..config import TeleoperatorConfig
|
||||
@TeleoperatorConfig.register_subclass("gamepad")
|
||||
@dataclass
|
||||
class GamepadTeleopConfig(TeleoperatorConfig):
|
||||
"""Configuration for the gamepad teleoperator.
|
||||
|
||||
Args:
|
||||
use_gripper (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include a `gripper` entry in the produced actions.
|
||||
hidapi_fallback (`bool`, *optional*, defaults to `False`):
|
||||
Read the gamepad through `hidapi` instead of `pygame`. Set this on macOS if `pygame` does not
|
||||
reliably detect input from your controller.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
use_gripper: bool = True
|
||||
# Use hidapi instead of pygame for controllers that pygame cannot detect reliably.
|
||||
hidapi_fallback: bool = False
|
||||
|
||||
@@ -34,88 +34,68 @@ else:
|
||||
|
||||
|
||||
class InputController:
|
||||
"""Base class for input controllers that generate motion deltas for gamepad-style teleoperation.
|
||||
|
||||
Subclasses override `start`, `stop`, `update`, and `get_deltas` to read an actual device; this base
|
||||
class returns inert defaults.
|
||||
"""
|
||||
"""Base class for input controllers that generate motion deltas."""
|
||||
|
||||
def __init__(self, x_step_size=1.0, y_step_size=1.0, z_step_size=1.0):
|
||||
"""Instantiate the controller's step sizes and reset its state.
|
||||
"""
|
||||
Initialize the controller.
|
||||
|
||||
Args:
|
||||
x_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along X, in meters.
|
||||
y_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along Y, in meters.
|
||||
z_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along Z, in meters.
|
||||
x_step_size: Base movement step size in meters
|
||||
y_step_size: Base movement step size in meters
|
||||
z_step_size: Base movement step size in meters
|
||||
"""
|
||||
self.x_step_size = x_step_size
|
||||
self.y_step_size = y_step_size
|
||||
self.z_step_size = z_step_size
|
||||
self.running = True
|
||||
self.episode_end_status = None # None, or a TeleopEvents member (SUCCESS, FAILURE, RERECORD_EPISODE)
|
||||
self.episode_end_status = None # None, "success", or "failure"
|
||||
self.intervention_flag = False
|
||||
self.open_gripper_command = False
|
||||
self.close_gripper_command = False
|
||||
|
||||
def start(self):
|
||||
"""Start the controller and initialize resources. Subclasses open the actual device here."""
|
||||
"""Start the controller and initialize resources."""
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
"""Stop the controller and release resources. Subclasses close the actual device here."""
|
||||
"""Stop the controller and release resources."""
|
||||
pass
|
||||
|
||||
def get_deltas(self):
|
||||
"""Get the current movement deltas.
|
||||
|
||||
Returns:
|
||||
`tuple[float, float, float]`: `(dx, dy, dz)` in meters. Always `(0.0, 0.0, 0.0)` on the base
|
||||
class.
|
||||
"""
|
||||
"""Get the current movement deltas (dx, dy, dz) in meters."""
|
||||
return 0.0, 0.0, 0.0
|
||||
|
||||
def update(self):
|
||||
"""Refresh the controller's internal state. Call this once per frame before reading deltas or events."""
|
||||
"""Update controller state - call this once per frame."""
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
"""Support for use in `with` statements. Calls `start`."""
|
||||
"""Support for use in 'with' statements."""
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Ensure resources are released when exiting a `with` block, even on error."""
|
||||
"""Ensure resources are released when exiting 'with' block."""
|
||||
self.stop()
|
||||
|
||||
def get_episode_end_status(self):
|
||||
"""Read and clear the current episode end status.
|
||||
"""
|
||||
Get the current episode end status.
|
||||
|
||||
Returns:
|
||||
`TeleopEvents | None`: `None` if the episode should continue, otherwise whichever
|
||||
[`~teleoperators.TeleopEvents`] member (e.g. `SUCCESS`, `FAILURE`, `RERECORD_EPISODE`) a
|
||||
subclass most recently recorded.
|
||||
None if episode should continue, "success" or "failure" otherwise
|
||||
"""
|
||||
status = self.episode_end_status
|
||||
self.episode_end_status = None # Reset after reading
|
||||
return status
|
||||
|
||||
def should_intervene(self):
|
||||
"""Whether the intervention flag is currently set.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if a human is currently intervening.
|
||||
"""
|
||||
"""Return True if intervention flag was set."""
|
||||
return self.intervention_flag
|
||||
|
||||
def gripper_command(self):
|
||||
"""Derive a gripper command from the open/close button flags.
|
||||
|
||||
Returns:
|
||||
`str`: `"open"` or `"close"` if exactly one of the flags is set, `"stay"` otherwise.
|
||||
"""
|
||||
"""Return the current gripper command."""
|
||||
if self.open_gripper_command == self.close_gripper_command:
|
||||
return "stay"
|
||||
elif self.open_gripper_command:
|
||||
@@ -125,14 +105,9 @@ class InputController:
|
||||
|
||||
|
||||
class KeyboardController(InputController):
|
||||
"""Generate motion deltas from keyboard input via `pynput`, as an alternative to a physical gamepad.
|
||||
|
||||
Arrow keys drive X/Y, shift/shift_r drive Z, `enter`/`backspace` end the episode with success/failure,
|
||||
and `esc` stops the listener.
|
||||
"""
|
||||
"""Generate motion deltas from keyboard input."""
|
||||
|
||||
def __init__(self, x_step_size=1.0, y_step_size=1.0, z_step_size=1.0):
|
||||
"""See `InputController.__init__`; the step sizes have the same meaning here."""
|
||||
super().__init__(x_step_size, y_step_size, z_step_size)
|
||||
self.key_states = {
|
||||
"forward_x": False,
|
||||
@@ -148,7 +123,7 @@ class KeyboardController(InputController):
|
||||
self.listener = None
|
||||
|
||||
def start(self):
|
||||
"""Start the `pynput` keyboard listener, if the current session can capture key events."""
|
||||
"""Start the keyboard listener."""
|
||||
if not pynput_can_capture():
|
||||
logging.warning(
|
||||
"Keyboard control is unavailable in this environment. pynput cannot capture keys "
|
||||
@@ -161,7 +136,6 @@ class KeyboardController(InputController):
|
||||
from pynput import keyboard
|
||||
|
||||
def on_press(key):
|
||||
"""Update key/episode state for a key-down event."""
|
||||
try:
|
||||
if key == keyboard.Key.up:
|
||||
self.key_states["forward_x"] = True
|
||||
@@ -189,7 +163,6 @@ class KeyboardController(InputController):
|
||||
pass
|
||||
|
||||
def on_release(key):
|
||||
"""Update key state for a key-up event."""
|
||||
try:
|
||||
if key == keyboard.Key.up:
|
||||
self.key_states["forward_x"] = False
|
||||
@@ -221,16 +194,12 @@ class KeyboardController(InputController):
|
||||
print(" ESC: Exit")
|
||||
|
||||
def stop(self):
|
||||
"""Stop the `pynput` keyboard listener."""
|
||||
"""Stop the keyboard listener."""
|
||||
if self.listener and self.listener.is_alive():
|
||||
self.listener.stop()
|
||||
|
||||
def get_deltas(self):
|
||||
"""Get the current movement deltas from held-down arrow/shift keys.
|
||||
|
||||
Returns:
|
||||
`tuple[float, float, float]`: `(dx, dy, dz)` in meters.
|
||||
"""
|
||||
"""Get the current movement deltas from keyboard state."""
|
||||
delta_x = delta_y = delta_z = 0.0
|
||||
|
||||
if self.key_states["forward_x"]:
|
||||
@@ -250,29 +219,9 @@ class KeyboardController(InputController):
|
||||
|
||||
|
||||
class GamepadController(InputController):
|
||||
"""Generate motion deltas from gamepad input via `pygame`.
|
||||
|
||||
Left stick drives X/Y, the right stick's vertical axis drives Z. Y/Triangle, A/Cross, and X/Square
|
||||
end the episode with success, failure, or rerecord respectively; RB/LT open and close the gripper;
|
||||
holding RB also sets the intervention flag.
|
||||
"""
|
||||
"""Generate motion deltas from gamepad input."""
|
||||
|
||||
def __init__(self, x_step_size=1.0, y_step_size=1.0, z_step_size=1.0, deadzone=0.1):
|
||||
"""Instantiate the controller.
|
||||
|
||||
Args:
|
||||
x_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along X, in meters.
|
||||
y_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along Y, in meters.
|
||||
z_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along Z, in meters.
|
||||
deadzone (`float`, *optional*, defaults to 0.1):
|
||||
Minimum absolute stick reading before it is treated as input, to filter out drift.
|
||||
|
||||
Raises:
|
||||
ImportError: If `pygame` is not installed.
|
||||
"""
|
||||
require_package("pygame", extra="gamepad")
|
||||
super().__init__(x_step_size, y_step_size, z_step_size)
|
||||
self.deadzone = deadzone
|
||||
@@ -280,7 +229,7 @@ class GamepadController(InputController):
|
||||
self.intervention_flag = False
|
||||
|
||||
def start(self):
|
||||
"""Initialize `pygame` and connect to the first detected joystick."""
|
||||
"""Initialize pygame and the gamepad."""
|
||||
pygame.init()
|
||||
pygame.joystick.init()
|
||||
|
||||
@@ -302,7 +251,7 @@ class GamepadController(InputController):
|
||||
print(" X/Square button: Rerecord episode")
|
||||
|
||||
def stop(self):
|
||||
"""Clean up `pygame` joystick and display resources."""
|
||||
"""Clean up pygame resources."""
|
||||
if pygame.joystick.get_init():
|
||||
if self.joystick:
|
||||
self.joystick.quit()
|
||||
@@ -310,7 +259,7 @@ class GamepadController(InputController):
|
||||
pygame.quit()
|
||||
|
||||
def update(self):
|
||||
"""Drain pending `pygame` events to refresh button, episode, and intervention state."""
|
||||
"""Process pygame events to get fresh gamepad readings."""
|
||||
for event in pygame.event.get():
|
||||
if event.type == pygame.JOYBUTTONDOWN:
|
||||
if event.button == 3:
|
||||
@@ -348,12 +297,7 @@ class GamepadController(InputController):
|
||||
self.intervention_flag = False
|
||||
|
||||
def get_deltas(self):
|
||||
"""Get the current movement deltas from the joystick axes, after applying the deadzone.
|
||||
|
||||
Returns:
|
||||
`tuple[float, float, float]`: `(dx, dy, dz)` in meters. `(0.0, 0.0, 0.0)` if reading the
|
||||
joystick raises `pygame.error` (e.g. the controller was disconnected).
|
||||
"""
|
||||
"""Get the current movement deltas from gamepad state."""
|
||||
try:
|
||||
# Read joystick axes
|
||||
# Left stick X and Y (typically axes 0 and 1)
|
||||
@@ -381,12 +325,7 @@ class GamepadController(InputController):
|
||||
|
||||
|
||||
class GamepadControllerHID(InputController):
|
||||
"""Generate motion deltas from gamepad input by reading raw HID reports via `hidapi`.
|
||||
|
||||
An alternative to `GamepadController` for controllers `pygame` does not reliably detect (notably on
|
||||
macOS). Byte offsets in `update` are tuned for the Logitech RumblePad 2 and may need adjusting for
|
||||
other controllers.
|
||||
"""
|
||||
"""Generate motion deltas from gamepad input using HIDAPI."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -395,20 +334,13 @@ class GamepadControllerHID(InputController):
|
||||
z_step_size=1.0,
|
||||
deadzone=0.1,
|
||||
):
|
||||
"""Instantiate the controller.
|
||||
"""
|
||||
Initialize the HID gamepad controller.
|
||||
|
||||
Args:
|
||||
x_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along X, in meters.
|
||||
y_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along Y, in meters.
|
||||
z_step_size (`float`, *optional*, defaults to 1.0):
|
||||
Movement step size along Z, in meters.
|
||||
deadzone (`float`, *optional*, defaults to 0.1):
|
||||
Minimum absolute stick reading before it is treated as input, to filter out drift.
|
||||
|
||||
Raises:
|
||||
ImportError: If `hidapi` is not installed.
|
||||
step_size: Base movement step size in meters
|
||||
z_scale: Scaling factor for Z-axis movement
|
||||
deadzone: Joystick deadzone to prevent drift
|
||||
"""
|
||||
require_package("hidapi", extra="gamepad", import_name="hid")
|
||||
super().__init__(x_step_size, y_step_size, z_step_size)
|
||||
@@ -426,14 +358,7 @@ class GamepadControllerHID(InputController):
|
||||
self.buttons = {}
|
||||
|
||||
def find_device(self):
|
||||
"""Look for a supported gamepad among enumerated HID devices.
|
||||
|
||||
Matches the first device whose product string contains `"Logitech"`, `"Xbox"`, `"PS4"`, or
|
||||
`"PS5"`.
|
||||
|
||||
Returns:
|
||||
`dict | None`: The `hidapi` device info dict, or `None` if no matching device was found.
|
||||
"""
|
||||
"""Look for the gamepad device by vendor and product ID."""
|
||||
devices = hid.enumerate()
|
||||
for device in devices:
|
||||
device_name = device["product_string"]
|
||||
@@ -446,7 +371,7 @@ class GamepadControllerHID(InputController):
|
||||
return None
|
||||
|
||||
def start(self):
|
||||
"""Find and open the gamepad's HID device in non-blocking mode."""
|
||||
"""Connect to the gamepad using HIDAPI."""
|
||||
self.device_info = self.find_device()
|
||||
if not self.device_info:
|
||||
self.running = False
|
||||
@@ -481,9 +406,9 @@ class GamepadControllerHID(InputController):
|
||||
self.device = None
|
||||
|
||||
def update(self):
|
||||
"""Read and process the latest gamepad HID report.
|
||||
|
||||
Reads the device 10 times in a row, since a single `hidapi` read can otherwise return stale data.
|
||||
"""
|
||||
Read and process the latest gamepad data.
|
||||
Due to an issue with the HIDAPI, we need to read the read the device several times in order to get a stable reading
|
||||
"""
|
||||
for _ in range(10):
|
||||
self._update()
|
||||
@@ -539,11 +464,7 @@ class GamepadControllerHID(InputController):
|
||||
logging.error(f"Error reading from gamepad: {e}")
|
||||
|
||||
def get_deltas(self):
|
||||
"""Get the current movement deltas from the last-read HID report.
|
||||
|
||||
Returns:
|
||||
`tuple[float, float, float]`: `(dx, dy, dz)` in meters.
|
||||
"""
|
||||
"""Get the current movement deltas from gamepad state."""
|
||||
# Calculate deltas - invert as needed based on controller orientation
|
||||
delta_x = -self.left_x * self.x_step_size # Forward/backward
|
||||
delta_y = -self.left_y * self.y_step_size # Left/right
|
||||
|
||||
@@ -32,14 +32,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GripperAction(IntEnum):
|
||||
"""Gripper command levels produced by a gamepad's gripper buttons.
|
||||
|
||||
**Attributes**:
|
||||
- **CLOSE** (`int`) -- Close the gripper.
|
||||
- **STAY** (`int`) -- Leave the gripper where it is.
|
||||
- **OPEN** (`int`) -- Open the gripper.
|
||||
"""
|
||||
|
||||
CLOSE = 0
|
||||
STAY = 1
|
||||
OPEN = 2
|
||||
@@ -53,24 +45,14 @@ gripper_action_map = {
|
||||
|
||||
|
||||
class GamepadTeleop(Teleoperator):
|
||||
"""Teleoperator that reads a gamepad's analog sticks and buttons via `pygame` (or `hidapi`).
|
||||
|
||||
[`~teleoperators.Teleoperator.get_action`] reports the left stick as `delta_x`/`delta_y` and the
|
||||
right stick's vertical axis as `delta_z`, plus an optional gripper command. See `gamepad_utils.py`'s
|
||||
`GamepadController` (`pygame`) and `GamepadControllerHID` (`hidapi`) for the exact axis/button
|
||||
mapping.
|
||||
"""
|
||||
Teleop class to use gamepad inputs for control.
|
||||
"""
|
||||
|
||||
config_class = GamepadTeleopConfig
|
||||
name = "gamepad"
|
||||
|
||||
def __init__(self, config: GamepadTeleopConfig):
|
||||
"""Instantiate the teleoperator.
|
||||
|
||||
Args:
|
||||
config (`GamepadTeleopConfig`):
|
||||
Configuration for this gamepad teleoperator.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
self.robot_type = config.type
|
||||
@@ -86,12 +68,6 @@ class GamepadTeleop(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict`: A 3-element (or 4-element if `config.use_gripper` is `True`) `float32` vector named
|
||||
`delta_x`, `delta_y`, `delta_z`, and optionally `gripper`.
|
||||
"""
|
||||
if self.config.use_gripper:
|
||||
return {
|
||||
"dtype": "float32",
|
||||
@@ -107,15 +83,9 @@ class GamepadTeleop(Teleoperator):
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`]. `GamepadTeleop` accepts no feedback."""
|
||||
return {}
|
||||
|
||||
def connect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.connect`].
|
||||
|
||||
Starts a `GamepadControllerHID` if `config.hidapi_fallback` is `True`, otherwise a
|
||||
`GamepadController`.
|
||||
"""
|
||||
if self.hidapi_fallback:
|
||||
from .gamepad_utils import GamepadControllerHID as Gamepad
|
||||
else:
|
||||
@@ -126,18 +96,6 @@ class GamepadTeleop(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Read the gamepad's current stick positions and gripper button state.
|
||||
|
||||
The left analog stick drives `delta_x`/`delta_y`; the right stick's vertical axis drives
|
||||
`delta_z`. When `config.use_gripper` is `True`, the gripper buttons additionally produce a
|
||||
`gripper` entry (one of `GripperAction.CLOSE`, `STAY`, or `OPEN`).
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: `delta_x`, `delta_y`, `delta_z`, and, if enabled, `gripper`.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If [`~teleoperators.Teleoperator.connect`] has not been called.
|
||||
"""
|
||||
# Update the controller to get fresh inputs
|
||||
self.gamepad.update()
|
||||
|
||||
@@ -163,15 +121,16 @@ class GamepadTeleop(Teleoperator):
|
||||
return action_dict
|
||||
|
||||
def get_teleop_events(self) -> dict[str, Any]:
|
||||
"""Read auxiliary gamepad events used to drive episode control during recording.
|
||||
|
||||
Holding the intervention button counts as an active intervention; the success/failure/rerecord
|
||||
buttons are read once as one-shot signals, then cleared.
|
||||
"""
|
||||
Get extra control events from the gamepad such as intervention status,
|
||||
episode termination, success indicators, etc.
|
||||
|
||||
Returns:
|
||||
`dict[TeleopEvents, bool]`: Values for the [`~teleoperators.TeleopEvents`] keys
|
||||
`IS_INTERVENTION`, `TERMINATE_EPISODE`, `SUCCESS`, and `RERECORD_EPISODE`. All `False` if
|
||||
[`~teleoperators.Teleoperator.connect`] has not been called yet.
|
||||
Dictionary containing:
|
||||
- is_intervention: bool - Whether human is currently intervening
|
||||
- terminate_episode: bool - Whether to terminate the current episode
|
||||
- success: bool - Whether the episode was successful
|
||||
- rerecord_episode: bool - Whether to rerecord the episode
|
||||
"""
|
||||
if self.gamepad is None:
|
||||
return {
|
||||
@@ -204,32 +163,32 @@ class GamepadTeleop(Teleoperator):
|
||||
}
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.disconnect`]. Stops and releases the underlying controller."""
|
||||
"""Disconnect from the gamepad."""
|
||||
if self.gamepad is not None:
|
||||
self.gamepad.stop()
|
||||
self.gamepad = None
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
"""Check if gamepad is connected."""
|
||||
return self.gamepad is not None
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.calibrate`]. No-op: the gamepad does not require calibration."""
|
||||
"""Calibrate the gamepad."""
|
||||
# No calibration needed for gamepad
|
||||
pass
|
||||
|
||||
def is_calibrated(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_calibrated`]. Always `True`: no calibration is required."""
|
||||
"""Check if gamepad is calibrated."""
|
||||
# Gamepad doesn't require calibration
|
||||
return True
|
||||
|
||||
def configure(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.configure`]. No-op: the gamepad needs no configuration."""
|
||||
"""Configure the gamepad."""
|
||||
# No additional configuration needed
|
||||
pass
|
||||
|
||||
def send_feedback(self, feedback: dict) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.send_feedback`]. No-op: `GamepadTeleop` accepts no feedback."""
|
||||
"""Send feedback to the gamepad."""
|
||||
# Gamepad doesn't support feedback
|
||||
pass
|
||||
|
||||
@@ -22,34 +22,11 @@ from ..config import TeleoperatorConfig
|
||||
@TeleoperatorConfig.register_subclass("homunculus_glove")
|
||||
@dataclass
|
||||
class HomunculusGloveConfig(TeleoperatorConfig):
|
||||
"""Configuration for the Homunculus Glove teleoperator.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the glove is connected to, e.g. `/dev/ttyACM0`.
|
||||
side (`str`):
|
||||
Which hand the glove is worn on, `"left"` or `"right"`. Selects which joints get their drive
|
||||
mode inverted so the produced action matches the HopeJR hand convention.
|
||||
baud_rate (`int`, *optional*, defaults to 115200):
|
||||
Serial communication speed in bauds.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
port: str # Port to connect to the glove
|
||||
side: str # "left" / "right"
|
||||
baud_rate: int = 115_200
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that `side` is one of `"left"` or `"right"`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `side` is neither `"left"` nor `"right"`.
|
||||
"""
|
||||
if self.side not in ["right", "left"]:
|
||||
raise ValueError(self.side)
|
||||
|
||||
@@ -57,20 +34,5 @@ class HomunculusGloveConfig(TeleoperatorConfig):
|
||||
@TeleoperatorConfig.register_subclass("homunculus_arm")
|
||||
@dataclass
|
||||
class HomunculusArmConfig(TeleoperatorConfig):
|
||||
"""Configuration for the Homunculus Arm teleoperator.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to, e.g. `/dev/ttyACM0`.
|
||||
baud_rate (`int`, *optional*, defaults to 115200):
|
||||
Serial communication speed in bauds.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
port: str # Port to connect to the arm
|
||||
baud_rate: int = 115_200
|
||||
|
||||
@@ -37,25 +37,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HomunculusArm(Teleoperator):
|
||||
"""Homunculus Arm designed by Hugging Face: a wearable exoskeleton arm read over a serial link.
|
||||
|
||||
The arm streams raw encoder values for each joint continuously over a background thread; readings are
|
||||
smoothed with an exponential moving average before being normalized and returned as an action. It only
|
||||
produces actions and accepts no feedback.
|
||||
|
||||
See [`~teleoperators.Teleoperator`] for the contract every method here implements.
|
||||
"""
|
||||
Homunculus Arm designed by Hugging Face.
|
||||
"""
|
||||
|
||||
config_class = HomunculusArmConfig
|
||||
name = "homunculus_arm"
|
||||
|
||||
def __init__(self, config: HomunculusArmConfig):
|
||||
"""Open the serial connection and set up the background reader thread.
|
||||
|
||||
Args:
|
||||
config (`HomunculusArmConfig`):
|
||||
The teleoperator's configuration. Its `port` determines what is connected.
|
||||
"""
|
||||
require_package("pyserial", extra="pyserial-dep", import_name="serial")
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
@@ -99,43 +88,19 @@ class HomunculusArm(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
"""The arm's joint positions.
|
||||
|
||||
Returns:
|
||||
`dict`: `"<joint>.pos"` keys mapped to `float`, one per entry in `self.joints`.
|
||||
"""
|
||||
return {f"{joint}.pos": float for joint in self.joints}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict:
|
||||
"""This arm accepts no feedback.
|
||||
|
||||
Returns:
|
||||
`dict`: Always empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`].
|
||||
|
||||
The serial port is open and the background reader thread is alive.
|
||||
"""
|
||||
with self.serial_lock:
|
||||
return self.serial.is_open and self.thread.is_alive()
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Open the serial port, start the background reader thread, and wait for the first reading.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to run calibration when no calibration file exists yet. Calibration is
|
||||
interactive and prompts on stdin.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If no state is received from the arm within 2 seconds of starting.
|
||||
"""
|
||||
if not self.serial.is_open:
|
||||
self.serial.open()
|
||||
self.thread.start()
|
||||
@@ -151,19 +116,9 @@ class HomunculusArm(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Whether a calibration file has been saved for this arm.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if the calibration file exists on disk.
|
||||
"""
|
||||
return self.calibration_fpath.is_file()
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Interactively record each joint's range of motion and save it as the arm's calibration.
|
||||
|
||||
Prompts the operator to move every joint through its full range, then persists the observed
|
||||
min/max encoder values to the calibration file.
|
||||
"""
|
||||
print(
|
||||
"\nMove all joints through their entire range of motion."
|
||||
"\nRecording positions. Press ENTER to stop..."
|
||||
@@ -242,7 +197,6 @@ class HomunculusArm(Teleoperator):
|
||||
return mins, maxes
|
||||
|
||||
def configure(self) -> None:
|
||||
"""No-op: the arm requires no runtime configuration beyond calibration."""
|
||||
pass
|
||||
|
||||
# TODO(Steven): This function is copy/paste from the `HomunculusGlove` class. Consider moving it to an utility to reduce duplicated code.
|
||||
@@ -285,9 +239,9 @@ class HomunculusArm(Teleoperator):
|
||||
def _read(
|
||||
self, joints: list[str] | None = None, normalize: bool = True, timeout: float = 1
|
||||
) -> dict[str, int | float]:
|
||||
"""Return the most recent values from the reader thread.
|
||||
|
||||
Optionally applies calibration.
|
||||
"""
|
||||
Return the most recent (single) values from self.last_d,
|
||||
optionally applying calibration.
|
||||
"""
|
||||
if not self.new_state_event.wait(timeout=timeout):
|
||||
raise TimeoutError(f"{self}: Timed out waiting for state after {timeout}s.")
|
||||
@@ -311,9 +265,9 @@ class HomunculusArm(Teleoperator):
|
||||
return state
|
||||
|
||||
def _read_loop(self):
|
||||
"""Continuously read from the serial buffer in its own thread.
|
||||
|
||||
Sends values to the main thread through a queue.
|
||||
"""
|
||||
Continuously read from the serial buffer in its own thread and sends values to the main thread through
|
||||
a queue.
|
||||
"""
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
@@ -351,28 +305,14 @@ class HomunculusArm(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Read the most recent EMA-smoothed, normalized joint positions.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `"<joint>.pos"` keys mapped to their normalized position.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If no new reading arrives from the background thread within 1 second.
|
||||
"""
|
||||
joint_positions = self._read()
|
||||
return {f"{joint}.pos": pos for joint, pos in joint_positions.items()}
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported: the arm has no actuators to receive feedback.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Stop the background reader thread and close the serial port."""
|
||||
self.stop_event.set()
|
||||
self.thread.join(timeout=1)
|
||||
self.serial.close()
|
||||
|
||||
@@ -63,27 +63,14 @@ RIGHT_HAND_INVERSIONS = [
|
||||
|
||||
|
||||
class HomunculusGlove(Teleoperator):
|
||||
"""Homunculus Glove designed by NepYope & Hugging Face: a wearable exoskeleton glove read over a serial link.
|
||||
|
||||
The glove streams raw encoder values for each finger joint continuously over a background thread;
|
||||
readings are smoothed with an exponential moving average, normalized, then remapped from glove joint
|
||||
names to HopeJR hand joint names via [`~teleoperators.homunculus.homunculus_glove_to_hope_jr_hand`]. It
|
||||
only produces actions and accepts no feedback.
|
||||
|
||||
See [`~teleoperators.Teleoperator`] for the contract every method here implements.
|
||||
"""
|
||||
Homunculus Glove designed by NepYope & Hugging Face.
|
||||
"""
|
||||
|
||||
config_class = HomunculusGloveConfig
|
||||
name = "homunculus_glove"
|
||||
|
||||
def __init__(self, config: HomunculusGloveConfig):
|
||||
"""Open the serial connection and set up the background reader thread.
|
||||
|
||||
Args:
|
||||
config (`HomunculusGloveConfig`):
|
||||
The teleoperator's configuration. Its `port` determines what is connected and `side`
|
||||
selects which joints are inverted for the left vs. right hand.
|
||||
"""
|
||||
require_package("pyserial", extra="pyserial-dep", import_name="serial")
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
@@ -127,43 +114,19 @@ class HomunculusGlove(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
"""The glove's raw per-joint positions, before remapping to HopeJR hand joint names.
|
||||
|
||||
Returns:
|
||||
`dict`: `"<joint>.pos"` keys mapped to `float`, one per entry in `self.joints`.
|
||||
"""
|
||||
return {f"{joint}.pos": float for joint in self.joints}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict:
|
||||
"""This glove accepts no feedback.
|
||||
|
||||
Returns:
|
||||
`dict`: Always empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`].
|
||||
|
||||
The serial port is open and the background reader thread is alive.
|
||||
"""
|
||||
with self.serial_lock:
|
||||
return self.serial.is_open and self.thread.is_alive()
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Open the serial port, start the background reader thread, and wait for the first reading.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to run calibration when no calibration file exists yet. Calibration is
|
||||
interactive and prompts on stdin.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If no state is received from the glove within 2 seconds of starting.
|
||||
"""
|
||||
if not self.serial.is_open:
|
||||
self.serial.open()
|
||||
self.thread.start()
|
||||
@@ -179,19 +142,9 @@ class HomunculusGlove(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Whether a calibration file has been saved for this glove.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if the calibration file exists on disk.
|
||||
"""
|
||||
return self.calibration_fpath.is_file()
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Interactively record each finger's range of motion and save it as the glove's calibration.
|
||||
|
||||
Prompts the operator to move each finger through its full range, one finger at a time, then
|
||||
persists the observed min/max encoder values to the calibration file.
|
||||
"""
|
||||
range_mins, range_maxes = {}, {}
|
||||
for finger in ["thumb", "index", "middle", "ring", "pinky"]:
|
||||
print(
|
||||
@@ -275,7 +228,6 @@ class HomunculusGlove(Teleoperator):
|
||||
return mins, maxes
|
||||
|
||||
def configure(self) -> None:
|
||||
"""No-op: the glove requires no runtime configuration beyond calibration."""
|
||||
pass
|
||||
|
||||
# TODO(Steven): This function is copy/paste from the `HomunculusArm` class. Consider moving it to an utility to reduce duplicated code.
|
||||
@@ -319,9 +271,9 @@ class HomunculusGlove(Teleoperator):
|
||||
def _read(
|
||||
self, joints: list[str] | None = None, normalize: bool = True, timeout: float = 1
|
||||
) -> dict[str, int | float]:
|
||||
"""Return the most recent values from the reader thread.
|
||||
|
||||
Optionally applies calibration.
|
||||
"""
|
||||
Return the most recent (single) values from self.last_d,
|
||||
optionally applying calibration.
|
||||
"""
|
||||
if not self.new_state_event.wait(timeout=timeout):
|
||||
raise TimeoutError(f"{self}: Timed out waiting for state after {timeout}s.")
|
||||
@@ -347,9 +299,9 @@ class HomunculusGlove(Teleoperator):
|
||||
return state
|
||||
|
||||
def _read_loop(self):
|
||||
"""Continuously read from the serial buffer in its own thread.
|
||||
|
||||
Sends values to the main thread through a queue.
|
||||
"""
|
||||
Continuously read from the serial buffer in its own thread and sends values to the main thread through
|
||||
a queue.
|
||||
"""
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
@@ -379,32 +331,16 @@ class HomunculusGlove(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Read the most recent EMA-smoothed, normalized joint positions, remapped to HopeJR hand joints.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `"<joint>.pos"` keys, named after the HopeJR hand's joints, mapped to
|
||||
their normalized position. See
|
||||
[`~teleoperators.homunculus.homunculus_glove_to_hope_jr_hand`] for the remapping.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If no new reading arrives from the background thread within 1 second.
|
||||
"""
|
||||
joint_positions = self._read()
|
||||
return homunculus_glove_to_hope_jr_hand(
|
||||
{f"{joint}.pos": pos for joint, pos in joint_positions.items()}
|
||||
)
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported: the glove has no actuators to receive feedback.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Stop the background reader thread and close the serial port."""
|
||||
self.stop_event.set()
|
||||
self.thread.join(timeout=1)
|
||||
self.serial.close()
|
||||
|
||||
@@ -19,67 +19,14 @@ PINKY_SPLAY = 0.5
|
||||
|
||||
|
||||
def get_ulnar_flexion(flexion: float, abduction: float, splay: float):
|
||||
"""Derive the ulnar-side tendon command for a HopeJR finger from its glove-sensed MCP angles.
|
||||
|
||||
The HopeJR hand flexes a finger with a pair of opposing tendons (radial and ulnar) rather than
|
||||
independent flexion and abduction joints. This blends the glove's flexion and abduction readings for
|
||||
one MCP joint into the ulnar tendon's share of the motion: an abduction toward the ulnar side pulls
|
||||
this tendon further, while `splay` sets how much of the abduction reading leaks into it versus pure
|
||||
flexion.
|
||||
|
||||
Args:
|
||||
flexion (`float`):
|
||||
MCP flexion reading for the finger, as reported by the glove.
|
||||
abduction (`float`):
|
||||
MCP abduction reading for the finger, as reported by the glove. Positive values pull toward
|
||||
the radial side and are subtracted here.
|
||||
splay (`float`):
|
||||
Fraction, in `[0, 1]`, of the tendon command driven by abduction rather than flexion.
|
||||
|
||||
Returns:
|
||||
`float`: The ulnar tendon's target position.
|
||||
"""
|
||||
return -abduction * splay + flexion * (1 - splay)
|
||||
|
||||
|
||||
def get_radial_flexion(flexion: float, abduction: float, splay: float):
|
||||
"""Derive the radial-side tendon command for a HopeJR finger from its glove-sensed MCP angles.
|
||||
|
||||
The counterpart to [`get_ulnar_flexion`]: same blend of flexion and abduction, but abduction toward
|
||||
the radial side adds to this tendon's target instead of subtracting from it.
|
||||
|
||||
Args:
|
||||
flexion (`float`):
|
||||
MCP flexion reading for the finger, as reported by the glove.
|
||||
abduction (`float`):
|
||||
MCP abduction reading for the finger, as reported by the glove. Positive values pull toward
|
||||
the radial side and are added here.
|
||||
splay (`float`):
|
||||
Fraction, in `[0, 1]`, of the tendon command driven by abduction rather than flexion.
|
||||
|
||||
Returns:
|
||||
`float`: The radial tendon's target position.
|
||||
"""
|
||||
return abduction * splay + flexion * (1 - splay)
|
||||
|
||||
|
||||
def homunculus_glove_to_hope_jr_hand(glove_action: dict[str, float]) -> dict[str, float]:
|
||||
"""Translate a Homunculus Glove action into a HopeJR hand action.
|
||||
|
||||
The glove reports one flexion and one abduction value per finger's MCP joint, plus a DIP/PIP reading,
|
||||
while the HopeJR hand is driven by a pair of tendons (radial and ulnar flexors) per finger and a
|
||||
coupled PIP/DIP joint. This remaps and blends the glove's per-joint keys into the hand's per-tendon
|
||||
keys via [`get_radial_flexion`] and [`get_ulnar_flexion`]; the thumb, whose joints map one-to-one, is
|
||||
passed through unchanged.
|
||||
|
||||
Args:
|
||||
glove_action (`dict[str, float]`):
|
||||
Action produced by [`~teleoperators.homunculus.HomunculusGlove.get_action`], keyed by glove
|
||||
joint name.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: The equivalent action keyed by HopeJR hand joint name.
|
||||
"""
|
||||
return {
|
||||
"thumb_cmc.pos": glove_action["thumb_cmc.pos"],
|
||||
"thumb_mcp.pos": glove_action["thumb_mcp.pos"],
|
||||
|
||||
@@ -23,16 +23,7 @@ from ..config import TeleoperatorConfig
|
||||
@TeleoperatorConfig.register_subclass("keyboard")
|
||||
@dataclass
|
||||
class KeyboardTeleopConfig(TeleoperatorConfig):
|
||||
"""Configuration for the plain keyboard teleoperator.
|
||||
|
||||
Args:
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
"""KeyboardTeleopConfig"""
|
||||
|
||||
# TODO(Steven): Consider setting in here the keys that we want to capture/listen
|
||||
|
||||
@@ -40,17 +31,12 @@ class KeyboardTeleopConfig(TeleoperatorConfig):
|
||||
@TeleoperatorConfig.register_subclass("keyboard_ee")
|
||||
@dataclass
|
||||
class KeyboardEndEffectorTeleopConfig(KeyboardTeleopConfig):
|
||||
"""Configuration for controlling a robot end-effector with keyboard inputs.
|
||||
"""Configuration for keyboard end-effector teleoperator.
|
||||
|
||||
Args:
|
||||
use_gripper (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include a `gripper` entry in the produced actions.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
Used for controlling robot end-effectors with keyboard inputs.
|
||||
|
||||
**Attributes**:
|
||||
- **use_gripper** (`bool`) -- Whether to include gripper control in actions
|
||||
"""
|
||||
|
||||
use_gripper: bool = True
|
||||
@@ -59,29 +45,18 @@ class KeyboardEndEffectorTeleopConfig(KeyboardTeleopConfig):
|
||||
@TeleoperatorConfig.register_subclass("keyboard_rover")
|
||||
@dataclass
|
||||
class KeyboardRoverTeleopConfig(TeleoperatorConfig):
|
||||
"""Configuration for the WASD-style keyboard teleoperator for mobile robots like EarthRover Mini Plus.
|
||||
"""Configuration for keyboard rover teleoperator.
|
||||
|
||||
Args:
|
||||
linear_speed (`float`, *optional*, defaults to 1.0):
|
||||
Initial linear velocity magnitude (-1 to 1 range for SDK robots).
|
||||
angular_speed (`float`, *optional*, defaults to 1.0):
|
||||
Initial angular velocity magnitude (-1 to 1 range for SDK robots).
|
||||
speed_increment (`float`, *optional*, defaults to 0.1):
|
||||
Amount `current_linear_speed` changes by on each `+`/`-` key press.
|
||||
turn_assist_ratio (`float`, *optional*, defaults to 0.3):
|
||||
Forward-motion multiplier applied when turning with `a`/`d` while otherwise stationary.
|
||||
angular_speed_ratio (`float`, *optional*, defaults to 0.6):
|
||||
Ratio of angular to linear speed increment, so both scale together on `+`/`-`.
|
||||
min_linear_speed (`float`, *optional*, defaults to 0.1):
|
||||
Floor for `current_linear_speed` when decreasing it.
|
||||
min_angular_speed (`float`, *optional*, defaults to 0.05):
|
||||
Floor for `current_angular_speed` when decreasing it.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
Used for controlling mobile robots like EarthRover Mini Plus with WASD controls.
|
||||
|
||||
**Attributes**:
|
||||
- **linear_speed** (`float`) -- Default linear velocity magnitude (-1 to 1 range for SDK robots)
|
||||
- **angular_speed** (`float`) -- Default angular velocity magnitude (-1 to 1 range for SDK robots)
|
||||
- **speed_increment** (`float`) -- Amount to increase/decrease speed with +/- keys
|
||||
- **turn_assist_ratio** (`float`) -- Forward motion multiplier when turning with A/D keys (0.0-1.0)
|
||||
- **angular_speed_ratio** (`float`) -- Ratio of angular to linear speed for synchronized adjustments
|
||||
- **min_linear_speed** (`float`) -- Minimum linear speed when decreasing (prevents zero speed)
|
||||
- **min_angular_speed** (`float`) -- Minimum angular speed when decreasing (prevents zero speed)
|
||||
"""
|
||||
|
||||
linear_speed: float = 1.0
|
||||
|
||||
@@ -43,28 +43,14 @@ if PYNPUT_AVAILABLE:
|
||||
|
||||
|
||||
class KeyboardTeleop(Teleoperator):
|
||||
"""Teleoperator that reads raw keyboard key states via `pynput` for manual control.
|
||||
|
||||
[`~teleoperators.Teleoperator.get_action`] reports every key currently held down. Requires an
|
||||
interactive desktop session capable of capturing global key events — an X11 session (Linux), a
|
||||
Windows desktop, or macOS with Accessibility / Input Monitoring permission granted. On Wayland or a
|
||||
headless machine, [`~teleoperators.Teleoperator.connect`] logs a warning and the teleoperator produces
|
||||
no actions.
|
||||
"""
|
||||
Teleop class to use keyboard inputs for control.
|
||||
"""
|
||||
|
||||
config_class = KeyboardTeleopConfig
|
||||
name = "keyboard"
|
||||
|
||||
def __init__(self, config: KeyboardTeleopConfig):
|
||||
"""Instantiate the teleoperator.
|
||||
|
||||
Args:
|
||||
config (`KeyboardTeleopConfig`):
|
||||
Configuration for this keyboard teleoperator.
|
||||
|
||||
Raises:
|
||||
ImportError: If `pynput` is not installed.
|
||||
"""
|
||||
require_package("pynput", extra="pynput-dep")
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
@@ -77,11 +63,6 @@ class KeyboardTeleop(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict`: Motor count and names taken from `self.arm`.
|
||||
"""
|
||||
return {
|
||||
"dtype": "float32",
|
||||
"shape": (len(self.arm),),
|
||||
@@ -90,26 +71,18 @@ class KeyboardTeleop(Teleoperator):
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`]. `KeyboardTeleop` accepts no feedback."""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
return PYNPUT_AVAILABLE and isinstance(self.listener, keyboard.Listener) and self.listener.is_alive()
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_calibrated`]. Keyboard input does not require calibration."""
|
||||
pass
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.connect`].
|
||||
|
||||
Starts a `pynput` keyboard listener if the current session can capture key events; otherwise logs
|
||||
a warning and leaves the teleoperator producing no actions.
|
||||
"""
|
||||
if PYNPUT_AVAILABLE and pynput_can_capture():
|
||||
logging.info("pynput is available - enabling local keyboard listener.")
|
||||
self.listener = keyboard.Listener(
|
||||
@@ -128,7 +101,6 @@ class KeyboardTeleop(Teleoperator):
|
||||
self.listener = None
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.calibrate`]. No-op: keyboard input does not require calibration."""
|
||||
pass
|
||||
|
||||
def _on_press(self, key):
|
||||
@@ -151,20 +123,10 @@ class KeyboardTeleop(Teleoperator):
|
||||
self.current_pressed[key_char] = is_pressed
|
||||
|
||||
def configure(self):
|
||||
"""See [`~teleoperators.Teleoperator.configure`]. No-op: keyboard input needs no configuration."""
|
||||
pass
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Read the keys currently held down.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: One entry per key character currently pressed, each mapped to `None`. An
|
||||
empty dict means no key is currently held.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If [`~teleoperators.Teleoperator.connect`] has not been called.
|
||||
"""
|
||||
before_read_t = time.perf_counter()
|
||||
|
||||
self._drain_pressed_keys()
|
||||
@@ -176,45 +138,30 @@ class KeyboardTeleop(Teleoperator):
|
||||
return dict.fromkeys(action, None)
|
||||
|
||||
def send_feedback(self, feedback: dict[str, Any]) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.send_feedback`]. No-op: `KeyboardTeleop` accepts no feedback."""
|
||||
pass
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.disconnect`]. Stops the keyboard listener, if one is running."""
|
||||
if self.listener is not None:
|
||||
self.listener.stop()
|
||||
|
||||
|
||||
class KeyboardEndEffectorTeleop(KeyboardTeleop):
|
||||
"""Keyboard teleoperator for end-effector (Cartesian delta) control.
|
||||
|
||||
Arrow keys and shift map to `delta_x`/`delta_y`/`delta_z`; `ctrl_l`/`ctrl_r` map to the gripper.
|
||||
Designed for use with the `So100FollowerEndEffector` robot.
|
||||
"""
|
||||
Teleop class to use keyboard inputs for end effector control.
|
||||
Designed to be used with the `So100FollowerEndEffector` robot.
|
||||
"""
|
||||
|
||||
config_class = KeyboardEndEffectorTeleopConfig
|
||||
name = "keyboard_ee"
|
||||
|
||||
def __init__(self, config: KeyboardEndEffectorTeleopConfig):
|
||||
"""Instantiate the teleoperator.
|
||||
|
||||
Args:
|
||||
config (`KeyboardEndEffectorTeleopConfig`):
|
||||
Configuration for this keyboard end-effector teleoperator.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
self.misc_keys_queue = Queue()
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict`: A 3-element (or 4-element if `config.use_gripper` is `True`) `float32` vector named
|
||||
`delta_x`, `delta_y`, `delta_z`, and optionally `gripper`.
|
||||
"""
|
||||
if self.config.use_gripper:
|
||||
return {
|
||||
"dtype": "float32",
|
||||
@@ -230,19 +177,6 @@ class KeyboardEndEffectorTeleop(KeyboardTeleop):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Translate held-down keys into an end-effector Cartesian delta.
|
||||
|
||||
Arrow keys drive `delta_x`/`delta_y`; `shift`/`shift_r` drive `delta_z`. `ctrl_r` opens the
|
||||
gripper and `ctrl_l` closes it (only present when `config.use_gripper` is `True`); any other
|
||||
pressed key is queued for [`~teleoperators.keyboard.KeyboardEndEffectorTeleop.get_teleop_events`]
|
||||
instead of affecting the action.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: `delta_x`, `delta_y`, `delta_z`, and, if enabled, `gripper`.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If [`~teleoperators.Teleoperator.connect`] has not been called.
|
||||
"""
|
||||
self._drain_pressed_keys()
|
||||
delta_x = 0.0
|
||||
delta_y = 0.0
|
||||
@@ -286,15 +220,22 @@ class KeyboardEndEffectorTeleop(KeyboardTeleop):
|
||||
return action_dict
|
||||
|
||||
def get_teleop_events(self) -> dict[str, Any]:
|
||||
"""Read auxiliary keyboard events used to drive episode control during recording.
|
||||
"""
|
||||
Get extra control events from the keyboard such as intervention status,
|
||||
episode termination, success indicators, etc.
|
||||
|
||||
Any of the movement/gripper keys held down counts as an active intervention. `s`, `r`, and `q`
|
||||
are read once as one-shot signals for success, rerecord, and quit respectively; reading this
|
||||
method clears the currently tracked key state.
|
||||
Keyboard mappings:
|
||||
- Any movement keys pressed = intervention active
|
||||
- 's' key = success (terminate episode successfully)
|
||||
- 'r' key = rerecord episode (terminate and rerecord)
|
||||
- 'q' key = quit episode (terminate without success)
|
||||
|
||||
Returns:
|
||||
`dict[TeleopEvents, bool]`: Values for the [`~teleoperators.TeleopEvents`] keys
|
||||
`IS_INTERVENTION`, `TERMINATE_EPISODE`, `SUCCESS`, and `RERECORD_EPISODE`.
|
||||
Dictionary containing:
|
||||
- is_intervention: bool - Whether human is currently intervening
|
||||
- terminate_episode: bool - Whether to terminate the current episode
|
||||
- success: bool - Whether the episode was successful
|
||||
- rerecord_episode: bool - Whether to rerecord the episode
|
||||
"""
|
||||
if not self.is_connected:
|
||||
return {
|
||||
@@ -345,24 +286,49 @@ class KeyboardEndEffectorTeleop(KeyboardTeleop):
|
||||
|
||||
|
||||
class KeyboardRoverTeleop(KeyboardTeleop):
|
||||
"""Keyboard teleoperator for mobile robots such as EarthRover Mini Plus.
|
||||
"""
|
||||
Keyboard teleoperator for mobile robots like EarthRover Mini Plus.
|
||||
|
||||
Provides WASD-style driving controls: `w`/`s` drive forward/backward, `a`/`d` turn (with a forward
|
||||
motion assist), `q`/`e` rotate in place, `x` is an emergency stop, and `+`/`-` adjust speed. `ESC`
|
||||
disconnects the teleoperator.
|
||||
Provides intuitive WASD-style controls for driving a mobile robot:
|
||||
- Linear movement (forward/backward)
|
||||
- Angular movement (turning/rotation)
|
||||
- Speed adjustment
|
||||
- Emergency stop
|
||||
|
||||
Keyboard Controls:
|
||||
Movement:
|
||||
- W: Move forward
|
||||
- S: Move backward
|
||||
- A: Turn left (with forward motion)
|
||||
- D: Turn right (with forward motion)
|
||||
- Q: Rotate left in place
|
||||
- E: Rotate right in place
|
||||
- X: Emergency stop
|
||||
|
||||
Speed Control:
|
||||
- +/=: Increase speed
|
||||
- -: Decrease speed
|
||||
|
||||
System:
|
||||
- ESC: Disconnect teleoperator
|
||||
|
||||
**Attributes**:
|
||||
- **current_linear_speed** (`float`) -- Current linear velocity magnitude, adjustable at runtime
|
||||
with `+`/`-`.
|
||||
- **current_angular_speed** (`float`) -- Current angular velocity magnitude, adjustable at
|
||||
runtime with `+`/`-`.
|
||||
- **config** -- Teleoperator configuration
|
||||
- **current_linear_speed** -- Current linear velocity magnitude
|
||||
- **current_angular_speed** -- Current angular velocity magnitude
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.keyboard import KeyboardRoverTeleop, KeyboardRoverTeleopConfig
|
||||
>>> teleop = KeyboardRoverTeleop(KeyboardRoverTeleopConfig(linear_speed=1.0)) # doctest: +SKIP
|
||||
>>> teleop.connect() # doctest: +SKIP
|
||||
>>> teleop.get_action() # doctest: +SKIP
|
||||
from lerobot.teleoperators.keyboard import KeyboardRoverTeleop, KeyboardRoverTeleopConfig
|
||||
|
||||
teleop = KeyboardRoverTeleop(
|
||||
KeyboardRoverTeleopConfig(linear_speed=1.0, angular_speed=1.0, speed_increment=0.1)
|
||||
)
|
||||
teleop.connect()
|
||||
|
||||
while teleop.is_connected:
|
||||
action = teleop.get_action()
|
||||
robot.send_action(action)
|
||||
```
|
||||
"""
|
||||
|
||||
@@ -370,12 +336,6 @@ class KeyboardRoverTeleop(KeyboardTeleop):
|
||||
name = "keyboard_rover"
|
||||
|
||||
def __init__(self, config: KeyboardRoverTeleopConfig):
|
||||
"""Instantiate the teleoperator.
|
||||
|
||||
Args:
|
||||
config (`KeyboardRoverTeleopConfig`):
|
||||
Configuration for this keyboard rover teleoperator.
|
||||
"""
|
||||
super().__init__(config)
|
||||
# Add rover-specific speed settings
|
||||
self.current_linear_speed = config.linear_speed
|
||||
@@ -383,11 +343,7 @@ class KeyboardRoverTeleop(KeyboardTeleop):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict`: `linear_velocity` and `angular_velocity`, each mapped to `float`.
|
||||
"""
|
||||
"""Return action format for rover (linear and angular velocities)."""
|
||||
return {
|
||||
"linear_velocity": float,
|
||||
"angular_velocity": float,
|
||||
@@ -395,11 +351,11 @@ class KeyboardRoverTeleop(KeyboardTeleop):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_calibrated`]. Rover teleop does not require calibration."""
|
||||
"""Rover teleop doesn't require calibration."""
|
||||
return True
|
||||
|
||||
def _drain_pressed_keys(self):
|
||||
"""Update current_pressed state from event queue without clearing held keys."""
|
||||
"""Update current_pressed state from event queue without clearing held keys"""
|
||||
while not self.event_queue.empty():
|
||||
key_char, is_pressed = self.event_queue.get_nowait()
|
||||
if is_pressed:
|
||||
@@ -410,18 +366,11 @@ class KeyboardRoverTeleop(KeyboardTeleop):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Translate held-down WASD-style keys into linear and angular rover velocities.
|
||||
|
||||
`w`/`s` set the linear velocity; `a`/`d` turn while adding a forward-motion assist
|
||||
(`config.turn_assist_ratio`) when not already moving; `q`/`e` rotate in place; `x` stops both
|
||||
axes. `+`/`-` adjust `current_linear_speed` and `current_angular_speed` in place, clamped to
|
||||
`config.min_linear_speed` / `config.min_angular_speed`.
|
||||
"""
|
||||
Get the current action based on pressed keys.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `linear_velocity` and `angular_velocity`.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If [`~teleoperators.Teleoperator.connect`] has not been called.
|
||||
RobotAction with 'linear_velocity' and 'angular_velocity' keys.
|
||||
"""
|
||||
before_read_t = time.perf_counter()
|
||||
|
||||
|
||||
@@ -22,28 +22,6 @@ from ..config import TeleoperatorConfig
|
||||
@TeleoperatorConfig.register_subclass("koch_leader")
|
||||
@dataclass
|
||||
class KochLeaderConfig(TeleoperatorConfig):
|
||||
"""Configuration for the Koch leader arm.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to, e.g. `/dev/ttyACM0` on Linux or `COM3` on Windows. Run
|
||||
`lerobot-find-port` to identify it.
|
||||
gripper_open_pos (`float`, *optional*, defaults to 50.0):
|
||||
Goal position written to the gripper motor, held under current-based position control so the
|
||||
gripper springs back to this position when released, letting it be used as a physical trigger.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular arm; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.koch_leader import KochLeader, KochLeaderConfig
|
||||
>>> config = KochLeaderConfig(port="/dev/ttyACM0") # doctest: +SKIP
|
||||
>>> teleop = KochLeader(config) # doctest: +SKIP
|
||||
```
|
||||
"""
|
||||
|
||||
# Port to connect to the arm
|
||||
port: str
|
||||
|
||||
|
||||
@@ -32,34 +32,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KochLeader(Teleoperator):
|
||||
"""The Koch leader arm, in either of its two revisions, held by an operator to teleoperate a follower arm.
|
||||
|
||||
"""
|
||||
- [Koch v1.0](https://github.com/AlexanderKoch-Koch/low_cost_robot), with and without the wrist-to-elbow
|
||||
expansion, developed by Alexander Koch from [Tau Robotics](https://tau-robotics.com).
|
||||
- [Koch v1.1](https://github.com/jess-moss/koch-v1-1), developed by Jess Moss.
|
||||
|
||||
Actions are keyed `"<motor>.pos"`. See [`~teleoperators.Teleoperator`] for the contract every method
|
||||
here implements.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.koch_leader import KochLeader, KochLeaderConfig
|
||||
>>> teleop = KochLeader(KochLeaderConfig(port="/dev/ttyACM0")) # doctest: +SKIP
|
||||
>>> with teleop: # doctest: +SKIP
|
||||
... action = teleop.get_action()
|
||||
```
|
||||
expansion, developed by Alexander Koch from [Tau Robotics](https://tau-robotics.com)
|
||||
- [Koch v1.1](https://github.com/jess-moss/koch-v1-1) developed by Jess Moss
|
||||
"""
|
||||
|
||||
config_class = KochLeaderConfig
|
||||
name = "koch_leader"
|
||||
|
||||
def __init__(self, config: KochLeaderConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`KochLeaderConfig`):
|
||||
The teleoperator's configuration. Its `port` determines what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
self.bus = DynamixelMotorsBus(
|
||||
@@ -77,42 +59,18 @@ class KochLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The arm's joint positions.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"<motor>.pos"` keys mapped to `float`.
|
||||
"""
|
||||
return {f"{motor}.pos": float for motor in self.bus.motors}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""Same as [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
This arm does not support feedback; [`~teleoperators.koch_leader.KochLeader.send_feedback`] always
|
||||
raises `NotImplementedError`.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Always empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
return self.bus.is_connected
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Connect the motor bus, calibrating and configuring the arm.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to run calibration when the motors disagree with the calibration file, or no file
|
||||
exists yet. Calibration is interactive and prompts on stdin.
|
||||
|
||||
Raises:
|
||||
DeviceAlreadyConnectedError: If the teleoperator is already connected.
|
||||
"""
|
||||
self.bus.connect()
|
||||
if not self.is_calibrated and calibrate:
|
||||
logger.info(
|
||||
@@ -125,16 +83,9 @@ class KochLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_calibrated`]."""
|
||||
return self.bus.is_calibrated
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Calibrate the arm, writing the result to the motors and the calibration file.
|
||||
|
||||
This is interactive: it prompts on stdin to reuse an existing calibration file, and otherwise asks
|
||||
you to move the arm to its middle position and then through each joint's full range. The
|
||||
`elbow_flex` motor is inverted, and `shoulder_pan` and `wrist_roll` are treated as full-turn joints.
|
||||
"""
|
||||
self.bus.disable_torque()
|
||||
if self.calibration:
|
||||
# Calibration file exists, ask user whether to use it or run new calibration
|
||||
@@ -181,12 +132,6 @@ class KochLeader(Teleoperator):
|
||||
logger.info(f"Calibration saved to {self.calibration_fpath}")
|
||||
|
||||
def configure(self) -> None:
|
||||
"""Write the operating modes to every motor, including the gripper's spring-back trigger behavior.
|
||||
|
||||
All motors except the gripper are set to extended position mode. The gripper is set to
|
||||
current-based position control and driven to `gripper_open_pos`, with torque enabled, so it springs
|
||||
back to that position when released and can be used as a physical trigger.
|
||||
"""
|
||||
self.bus.disable_torque()
|
||||
self.bus.configure_motors()
|
||||
for motor in self.bus.motors:
|
||||
@@ -209,11 +154,6 @@ class KochLeader(Teleoperator):
|
||||
self.bus.write("Goal_Position", "gripper", self.config.gripper_open_pos)
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one at a time.
|
||||
|
||||
Run this once when building an arm. It is interactive: it prompts you to connect the controller
|
||||
board to a single motor at a time, working from the gripper back to the base.
|
||||
"""
|
||||
for motor in reversed(self.bus.motors):
|
||||
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
||||
self.bus.setup_motor(motor)
|
||||
@@ -221,14 +161,6 @@ class KochLeader(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Same as [`~teleoperators.Teleoperator.get_action`].
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `"<motor>.pos"` keys mapped to the arm's current joint positions.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
action = self.bus.sync_read("Present_Position")
|
||||
action = {f"{motor}.pos": val for motor, val in action.items()}
|
||||
@@ -237,20 +169,10 @@ class KochLeader(Teleoperator):
|
||||
return action
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not implemented for this arm.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always. This arm does not support force feedback.
|
||||
"""
|
||||
# TODO(rcadene, aliberts): Implement force feedback
|
||||
raise NotImplementedError
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Same as [`~teleoperators.Teleoperator.disconnect`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
self.bus.disconnect()
|
||||
logger.info(f"{self} disconnected.")
|
||||
|
||||
@@ -22,28 +22,6 @@ from ..config import TeleoperatorConfig
|
||||
@TeleoperatorConfig.register_subclass("omx_leader")
|
||||
@dataclass
|
||||
class OmxLeaderConfig(TeleoperatorConfig):
|
||||
"""Configuration for the OMX leader arm.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to, e.g. `/dev/ttyACM0` on Linux or `COM3` on Windows. Run
|
||||
`lerobot-find-port` to identify it.
|
||||
gripper_open_pos (`float`, *optional*, defaults to 60.0):
|
||||
Goal position written to the gripper motor, held under current-based position control so the
|
||||
gripper springs back to this position when released, letting it be used as a physical trigger.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular arm; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.omx_leader import OmxLeader, OmxLeaderConfig
|
||||
>>> config = OmxLeaderConfig(port="/dev/ttyACM0") # doctest: +SKIP
|
||||
>>> teleop = OmxLeader(config) # doctest: +SKIP
|
||||
```
|
||||
"""
|
||||
|
||||
# Port to connect to the arm
|
||||
port: str
|
||||
|
||||
|
||||
@@ -32,33 +32,15 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OmxLeader(Teleoperator):
|
||||
"""The OMX leader arm, held by an operator to teleoperate a follower arm.
|
||||
|
||||
[OMX](https://github.com/ROBOTIS-GIT/open_manipulator), developed by Woojin Wie and Junha Cha from
|
||||
[ROBOTIS](https://ai.robotis.com/).
|
||||
|
||||
Actions are keyed `"<motor>.pos"`. See [`~teleoperators.Teleoperator`] for the contract every method
|
||||
here implements.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.omx_leader import OmxLeader, OmxLeaderConfig
|
||||
>>> teleop = OmxLeader(OmxLeaderConfig(port="/dev/ttyACM0")) # doctest: +SKIP
|
||||
>>> with teleop: # doctest: +SKIP
|
||||
... action = teleop.get_action()
|
||||
```
|
||||
"""
|
||||
- [OMX](https://github.com/ROBOTIS-GIT/open_manipulator),
|
||||
expansion, developed by Woojin Wie and Junha Cha from [ROBOTIS](https://ai.robotis.com/)
|
||||
"""
|
||||
|
||||
config_class = OmxLeaderConfig
|
||||
name = "omx_leader"
|
||||
|
||||
def __init__(self, config: OmxLeaderConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`OmxLeaderConfig`):
|
||||
The teleoperator's configuration. Its `port` determines what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
self.bus = DynamixelMotorsBus(
|
||||
@@ -76,42 +58,18 @@ class OmxLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The arm's joint positions.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"<motor>.pos"` keys mapped to `float`.
|
||||
"""
|
||||
return {f"{motor}.pos": float for motor in self.bus.motors}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""Same as [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
This arm does not support feedback; [`~teleoperators.omx_leader.OmxLeader.send_feedback`] always
|
||||
raises `NotImplementedError`.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Always empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
return self.bus.is_connected
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Connect the motor bus, calibrating and configuring the arm.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to write the factory default calibration when the motors disagree with the
|
||||
calibration file, or no file exists yet.
|
||||
|
||||
Raises:
|
||||
DeviceAlreadyConnectedError: If the teleoperator is already connected.
|
||||
"""
|
||||
self.bus.connect()
|
||||
if not self.is_calibrated and calibrate:
|
||||
logger.info(
|
||||
@@ -124,15 +82,9 @@ class OmxLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_calibrated`]."""
|
||||
return self.bus.is_calibrated
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Write the factory default calibration to the motors and the calibration file.
|
||||
|
||||
Unlike other SO/Koch-family arms, this is not interactive: the OMX arm's homing offsets and ranges
|
||||
of motion are fixed factory defaults, so no manual positioning is required.
|
||||
"""
|
||||
self.bus.disable_torque()
|
||||
logger.info(f"\nUsing factory default calibration values for {self}")
|
||||
logger.info(f"\nWriting default configuration of {self} to the motors")
|
||||
@@ -161,13 +113,6 @@ class OmxLeader(Teleoperator):
|
||||
logger.info(f"Calibration saved to {self.calibration_fpath}")
|
||||
|
||||
def configure(self) -> None:
|
||||
"""Write the operating and drive modes to every motor, including the gripper's spring-back trigger.
|
||||
|
||||
All motors except the gripper are set to extended position mode with a non-inverted drive mode. The
|
||||
gripper's drive mode is inverted, and it is set to current-based position control with a reduced
|
||||
current limit and driven to `gripper_open_pos`, with torque enabled, so it springs back to that
|
||||
position when released and can be used as a physical trigger.
|
||||
"""
|
||||
self.bus.disable_torque()
|
||||
self.bus.configure_motors()
|
||||
for motor in self.bus.motors:
|
||||
@@ -198,11 +143,6 @@ class OmxLeader(Teleoperator):
|
||||
self.bus.write("Goal_Position", "gripper", self.config.gripper_open_pos)
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one at a time.
|
||||
|
||||
Run this once when building an arm. It is interactive: it prompts you to connect the controller
|
||||
board to a single motor at a time, working from the gripper back to the base.
|
||||
"""
|
||||
for motor in reversed(self.bus.motors):
|
||||
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
||||
self.bus.setup_motor(motor)
|
||||
@@ -210,14 +150,6 @@ class OmxLeader(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Same as [`~teleoperators.Teleoperator.get_action`].
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `"<motor>.pos"` keys mapped to the arm's current joint positions.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
action = self.bus.sync_read("Present_Position")
|
||||
action = {f"{motor}.pos": val for motor, val in action.items()}
|
||||
@@ -226,20 +158,10 @@ class OmxLeader(Teleoperator):
|
||||
return action
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not implemented for this arm.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always. This arm does not support force feedback.
|
||||
"""
|
||||
# TODO(rcadene, aliberts): Implement force feedback
|
||||
raise NotImplementedError
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Same as [`~teleoperators.Teleoperator.disconnect`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
self.bus.disconnect()
|
||||
logger.info(f"{self} disconnected.")
|
||||
|
||||
@@ -76,41 +76,4 @@ class OpenArmLeaderConfigBase:
|
||||
@TeleoperatorConfig.register_subclass("openarm_leader")
|
||||
@dataclass
|
||||
class OpenArmLeaderConfig(TeleoperatorConfig, OpenArmLeaderConfigBase):
|
||||
"""Configuration for the OpenArm leader/teleoperator arm (CAN bus, Damiao motors).
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
CAN interface the arm is connected to, e.g. `"can0"` on Linux.
|
||||
can_interface (`str`, *optional*, defaults to `"socketcan"`):
|
||||
CAN backend type: `"socketcan"` (Linux), `"slcan"` (serial), or `"auto"` (auto-detect).
|
||||
use_can_fd (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use CAN FD, which OpenArm uses by default.
|
||||
can_bitrate (`int`, *optional*, defaults to 1000000):
|
||||
Nominal CAN bus bitrate, in bits per second.
|
||||
can_data_bitrate (`int`, *optional*, defaults to 5000000):
|
||||
CAN FD data-phase bitrate, in bits per second. Only used when `use_can_fd` is `True`.
|
||||
motor_config (`dict[str, tuple[int, int, str]]`, *optional*):
|
||||
Maps motor name to `(send_can_id, recv_can_id, motor_type)`. Defaults to the standard 7-DOF
|
||||
plus gripper OpenArm layout, using DM8009 (shoulder), DM4340 (shoulder rotation, elbow), and
|
||||
DM4310 (wrist, gripper) Damiao motors.
|
||||
manual_control (`bool`, *optional*, defaults to `True`):
|
||||
Whether motors have torque disabled for manual movement. Required for a leader arm that is
|
||||
moved by hand.
|
||||
use_velocity_and_torque (`bool`, *optional*, defaults to `False`):
|
||||
Whether to expose `.vel` and `.torque` per motor in [`~teleoperators.Teleoperator.action_features`],
|
||||
in addition to `.pos`.
|
||||
position_kp (`list[float]`, *optional*):
|
||||
Per-joint position gain, used for MIT torque control when `manual_control` is `False`.
|
||||
Defaults to the standard 8-value OpenArm gain set (one value per joint, plus gripper).
|
||||
position_kd (`list[float]`, *optional*):
|
||||
Per-joint velocity gain, used for MIT torque control when `manual_control` is `False`.
|
||||
Defaults to the standard 8-value OpenArm damping set (one value per joint, plus gripper).
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -30,33 +30,17 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenArmLeader(Teleoperator):
|
||||
"""OpenArm Leader/Teleoperator Arm with Damiao motors.
|
||||
"""
|
||||
OpenArm Leader/Teleoperator Arm with Damiao motors.
|
||||
|
||||
This teleoperator uses CAN bus communication to read positions from Damiao motors that are manually
|
||||
moved (torque disabled). For the bimanual setup, see [`~teleoperators.bi_openarm_leader.BiOpenArmLeader`], which composes
|
||||
two of these.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.openarm_leader import OpenArmLeader, OpenArmLeaderConfig
|
||||
>>> config = OpenArmLeaderConfig(port="can0")
|
||||
>>> leader = OpenArmLeader(config) # doctest: +SKIP
|
||||
>>> leader.connect() # doctest: +SKIP
|
||||
>>> action = leader.get_action() # doctest: +SKIP
|
||||
```
|
||||
This teleoperator uses CAN bus communication to read positions from
|
||||
Damiao motors that are manually moved (torque disabled).
|
||||
"""
|
||||
|
||||
config_class = OpenArmLeaderConfig
|
||||
name = "openarm_leader"
|
||||
|
||||
def __init__(self, config: OpenArmLeaderConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`OpenArmLeaderConfig`):
|
||||
The teleoperator's configuration. Its `port` and `motor_config` determine what is
|
||||
connected and how the CAN bus is laid out.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -82,11 +66,7 @@ class OpenArmLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Always includes `.pos` per motor; also includes `.vel` and `.torque` per motor when
|
||||
`config.use_velocity_and_torque` is `True`.
|
||||
"""
|
||||
"""Features produced by this teleoperator."""
|
||||
features: dict[str, type] = {}
|
||||
for motor in self.bus.motors:
|
||||
features[f"{motor}.pos"] = float
|
||||
@@ -97,23 +77,23 @@ class OpenArmLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
Always empty: feedback is not implemented for the OpenArm leader.
|
||||
"""
|
||||
"""Feedback features (not implemented for OpenArms)."""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
"""Check if teleoperator is connected."""
|
||||
return self.bus.is_connected
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.connect`].
|
||||
|
||||
For manual control, torque is disabled after connecting so the arm can be moved by hand.
|
||||
"""
|
||||
Connect to the teleoperator.
|
||||
|
||||
For manual control, we disable torque after connecting so the
|
||||
arm can be moved by hand.
|
||||
"""
|
||||
|
||||
# Connect to CAN bus
|
||||
logger.info(f"Connecting arm on {self.config.port}...")
|
||||
self.bus.connect()
|
||||
@@ -134,11 +114,12 @@ class OpenArmLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_calibrated`]."""
|
||||
"""Check if teleoperator is calibrated."""
|
||||
return self.bus.is_calibrated
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.calibrate`].
|
||||
"""
|
||||
Run calibration procedure for OpenArms leader.
|
||||
|
||||
The calibration procedure:
|
||||
1. Disable torque (if not already disabled)
|
||||
@@ -189,29 +170,26 @@ class OpenArmLeader(Teleoperator):
|
||||
print(f"Calibration saved to {self.calibration_fpath}")
|
||||
|
||||
def configure(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.configure`].
|
||||
|
||||
For manual control, torque is disabled so the arm can be moved by hand; otherwise the motors are
|
||||
configured for MIT torque control.
|
||||
"""
|
||||
Configure motors for manual teleoperation.
|
||||
|
||||
For manual control, we disable torque so the arm can be moved by hand.
|
||||
"""
|
||||
|
||||
return self.bus.disable_torque() if self.config.manual_control else self.bus.configure_motors()
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Not supported: raises `NotImplementedError`.
|
||||
|
||||
Motor ID configuration for CAN motors is typically done via manufacturer tools rather than through
|
||||
LeRobot.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Motor ID configuration is typically done via manufacturer tools for CAN motors."
|
||||
)
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""See [`~teleoperators.Teleoperator.get_action`].
|
||||
"""
|
||||
Get current action from the leader arm.
|
||||
|
||||
This is the main method for teleoperators - it reads the current state
|
||||
of the leader arm and returns it as an action that can be sent to a follower.
|
||||
|
||||
Reads all motor states (pos/vel/torque) in one CAN refresh cycle.
|
||||
"""
|
||||
@@ -234,20 +212,12 @@ class OpenArmLeader(Teleoperator):
|
||||
return action_dict
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported: raises `NotImplementedError`.
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, float]`):
|
||||
Unused.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError("Feedback is not yet implemented for OpenArm leader.")
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.disconnect`]."""
|
||||
"""Disconnect from teleoperator."""
|
||||
|
||||
# Disconnect CAN bus
|
||||
# For manual control, ensure torque is disabled before disconnecting
|
||||
self.bus.disconnect(disable_torque=self.config.manual_control)
|
||||
|
||||
@@ -36,22 +36,4 @@ class OpenArmMiniConfigBase:
|
||||
@TeleoperatorConfig.register_subclass("openarm_mini")
|
||||
@dataclass
|
||||
class OpenArmMiniConfig(TeleoperatorConfig, OpenArmMiniConfigBase):
|
||||
"""Configuration for the OpenArm Mini teleoperator (Feetech STS3215, 7DOF + gripper).
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the Feetech bus is connected to, e.g. `/dev/ttyUSB0`.
|
||||
side (`str`, *optional*):
|
||||
Which side of a bimanual pair this arm is: `"left"` or `"right"`. Controls per-joint
|
||||
direction flips applied during readout. `None` disables flipping.
|
||||
use_degrees (`bool`, *optional*, defaults to `True`):
|
||||
Keep `True` for backward compatibility with existing policies and datasets.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -46,32 +46,13 @@ GRIPPER_TELEOP_TO_DEGREES = -0.65
|
||||
class OpenArmMini(Teleoperator):
|
||||
"""OpenArm Mini single-arm teleoperator (Feetech STS3215, 7DOF + gripper).
|
||||
|
||||
For the bimanual setup, see [`~teleoperators.bi_openarm_mini.BiOpenArmMini`], which composes two of these.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.openarm_mini import OpenArmMini, OpenArmMiniConfig
|
||||
>>> config = OpenArmMiniConfig(port="/dev/ttyUSB0")
|
||||
>>> teleop = OpenArmMini(config) # doctest: +SKIP
|
||||
>>> teleop.connect() # doctest: +SKIP
|
||||
>>> action = teleop.get_action() # doctest: +SKIP
|
||||
```
|
||||
For the bimanual setup, see :class:`BiOpenArmMini` which composes two of these.
|
||||
"""
|
||||
|
||||
config_class = OpenArmMiniConfig
|
||||
name = "openarm_mini"
|
||||
|
||||
def __init__(self, config: OpenArmMiniConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`OpenArmMiniConfig`):
|
||||
The teleoperator's configuration. Its `port` and `side` determine what is connected and
|
||||
which per-joint direction flips are applied.
|
||||
|
||||
Raises:
|
||||
ValueError: If `config.side` is not `"left"`, `"right"`, or `None`.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -99,25 +80,18 @@ class OpenArmMini(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`]. One `.pos` entry per motor."""
|
||||
return {f"{motor}.pos": float for motor in self.bus.motors}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
Same shape as [`~teleoperators.Teleoperator.action_features`]: one `.pos` entry per motor.
|
||||
"""
|
||||
return self.action_features
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
return self.bus.is_connected
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.connect`]."""
|
||||
logger.info(f"Connecting arm on {self.config.port}...")
|
||||
self.bus.connect()
|
||||
|
||||
@@ -129,11 +103,11 @@ class OpenArmMini(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_calibrated`]."""
|
||||
return self.bus.is_calibrated
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.calibrate`].
|
||||
"""
|
||||
Run calibration procedure for a single OpenArm Mini arm.
|
||||
|
||||
1. Disable torque
|
||||
2. Ask user to position arm in hanging position with gripper closed
|
||||
@@ -227,23 +201,12 @@ class OpenArmMini(Teleoperator):
|
||||
print(f"\nCalibration complete and saved to {self.calibration_fpath}")
|
||||
|
||||
def configure(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.configure`].
|
||||
|
||||
Disables torque, applies bus-level motor configuration, then sets every motor to position
|
||||
operating mode.
|
||||
"""
|
||||
self.bus.disable_torque()
|
||||
self.bus.configure_motors()
|
||||
for motor in self.bus.motors:
|
||||
self.bus.write("Operating_Mode", motor, OperatingMode.POSITION.value)
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one at a time.
|
||||
|
||||
Run this once when building the teleoperator. Interactive: prompts you to connect the controller
|
||||
board to a single motor at a time, in reverse order so downstream motors on the daisy chain don't
|
||||
interfere.
|
||||
"""
|
||||
for motor in reversed(self.bus.motors):
|
||||
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
||||
self.bus.setup_motor(motor)
|
||||
@@ -251,11 +214,7 @@ class OpenArmMini(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""See [`~teleoperators.Teleoperator.get_action`].
|
||||
|
||||
Applies the `joint_6`/`joint_7` remap, the per-side direction flip configured by `config.side`,
|
||||
and the gripper teleop-to-degrees conversion before returning.
|
||||
"""
|
||||
"""Get current action (read positions from all motors)."""
|
||||
start = time.perf_counter()
|
||||
|
||||
positions = self.bus.sync_read("Present_Position")
|
||||
@@ -276,24 +235,13 @@ class OpenArmMini(Teleoperator):
|
||||
return action
|
||||
|
||||
def enable_torque(self) -> None:
|
||||
"""Enable torque on all motors, e.g. to hold position instead of being freely moved by hand."""
|
||||
self.bus.enable_torque()
|
||||
|
||||
def disable_torque(self) -> None:
|
||||
"""Disable torque on all motors so the arm can be moved by hand."""
|
||||
self.bus.disable_torque()
|
||||
|
||||
def write_goal_positions(self, positions: dict[str, float]) -> None:
|
||||
"""Write goal positions to the motors.
|
||||
|
||||
Applies the inverse of [`~teleoperators.openarm_mini.OpenArmMini.get_action`]'s remap, direction flip, and
|
||||
gripper unit conversion before writing.
|
||||
|
||||
Args:
|
||||
positions (`dict[str, float]`):
|
||||
Target positions keyed by `{motor}.pos`, in the same units [`~teleoperators.openarm_mini.OpenArmMini.get_action`]
|
||||
returns.
|
||||
"""
|
||||
"""Write goal positions to motors (inverse of get_action flip/gripper/remap logic)."""
|
||||
goals: dict[str, float] = {}
|
||||
for key, val in positions.items():
|
||||
if not key.endswith(".pos"):
|
||||
@@ -313,15 +261,9 @@ class OpenArmMini(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.send_feedback`].
|
||||
|
||||
Delegates to [`~teleoperators.openarm_mini.OpenArmMini.write_goal_positions`], moving the arm's motors to the
|
||||
given positions.
|
||||
"""
|
||||
self.write_goal_positions(feedback)
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.disconnect`]."""
|
||||
self.bus.disconnect()
|
||||
logger.info(f"{self} disconnected.")
|
||||
|
||||
@@ -23,14 +23,6 @@ from ..config import TeleoperatorConfig
|
||||
|
||||
|
||||
class PhoneOS(Enum):
|
||||
"""Which phone platform a `Phone` teleoperator talks to, selecting its backend implementation.
|
||||
|
||||
**Attributes**:
|
||||
- **ANDROID** (`str`) -- WebXR-based backend (`AndroidPhone`), driven through the `teleop` Python
|
||||
package.
|
||||
- **IOS** (`str`) -- ARKit-based backend (`IOSPhone`), driven through the HEBI Mobile I/O app.
|
||||
"""
|
||||
|
||||
ANDROID = "android"
|
||||
IOS = "ios"
|
||||
|
||||
@@ -38,35 +30,6 @@ class PhoneOS(Enum):
|
||||
@TeleoperatorConfig.register_subclass("phone")
|
||||
@dataclass
|
||||
class PhoneConfig(TeleoperatorConfig):
|
||||
"""Configuration for the [`~teleoperators.phone.Phone`] teleoperator.
|
||||
|
||||
Args:
|
||||
phone_os (`PhoneOS`, *optional*, defaults to `PhoneOS.IOS`):
|
||||
Which phone platform and backend to use. `PhoneOS.IOS` talks to the HEBI Mobile I/O app over
|
||||
ARKit; `PhoneOS.ANDROID` talks to a browser WebXR session over the `teleop` package.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
|
||||
Note:
|
||||
`camera_offset` is a fixed class attribute, not a constructor argument, so it currently cannot be
|
||||
overridden per instance or from the command line. It defaults to the offset between an iPhone 14
|
||||
Pro's camera and the phone's physical center (2cm lateral, 4cm vertical) and is applied to
|
||||
translate the ARKit/WebXR camera pose into the phone's own frame.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.phone import PhoneConfig
|
||||
>>> from lerobot.teleoperators.phone.config_phone import PhoneOS
|
||||
>>> config = PhoneConfig(phone_os=PhoneOS.ANDROID)
|
||||
>>> config.phone_os
|
||||
<PhoneOS.ANDROID: 'android'>
|
||||
```
|
||||
"""
|
||||
|
||||
phone_os: PhoneOS = PhoneOS.IOS
|
||||
camera_offset = np.array(
|
||||
[0.0, -0.02, 0.04]
|
||||
|
||||
@@ -26,7 +26,8 @@ from .config_phone import PhoneOS
|
||||
@ProcessorStepRegistry.register("map_phone_action_to_robot_action")
|
||||
@dataclass
|
||||
class MapPhoneActionToRobotAction(RobotActionProcessorStep):
|
||||
"""Maps calibrated phone pose actions to standardized robot action inputs.
|
||||
"""
|
||||
Maps calibrated phone pose actions to standardized robot action inputs.
|
||||
|
||||
This processor step acts as a bridge between the phone teleoperator's output
|
||||
and the robot's expected action format. It remaps the phone's 6-DoF pose
|
||||
@@ -44,22 +45,17 @@ class MapPhoneActionToRobotAction(RobotActionProcessorStep):
|
||||
_enabled_prev: bool = field(default=False, init=False, repr=False)
|
||||
|
||||
def action(self, action: RobotAction) -> RobotAction:
|
||||
"""Processes the phone action dictionary to create a robot action dictionary.
|
||||
"""
|
||||
Processes the phone action dictionary to create a robot action dictionary.
|
||||
|
||||
Args:
|
||||
action (`RobotAction`):
|
||||
The input action dictionary from the phone teleoperator, keyed `"phone.pos"`,
|
||||
`"phone.rot"`, `"phone.raw_inputs"`, and `"phone.enabled"`.
|
||||
act: The input action dictionary from the phone teleoperator.
|
||||
|
||||
Returns:
|
||||
`RobotAction`: A new action dictionary formatted for the robot controller, keyed
|
||||
`"enabled"`, `"target_x"`/`"target_y"`/`"target_z"`, `"target_wx"`/`"target_wy"`/`"target_wz"`,
|
||||
and `"gripper_vel"`.
|
||||
A new action dictionary formatted for the robot controller.
|
||||
|
||||
Raises:
|
||||
KeyError: If `"phone.pos"`, `"phone.rot"`, `"phone.raw_inputs"`, or `"phone.enabled"` is
|
||||
missing from `action`.
|
||||
ValueError: If `"phone.pos"` or `"phone.rot"` is `None`.
|
||||
ValueError: If 'pos' or 'rot' keys are missing from the input action.
|
||||
"""
|
||||
# Pop them from the action
|
||||
enabled = bool(action.pop("phone.enabled"))
|
||||
@@ -96,20 +92,6 @@ class MapPhoneActionToRobotAction(RobotActionProcessorStep):
|
||||
def transform_features(
|
||||
self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
|
||||
) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
|
||||
"""Replace the `phone.*` action feature entries with the robot action features `action` produces.
|
||||
|
||||
Drops the `"phone.enabled"`, `"phone.pos"`, `"phone.rot"`, and `"phone.raw_inputs"` feature
|
||||
entries, and adds one scalar (`shape=(1,)`) entry for each of `"enabled"`, `"target_x"`,
|
||||
`"target_y"`, `"target_z"`, `"target_wx"`, `"target_wy"`, `"target_wz"`, and `"gripper_vel"`.
|
||||
|
||||
Args:
|
||||
features (`dict[PipelineFeatureType, dict[str, PolicyFeature]]`):
|
||||
The pipeline's feature dictionary, keyed by pipeline feature type and then feature name.
|
||||
|
||||
Returns:
|
||||
`dict[PipelineFeatureType, dict[str, PolicyFeature]]`: The same dictionary, with the action
|
||||
feature entries updated in place.
|
||||
"""
|
||||
for feat in ["enabled", "pos", "rot", "raw_inputs"]:
|
||||
features[PipelineFeatureType.ACTION].pop(f"phone.{feat}", None)
|
||||
|
||||
|
||||
@@ -46,14 +46,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BasePhone:
|
||||
"""Shared calibration state and `Teleoperator` interface parts common to both phone backends.
|
||||
|
||||
`IOSPhone` and `AndroidPhone` mix this in alongside `Teleoperator` so that the action/feedback feature
|
||||
schemas, calibration status, and the no-op configuration step only need to be written once. Each
|
||||
backend implements the parts that genuinely differ: connecting, reading the raw pose, and capturing a
|
||||
calibration reference.
|
||||
"""
|
||||
|
||||
_enabled: bool = False
|
||||
_calib_pos: np.ndarray | None = None
|
||||
_calib_rot_inv: Rotation | None = None
|
||||
@@ -63,24 +55,10 @@ class BasePhone:
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Whether a calibration reference pose has been captured.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` once both a reference position and inverse rotation have been recorded by
|
||||
`calibrate`.
|
||||
"""
|
||||
return (self._calib_pos is not None) and (self._calib_rot_inv is not None)
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""Describe the action dictionary returned by `get_action`.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Maps `"phone.pos"` (3D position, shape `(3,)`), `"phone.rot"` (orientation,
|
||||
a `scipy.spatial.transform.Rotation`), `"phone.raw_inputs"` (device-specific analog/button or
|
||||
WebXR values), and `"phone.enabled"` (whether the teleoperation trigger is currently held) to
|
||||
their value types.
|
||||
"""
|
||||
return {
|
||||
"phone.pos": np.ndarray, # shape (3,)
|
||||
"phone.rot": Rotation, # scipy.spatial.transform.Rotation
|
||||
@@ -90,60 +68,22 @@ class BasePhone:
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""Feedback schema accepted by `send_feedback`.
|
||||
|
||||
No haptic or other feedback channel is implemented for phone teleoperators yet.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Currently always `None`, since `feedback_features` has no implementation
|
||||
yet; this deviates from the declared return type and should not be relied on.
|
||||
"""
|
||||
# No haptic or other feedback implemented yet
|
||||
pass
|
||||
|
||||
def configure(self) -> None:
|
||||
"""No-op. Phone teleoperators require no runtime configuration.
|
||||
|
||||
See [`~teleoperators.Teleoperator.configure`] for the base contract.
|
||||
"""
|
||||
# No additional configuration required for phone teleop
|
||||
pass
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not implemented. Phone teleoperators do not support feedback yet.
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, float]`):
|
||||
Feedback values; see [`~teleoperators.Teleoperator.send_feedback`] for the base contract.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always. Haptic feedback (phone vibration) is not implemented yet.
|
||||
"""
|
||||
# We could add haptic feedback (vibrations) here, but it's not implemented yet
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class IOSPhone(BasePhone, Teleoperator):
|
||||
"""ARKit-based teleoperator backend for iOS, driven through the HEBI Mobile I/O app.
|
||||
|
||||
Reads the phone's 6-DoF pose (position and orientation) captured by ARKit and relayed over the HEBI
|
||||
SDK, along with the app's 8 analog (`a1`-`a8`) and 8 digital (`b1`-`b8`) inputs. `Phone` instantiates
|
||||
this internally when `PhoneConfig.phone_os` is `PhoneOS.IOS`; use `Phone` directly rather than this
|
||||
class.
|
||||
"""
|
||||
|
||||
name = "ios_phone"
|
||||
|
||||
def __init__(self, config: PhoneConfig):
|
||||
"""Check for the optional dependencies this backend needs and store the configuration.
|
||||
|
||||
Args:
|
||||
config (`PhoneConfig`):
|
||||
Configuration shared with the parent `Phone` teleoperator.
|
||||
|
||||
Raises:
|
||||
ImportError: If the `hebi-py` or `teleop` packages are not installed.
|
||||
"""
|
||||
require_package("hebi-py", extra="phone", import_name="hebi")
|
||||
require_package("teleop", extra="phone")
|
||||
super().__init__(config)
|
||||
@@ -152,26 +92,10 @@ class IOSPhone(BasePhone, Teleoperator):
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`].
|
||||
|
||||
Returns:
|
||||
`bool`: `True` once a HEBI feedback group has been acquired by `connect`.
|
||||
"""
|
||||
return self._group is not None
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self) -> None:
|
||||
"""Look up the HEBI Mobile I/O group over the network, then calibrate.
|
||||
|
||||
Waits briefly for the HEBI lookup service to discover the phone running the Mobile I/O app under
|
||||
the `"HEBI"` family / `"mobileIO"` name, then immediately runs `calibrate`, which blocks until the
|
||||
user captures a reference pose in the app. Unlike
|
||||
[`~teleoperators.Teleoperator.connect`], this method always calibrates; there is no way to skip it.
|
||||
|
||||
Raises:
|
||||
DeviceAlreadyConnectedError: If already connected.
|
||||
RuntimeError: If no matching Mobile I/O group is found on the network.
|
||||
"""
|
||||
logger.info("Connecting to IPhone, make sure to open the HEBI Mobile I/O app.")
|
||||
lookup = hebi.Lookup()
|
||||
time.sleep(2.0)
|
||||
@@ -184,13 +108,6 @@ class IOSPhone(BasePhone, Teleoperator):
|
||||
self.calibrate()
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Block until the user captures a reference pose via the HEBI Mobile I/O app.
|
||||
|
||||
Prompts the user to hold the phone so its top edge points along the robot's +x axis and its
|
||||
screen faces the robot's +z axis, then to press and hold button `B1` in the app to capture that
|
||||
pose as the calibration reference. See [`~teleoperators.Teleoperator.calibrate`] for the base
|
||||
contract.
|
||||
"""
|
||||
print(
|
||||
"Hold the phone so that: top edge points forward in same direction as the robot (robot +x) and screen points up (robot +z)"
|
||||
)
|
||||
@@ -202,7 +119,8 @@ class IOSPhone(BasePhone, Teleoperator):
|
||||
print("Calibration done\n")
|
||||
|
||||
def _wait_for_capture_trigger(self) -> tuple[np.ndarray, Rotation]:
|
||||
"""Blocks execution until the calibration trigger is detected from the iOS device.
|
||||
"""
|
||||
Blocks execution until the calibration trigger is detected from the iOS device.
|
||||
|
||||
This method enters a loop, continuously reading the phone's state. It waits for the user to press
|
||||
and hold the 'B1' button in the HEBI Mobile I/O app. Once B1 is pressed, the loop breaks and
|
||||
@@ -229,7 +147,8 @@ class IOSPhone(BasePhone, Teleoperator):
|
||||
time.sleep(0.01)
|
||||
|
||||
def _read_current_pose(self) -> tuple[bool, np.ndarray | None, Rotation | None, object | None]:
|
||||
"""Reads the instantaneous 6-DoF pose from the connected iOS device via the HEBI SDK.
|
||||
"""
|
||||
Reads the instantaneous 6-DoF pose from the connected iOS device via the HEBI SDK.
|
||||
|
||||
This method fetches the latest feedback packet from the HEBI group, extracts the ARKit
|
||||
position and orientation, and converts them into a standard format. It also applies a
|
||||
@@ -264,20 +183,6 @@ class IOSPhone(BasePhone, Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict:
|
||||
"""Read the phone's current calibrated pose and raw HEBI inputs.
|
||||
|
||||
Applies the calibration captured by `calibrate` to the raw ARKit pose, and re-anchors the
|
||||
reference position on the rising edge of the `b1` "enable" button so that moving the phone while
|
||||
disabled does not cause a jump once teleoperation resumes.
|
||||
|
||||
Returns:
|
||||
`dict`: Matches `action_features`: `"phone.pos"`, `"phone.rot"`, `"phone.raw_inputs"` (the
|
||||
app's analog/digital channel values, keyed e.g. `"a1"`, `"b1"`), and `"phone.enabled"`. An
|
||||
empty `dict` if no pose has been received yet or the teleoperator has not been calibrated.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If `connect` has not been called.
|
||||
"""
|
||||
has_pose, raw_position, raw_rotation, fb_pose = self._read_current_pose()
|
||||
if not has_pose or not self.is_calibrated:
|
||||
return {}
|
||||
@@ -319,34 +224,13 @@ class IOSPhone(BasePhone, Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.disconnect`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If `connect` has not been called.
|
||||
"""
|
||||
self._group = None
|
||||
|
||||
|
||||
class AndroidPhone(BasePhone, Teleoperator):
|
||||
"""WebXR-based teleoperator backend for Android, driven through the `teleop` Python package.
|
||||
|
||||
Runs the `teleop` package's local WebXR server on a background thread and reads the pose and touch
|
||||
events posted by the phone's browser session. `Phone` instantiates this internally when
|
||||
`PhoneConfig.phone_os` is `PhoneOS.ANDROID`; use `Phone` directly rather than this class.
|
||||
"""
|
||||
|
||||
name = "android_phone"
|
||||
|
||||
def __init__(self, config: PhoneConfig):
|
||||
"""Check for the optional dependencies this backend needs and store the configuration.
|
||||
|
||||
Args:
|
||||
config (`PhoneConfig`):
|
||||
Configuration shared with the parent `Phone` teleoperator.
|
||||
|
||||
Raises:
|
||||
ImportError: If the `hebi-py` or `teleop` packages are not installed.
|
||||
"""
|
||||
require_package("hebi-py", extra="phone", import_name="hebi")
|
||||
require_package("teleop", extra="phone")
|
||||
super().__init__(config)
|
||||
@@ -359,26 +243,10 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`].
|
||||
|
||||
Returns:
|
||||
`bool`: `True` once the `teleop` background thread has been started by `connect`.
|
||||
"""
|
||||
return self._teleop is not None
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self) -> None:
|
||||
"""Start the `teleop` WebXR server on a background thread, then calibrate.
|
||||
|
||||
Subscribes to pose/message updates from the `teleop` package and starts its server loop on a
|
||||
daemon thread, then immediately runs `calibrate`, which blocks until the user captures a reference
|
||||
pose from the phone's browser session. Unlike
|
||||
[`~teleoperators.Teleoperator.connect`], this method always calibrates; there is no way to skip
|
||||
it.
|
||||
|
||||
Raises:
|
||||
DeviceAlreadyConnectedError: If already connected.
|
||||
"""
|
||||
logger.info("Starting teleop stream for Android...")
|
||||
self._teleop = Teleop()
|
||||
self._teleop.subscribe(self._android_callback)
|
||||
@@ -389,13 +257,6 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
self.calibrate()
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Block until the user captures a reference pose via touch on the WebXR page.
|
||||
|
||||
Prompts the user to hold the phone so its top edge points along the robot's +x axis and its
|
||||
screen faces the robot's +z axis, then to touch and move a finger on the WebXR page to capture
|
||||
that pose as the calibration reference. See [`~teleoperators.Teleoperator.calibrate`] for the base
|
||||
contract.
|
||||
"""
|
||||
print(
|
||||
"Hold the phone so that: top edge points forward in same direction as the robot (robot +x) and screen points up (robot +z)"
|
||||
)
|
||||
@@ -408,7 +269,8 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
print("Calibration done\n")
|
||||
|
||||
def _wait_for_capture_trigger(self) -> tuple[np.ndarray, Rotation]:
|
||||
"""Blocks execution until the calibration trigger is detected from the Android device.
|
||||
"""
|
||||
Blocks execution until the calibration trigger is detected from the Android device.
|
||||
|
||||
This method enters a loop, continuously checking the latest message received from the WebXR
|
||||
session. It waits for the user to touch and move their finger on the screen, which generates
|
||||
@@ -431,7 +293,8 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
time.sleep(0.01)
|
||||
|
||||
def _read_current_pose(self) -> tuple[bool, np.ndarray | None, Rotation | None, object | None]:
|
||||
"""Reads the latest 6-DoF pose received from the Android device's WebXR session.
|
||||
"""
|
||||
Reads the latest 6-DoF pose received from the Android device's WebXR session.
|
||||
|
||||
This method accesses the most recent pose data stored by the `_android_callback`. It uses a
|
||||
thread lock to safely read the shared `_latest_pose` variable. The pose, a 4x4 matrix, is
|
||||
@@ -454,7 +317,8 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
return True, pos, rot, pose
|
||||
|
||||
def _android_callback(self, pose: np.ndarray, message: dict) -> None:
|
||||
"""Callback function to handle incoming data from the Android teleop stream.
|
||||
"""
|
||||
Callback function to handle incoming data from the Android teleop stream.
|
||||
|
||||
This method is executed by the `teleop` package's subscriber thread whenever a new
|
||||
pose and message are received from the WebXR session on the Android phone. It updates
|
||||
@@ -472,20 +336,6 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict:
|
||||
"""Read the phone's current calibrated pose and raw touch/button state.
|
||||
|
||||
Applies the calibration captured by `calibrate` to the latest pose received from the `teleop`
|
||||
background thread, and re-anchors the reference position on the rising edge of the `"move"` touch
|
||||
event so that moving the phone while disabled does not cause a jump once teleoperation resumes.
|
||||
|
||||
Returns:
|
||||
`dict`: Matches `action_features`: `"phone.pos"`, `"phone.rot"`, `"phone.raw_inputs"`
|
||||
(`"move"`, `"scale"`, `"reservedButtonA"`, `"reservedButtonB"`), and `"phone.enabled"`. An
|
||||
empty `dict` if no pose has been received yet or the teleoperator has not been calibrated.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If `connect` has not been called.
|
||||
"""
|
||||
ok, raw_pos, raw_rot, pose = self._read_current_pose()
|
||||
if not ok or not self.is_calibrated:
|
||||
return {}
|
||||
@@ -519,11 +369,6 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Stop the `teleop` background thread.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If `connect` has not been called.
|
||||
"""
|
||||
self._teleop = None
|
||||
if self._teleop_thread and self._teleop_thread.is_alive():
|
||||
self._teleop_thread.join(timeout=1.0)
|
||||
@@ -532,42 +377,18 @@ class AndroidPhone(BasePhone, Teleoperator):
|
||||
|
||||
|
||||
class Phone(Teleoperator):
|
||||
"""Phone-based teleoperator: iOS via ARKit and the HEBI Mobile I/O app, Android via WebXR.
|
||||
"""
|
||||
Phone-based teleoperator using ARKit (iOS via HEBI Mobile I/O App) or the teleop Python package (Android via WebXR API).
|
||||
For HEBI Mobile I/O we also expose 8 analog (a1-a8) and 8 digital (b1-b8) inputs.
|
||||
|
||||
Reads the phone's 6-DoF pose and, for the HEBI Mobile I/O app, 8 analog (`a1`-`a8`) and 8 digital
|
||||
(`b1`-`b8`) inputs. Which backend is used is picked at construction time from
|
||||
`config.phone_os` and delegated to internally: [`~teleoperators.Teleoperator`] method calls on `Phone`
|
||||
forward to either an `IOSPhone` or an `AndroidPhone` instance.
|
||||
|
||||
Press and hold **B1** (iOS) or touch and move on the WebXR page (Android) to enable teleoperation.
|
||||
The first press/touch while enabled captures a reference pose; releasing and re-triggering re-anchors
|
||||
the reference position to wherever the phone currently is, so motion is always relative to where
|
||||
teleoperation was last resumed.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.phone import Phone, PhoneConfig
|
||||
>>> teleop = Phone(PhoneConfig()) # doctest: +SKIP
|
||||
>>> teleop.connect() # doctest: +SKIP
|
||||
>>> teleop.get_action() # doctest: +SKIP
|
||||
```
|
||||
Press and hold **B1** to enable teleoperation. While enabled, the first B1 press
|
||||
captures a reference pose and rotation, when disabled and pressed again the position is reapplied.
|
||||
"""
|
||||
|
||||
config_class = PhoneConfig
|
||||
name = "phone"
|
||||
|
||||
def __init__(self, config: PhoneConfig):
|
||||
"""Pick and construct the backend matching `config.phone_os`.
|
||||
|
||||
Args:
|
||||
config (`PhoneConfig`):
|
||||
Configuration selecting the phone platform (`config.phone_os`) and forwarded to the
|
||||
chosen backend.
|
||||
|
||||
Raises:
|
||||
ValueError: If `config.phone_os` is not a valid `PhoneOS` member.
|
||||
ImportError: If the `hebi-py` or `teleop` packages are not installed.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -582,89 +403,34 @@ class Phone(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_connected`].
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if the underlying `IOSPhone` or `AndroidPhone` backend is connected.
|
||||
"""
|
||||
return self._phone_impl.is_connected
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect and calibrate through the underlying backend.
|
||||
|
||||
Unlike [`~teleoperators.Teleoperator.connect`], this always calibrates; there is no `calibrate`
|
||||
argument to opt out.
|
||||
|
||||
Raises:
|
||||
DeviceAlreadyConnectedError: If already connected.
|
||||
RuntimeError: If the iOS backend cannot find the Mobile I/O group on the network.
|
||||
"""
|
||||
return self._phone_impl.connect()
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.calibrate`]. Delegates to the underlying backend."""
|
||||
return self._phone_impl.calibrate()
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""See [`~teleoperators.Teleoperator.is_calibrated`].
|
||||
|
||||
Returns:
|
||||
`bool`: `True` once a calibration reference pose has been captured.
|
||||
"""
|
||||
return self._phone_impl.is_calibrated
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"phone.pos"`, `"phone.rot"`, `"phone.raw_inputs"`, and `"phone.enabled"`
|
||||
mapped to their value types; see `get_action` for what each holds.
|
||||
"""
|
||||
return self._phone_impl.action_features
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""See [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Currently always `None`, since no feedback channel is implemented yet.
|
||||
"""
|
||||
return self._phone_impl.feedback_features
|
||||
|
||||
def configure(self) -> None:
|
||||
"""No-op. See [`~teleoperators.Teleoperator.configure`]."""
|
||||
return self._phone_impl.configure()
|
||||
|
||||
def get_action(self) -> dict:
|
||||
"""Read the phone's current calibrated pose and raw inputs from the underlying backend.
|
||||
|
||||
Returns:
|
||||
`dict`: Matches `action_features`. An empty `dict` if no pose has been received yet or the
|
||||
teleoperator has not been calibrated.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If `connect` has not been called.
|
||||
"""
|
||||
return self._phone_impl.get_action()
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not implemented. See [`~teleoperators.Teleoperator.send_feedback`].
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, float]`):
|
||||
Feedback values; unused.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always. Haptic feedback is not implemented yet.
|
||||
"""
|
||||
return self._phone_impl.send_feedback(feedback)
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""See [`~teleoperators.Teleoperator.disconnect`]. Delegates to the underlying backend.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If `connect` has not been called.
|
||||
"""
|
||||
return self._phone_impl.disconnect()
|
||||
|
||||
@@ -22,37 +22,6 @@ from ..config import TeleoperatorConfig
|
||||
@TeleoperatorConfig.register_subclass("reachy2_teleoperator")
|
||||
@dataclass
|
||||
class Reachy2TeleoperatorConfig(TeleoperatorConfig):
|
||||
"""Configuration for reading teleoperation actions from a Reachy 2.
|
||||
|
||||
Reachy 2 can act as its own teleoperator: instead of a leader arm, another Reachy 2 (or the same one in
|
||||
a different mode) reports its joint positions over the network as the action. There is no LeRobot
|
||||
calibration file; Reachy 2 manages its own calibration.
|
||||
|
||||
Which joints are reported is selected by the `with_*` flags: turning a part off removes its joints
|
||||
entirely. At least one part must stay enabled.
|
||||
|
||||
Args:
|
||||
ip_address (`str`, *optional*, defaults to `"localhost"`):
|
||||
Address of the Reachy 2 robot to read actions from.
|
||||
use_present_position (`bool`, *optional*, defaults to `False`):
|
||||
Whether to report each joint's present position as the action. If `False`, the joint's goal
|
||||
position is reported instead.
|
||||
with_mobile_base (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include the mobile base's velocity in actions.
|
||||
with_l_arm (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include the left arm's joints.
|
||||
with_r_arm (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include the right arm's joints.
|
||||
with_neck (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include the neck's joints.
|
||||
with_antennas (`bool`, *optional*, defaults to `True`):
|
||||
Whether to include the antennas' joints.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular teleoperator.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Unused: Reachy 2 manages its own calibration.
|
||||
"""
|
||||
|
||||
# IP address of the Reachy 2 robot used as teleoperator
|
||||
ip_address: str | None = "localhost"
|
||||
|
||||
@@ -68,11 +37,6 @@ class Reachy2TeleoperatorConfig(TeleoperatorConfig):
|
||||
with_antennas: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that at least one robot part is enabled.
|
||||
|
||||
Raises:
|
||||
ValueError: If every robot part is disabled, which would leave no joints to report.
|
||||
"""
|
||||
if not (
|
||||
self.with_mobile_base
|
||||
or self.with_l_arm
|
||||
|
||||
@@ -76,19 +76,14 @@ REACHY2_VEL = {
|
||||
|
||||
|
||||
class Reachy2Teleoperator(Teleoperator):
|
||||
"""[Reachy 2](https://www.pollen-robotics.com/reachy/), by Pollen Robotics."""
|
||||
"""
|
||||
[Reachy 2](https://www.pollen-robotics.com/reachy/), by Pollen Robotics.
|
||||
"""
|
||||
|
||||
config_class = Reachy2TeleoperatorConfig
|
||||
name = "reachy2_specific"
|
||||
|
||||
def __init__(self, config: Reachy2TeleoperatorConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`Reachy2TeleoperatorConfig`):
|
||||
The teleoperator's configuration. Its `ip_address` and `with_*` flags determine what is
|
||||
read.
|
||||
"""
|
||||
require_package("reachy2_sdk", extra="reachy2")
|
||||
super().__init__(config)
|
||||
|
||||
@@ -111,13 +106,6 @@ class Reachy2Teleoperator(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The joint positions (and mobile base velocity, if enabled) read from Reachy 2.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"<joint>.pos"` keys for each enabled part mapped to `float`, plus
|
||||
`"mobile_base.vx"`, `"mobile_base.vy"`, and `"mobile_base.vtheta"` when
|
||||
`config.with_mobile_base` is `True`.
|
||||
"""
|
||||
if self.config.with_mobile_base:
|
||||
return {
|
||||
**dict.fromkeys(
|
||||
@@ -134,32 +122,14 @@ class Reachy2Teleoperator(Teleoperator):
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""Always empty: this teleoperator does not accept feedback.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: An empty dictionary.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
return self.reachy.is_connected() if self.reachy is not None else False
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Open the gRPC connection to Reachy 2's teleoperation interface.
|
||||
|
||||
The `calibrate` argument is accepted for interface compatibility but has no effect: Reachy 2
|
||||
manages its own calibration.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Unused.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the connection could not be established.
|
||||
"""
|
||||
self.reachy = ReachySDK(self.config.ip_address)
|
||||
|
||||
if not self.is_connected:
|
||||
@@ -168,32 +138,16 @@ class Reachy2Teleoperator(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Always `True`: Reachy 2 manages its own calibration.
|
||||
|
||||
Returns:
|
||||
`bool`: Always `True`.
|
||||
"""
|
||||
return True
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""No-op: Reachy 2 manages its own calibration."""
|
||||
pass
|
||||
|
||||
def configure(self) -> None:
|
||||
"""No-op: Reachy 2 requires no additional configuration."""
|
||||
pass
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Read the current (or goal) joint positions and mobile base velocity from Reachy 2.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: Values keyed as described by
|
||||
[`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If [`~teleoperators.Teleoperator.connect`] has not been called.
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
|
||||
joint_action: dict[str, float] = {}
|
||||
@@ -216,14 +170,8 @@ class Reachy2Teleoperator(Teleoperator):
|
||||
return {**joint_action, **vel_action}
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always. This teleoperator does not accept feedback.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Close the gRPC connection to Reachy 2, if it is open."""
|
||||
if self.is_connected:
|
||||
self.reachy.disconnect()
|
||||
|
||||
@@ -21,14 +21,10 @@ from ..config import TeleoperatorConfig
|
||||
|
||||
@dataclass
|
||||
class RebotArm102LeaderConfig:
|
||||
"""Field definitions shared by the reBot Arm 102 leader.
|
||||
"""Base configuration class for the Seeed Studio StarArm102 / reBot Arm 102 leader.
|
||||
|
||||
The reBot Arm 102 is a 7-joint (incl. gripper) leader arm driven by FashionStar UART smart servos.
|
||||
Servo communication goes through ``motorbridge-smart-servo``.
|
||||
|
||||
This class only carries the fields. The registered configuration users instantiate is
|
||||
[`RebotArm102LeaderTeleopConfig`], which combines these with [`~teleoperators.TeleoperatorConfig`] and
|
||||
documents them all in one place — doc-builder renders only a class's own docstring, never its bases'.
|
||||
The reBot Arm 102 is a 7-joint (incl. gripper) leader arm driven by FashionStar
|
||||
UART smart servos. Servo communication goes through ``motorbridge-smart-servo``.
|
||||
"""
|
||||
|
||||
# USB-to-UART device the leader arm is connected to (e.g. "/dev/ttyUSB0").
|
||||
@@ -82,28 +78,6 @@ class RebotArm102LeaderConfig:
|
||||
@TeleoperatorConfig.register_subclass("rebot_102_leader")
|
||||
@dataclass
|
||||
class RebotArm102LeaderTeleopConfig(TeleoperatorConfig, RebotArm102LeaderConfig):
|
||||
"""Registered configuration for the reBot Arm 102 leader teleoperator.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
USB-to-UART device the leader arm is connected to, e.g. `/dev/ttyUSB0`.
|
||||
baudrate (`int`, *optional*, defaults to 1000000):
|
||||
Baud rate of the UART link to the FashionStar smart servos.
|
||||
joint_ids (`dict[str, int]`, *optional*):
|
||||
Servo id of each joint on the UART bus. Defaults to the reBot Arm 102's standard 7-joint
|
||||
layout (`shoulder_pan`, `shoulder_lift`, `elbow_flex`, `wrist_flex`, `wrist_yaw`,
|
||||
`wrist_roll`, `gripper`).
|
||||
joint_directions (`dict[str, int]`, *optional*):
|
||||
Per-joint sign applied to raw servo angles so the leader matches the follower convention. The
|
||||
gripper additionally carries a scale (e.g. `-6`) to widen its range to the reBot B601
|
||||
follower's gripper travel.
|
||||
joint_ranges (`dict[str, list[int]]`, *optional*):
|
||||
Per-joint `[min, max]` output range in degrees. Defaults to ranges matching the reBot B601
|
||||
follower's joint limits so leader actions can drive the follower key-for-key.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular arm; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
"""
|
||||
"""Registered configuration for the reBot Arm 102 leader teleoperator."""
|
||||
|
||||
pass
|
||||
|
||||
@@ -49,12 +49,6 @@ class RebotArm102Leader(Teleoperator):
|
||||
name = "rebot_102_leader"
|
||||
|
||||
def __init__(self, config: RebotArm102LeaderTeleopConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`RebotArm102LeaderTeleopConfig`):
|
||||
The teleoperator's configuration. Its `port` determines what is connected.
|
||||
"""
|
||||
require_package("motorbridge-smart-servo", extra="rebot", import_name="motorbridge_smart_servo")
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
@@ -64,39 +58,18 @@ class RebotArm102Leader(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The arm's joint positions, in degrees.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"<motor>.pos"` keys mapped to `float`.
|
||||
"""
|
||||
return {f"{motor}.pos": float for motor in self.motor_names}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""This arm accepts no feedback.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Always empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`]: the servo bus has been opened."""
|
||||
return self.bus is not None
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Open the UART servo bus, ping every configured joint, then calibrate and configure the arm.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to run calibration when the arm is not already calibrated. Calibration is
|
||||
interactive and prompts on stdin.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If a configured servo does not respond to a ping.
|
||||
"""
|
||||
logger.info(f"Connecting {self} on {self.config.port}...")
|
||||
bus = FashionStarServo(self.config.port, baudrate=self.config.baudrate)
|
||||
try:
|
||||
@@ -122,20 +95,9 @@ class RebotArm102Leader(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Whether every configured joint has a saved calibration entry.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if `self.calibration` has an entry for each of `self.motor_names`.
|
||||
"""
|
||||
return bool(self.calibration) and set(self.calibration) == set(self.motor_names)
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Set the zero position of every joint from the arm's current pose.
|
||||
|
||||
If a calibration file already exists, prompts the operator to reuse it or to redo calibration. To
|
||||
redo it, the operator manually moves the arm to its zero pose (gripper closed); each servo's
|
||||
origin point is then reset to that pose and the result is saved to the calibration file.
|
||||
"""
|
||||
if self.calibration:
|
||||
user_input = input(
|
||||
f"Press ENTER to use provided calibration file associated with the id {self.id}, "
|
||||
@@ -170,10 +132,6 @@ class RebotArm102Leader(Teleoperator):
|
||||
logger.info(f"Calibration saved to {self.calibration_fpath}")
|
||||
|
||||
def configure(self) -> None:
|
||||
"""Unlock every servo's torque and reset each one's multi-turn counter.
|
||||
|
||||
Run once after connecting so subsequent readings start from a known turn count.
|
||||
"""
|
||||
for motor_id in self.config.joint_ids.values():
|
||||
self.bus.unlock(motor_id)
|
||||
time.sleep(_SETTLE_SEC)
|
||||
@@ -207,16 +165,6 @@ class RebotArm102Leader(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> RobotAction:
|
||||
"""Read, unwrap, and sign-correct the current joint positions.
|
||||
|
||||
Each joint's raw multi-turn angle is unwrapped into its configured range (see
|
||||
`_round_to_valid_range`), then flipped and clipped according to `joint_directions` and
|
||||
`joint_ranges` so the result matches the follower's convention. If reading the servos fails, the
|
||||
last successfully read positions are reused and the caller is expected to stop teleoperation.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `"<motor>.pos"` keys mapped to the joint's position in degrees.
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
raw_positions = self._read_raw_positions()
|
||||
@@ -250,16 +198,10 @@ class RebotArm102Leader(Teleoperator):
|
||||
return action_dict
|
||||
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Not supported: the leader arm has no actuators to receive feedback.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always.
|
||||
"""
|
||||
raise NotImplementedError("Feedback is not implemented for the reBot Arm 102 leader.")
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Close the UART servo bus."""
|
||||
self.bus.close()
|
||||
self.bus = None
|
||||
logger.info(f"{self} disconnected.")
|
||||
|
||||
@@ -21,12 +21,7 @@ from ..config import TeleoperatorConfig
|
||||
|
||||
@dataclass
|
||||
class SOLeaderConfig:
|
||||
"""Field definitions shared by the SO-family leader arms.
|
||||
|
||||
This class only carries the fields. The registered configuration users instantiate is
|
||||
[`SOLeaderTeleopConfig`], which combines these with [`~teleoperators.TeleoperatorConfig`] and documents
|
||||
them all in one place — doc-builder renders only a class's own docstring, never its bases'.
|
||||
"""
|
||||
"""Base configuration class for SO Leader teleoperators."""
|
||||
|
||||
# Port to connect to the arm
|
||||
port: str
|
||||
@@ -45,36 +40,6 @@ class SOLeaderConfig:
|
||||
@TeleoperatorConfig.register_subclass("so100_leader")
|
||||
@dataclass
|
||||
class SOLeaderTeleopConfig(TeleoperatorConfig, SOLeaderConfig):
|
||||
"""Configuration for the SO-100 and SO-101 leader arms.
|
||||
|
||||
Both arms share this class; `SO100LeaderConfig` and `SO101LeaderConfig` are aliases for it. They differ
|
||||
in their calibration and gearing, not in their control code.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to, e.g. `/dev/ttyACM0` on Linux or `COM3` on Windows. Run
|
||||
`lerobot-find-port` to identify it.
|
||||
use_degrees (`bool`, *optional*, defaults to `True`):
|
||||
Whether to report joint positions in degrees. Keep `True` for compatibility with existing
|
||||
policies and datasets.
|
||||
num_read_retries (`int`, *optional*, defaults to 2):
|
||||
Extra attempts when a `sync_read` fails. Feetech buses occasionally return a corrupted status
|
||||
packet, especially when several joints move at once, which would otherwise abort the
|
||||
teleoperation loop. Retries are immediate and only happen on failure, so steady-state read cost
|
||||
is unchanged.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular arm; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.so_leader import SO101Leader, SO101LeaderConfig
|
||||
>>> config = SO101LeaderConfig(port="/dev/ttyACM0") # doctest: +SKIP
|
||||
>>> teleop = SO101Leader(config) # doctest: +SKIP
|
||||
```
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -31,34 +31,12 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SOLeader(Teleoperator):
|
||||
"""The SO-family leader arm: a 5-DOF arm plus gripper on a Feetech bus, held to teleoperate a follower arm.
|
||||
|
||||
`SO100Leader` and `SO101Leader` are aliases of this class. The two arms differ in calibration and
|
||||
gearing, not control code, so both are driven through the same implementation with a different
|
||||
`config_class` and `name`.
|
||||
|
||||
Actions are keyed `"<motor>.pos"`. See [`~teleoperators.Teleoperator`] for the contract every method
|
||||
here implements.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.so_leader import SO101Leader, SO101LeaderConfig
|
||||
>>> teleop = SO101Leader(SO101LeaderConfig(port="/dev/ttyACM0")) # doctest: +SKIP
|
||||
>>> with teleop: # doctest: +SKIP
|
||||
... action = teleop.get_action()
|
||||
```
|
||||
"""
|
||||
"""Generic SO leader base for SO-100/101/10X teleoperators."""
|
||||
|
||||
config_class = SOLeaderTeleopConfig
|
||||
name = "so_leader"
|
||||
|
||||
def __init__(self, config: SOLeaderTeleopConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`SOLeaderTeleopConfig`):
|
||||
The teleoperator's configuration. Its `port` determines what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
norm_mode_body = MotorNormMode.DEGREES if config.use_degrees else MotorNormMode.RANGE_M100_100
|
||||
@@ -77,42 +55,18 @@ class SOLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The arm's joint positions.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"<motor>.pos"` keys mapped to `float`.
|
||||
"""
|
||||
return {f"{motor}.pos": float for motor in self.bus.motors}
|
||||
|
||||
@property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""The arm's target joint positions, used to sync this leader arm to another pose.
|
||||
|
||||
Shares the same keys as [`~teleoperators.Teleoperator.action_features`], since feedback for this
|
||||
arm is a goal position written to each motor.
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: `"<motor>.pos"` keys mapped to `float`.
|
||||
"""
|
||||
return self.action_features
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`]."""
|
||||
return self.bus.is_connected
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Connect the motor bus, calibrating and configuring the arm.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to run calibration when the motors disagree with the calibration file, or no file
|
||||
exists yet. Calibration is interactive and prompts on stdin.
|
||||
|
||||
Raises:
|
||||
DeviceAlreadyConnectedError: If the teleoperator is already connected.
|
||||
"""
|
||||
self.bus.connect()
|
||||
if not self.is_calibrated and calibrate:
|
||||
logger.info(
|
||||
@@ -125,15 +79,9 @@ class SOLeader(Teleoperator):
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_calibrated`]."""
|
||||
return self.bus.is_calibrated
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Calibrate the arm, writing the result to the motors and the calibration file.
|
||||
|
||||
This is interactive: it prompts on stdin to reuse an existing calibration file, and otherwise asks
|
||||
you to move the arm to its middle position and then through each joint's full range.
|
||||
"""
|
||||
if self.calibration:
|
||||
# Calibration file exists, ask user whether to use it or run new calibration
|
||||
user_input = input(
|
||||
@@ -177,34 +125,18 @@ class SOLeader(Teleoperator):
|
||||
print(f"Calibration saved to {self.calibration_fpath}")
|
||||
|
||||
def configure(self) -> None:
|
||||
"""Disable torque and write the position-mode operating mode to every motor.
|
||||
|
||||
Torque is left disabled so the arm can be moved freely by hand while teleoperating.
|
||||
"""
|
||||
self.bus.disable_torque()
|
||||
self.bus.configure_motors()
|
||||
for motor in self.bus.motors:
|
||||
self.bus.write("Operating_Mode", motor, OperatingMode.POSITION.value)
|
||||
|
||||
def enable_torque(self) -> None:
|
||||
"""Enable torque on every motor.
|
||||
|
||||
Useful to briefly drive the arm to a position (e.g. via
|
||||
[`~teleoperators.so_leader.SOLeader.send_feedback`]) before releasing it back to free movement with
|
||||
[`~teleoperators.so_leader.SOLeader.disable_torque`].
|
||||
"""
|
||||
self.bus.enable_torque()
|
||||
|
||||
def disable_torque(self) -> None:
|
||||
"""Disable torque on every motor, letting the arm be moved freely by hand."""
|
||||
self.bus.disable_torque()
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one at a time.
|
||||
|
||||
Run this once when building an arm. It is interactive: it prompts you to connect the controller
|
||||
board to a single motor at a time, working from the gripper back to the base.
|
||||
"""
|
||||
for motor in reversed(self.bus.motors):
|
||||
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
||||
self.bus.setup_motor(motor)
|
||||
@@ -212,14 +144,6 @@ class SOLeader(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Same as [`~teleoperators.Teleoperator.get_action`].
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: `"<motor>.pos"` keys mapped to the arm's current joint positions.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
action = self.bus.sync_read("Present_Position", num_retry=self.config.num_read_retries)
|
||||
action = {f"{motor}.pos": val for motor, val in action.items()}
|
||||
@@ -229,29 +153,12 @@ class SOLeader(Teleoperator):
|
||||
|
||||
@check_if_not_connected
|
||||
def send_feedback(self, feedback: dict[str, float]) -> None:
|
||||
"""Write goal positions to the arm's motors, e.g. to sync it to a follower's current pose.
|
||||
|
||||
Torque must be enabled (see [`~teleoperators.so_leader.SOLeader.enable_torque`]) for the arm to
|
||||
actually move to the written positions.
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, float]`):
|
||||
`"<motor>.pos"` keys mapped to target positions. Keys not ending in `.pos` are ignored.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
goals = {k.removesuffix(".pos"): v for k, v in feedback.items() if k.endswith(".pos")}
|
||||
if goals:
|
||||
self.bus.sync_write("Goal_Position", goals)
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""Same as [`~teleoperators.Teleoperator.disconnect`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the teleoperator is not connected.
|
||||
"""
|
||||
self.bus.disconnect()
|
||||
logger.info(f"{self} disconnected.")
|
||||
|
||||
|
||||
@@ -21,15 +21,7 @@ from ..config import TeleoperatorConfig
|
||||
|
||||
@dataclass
|
||||
class ExoskeletonArmPortConfig:
|
||||
"""Serial port configuration for one exoskeleton arm.
|
||||
|
||||
Args:
|
||||
port (`str`, *optional*, defaults to `""`):
|
||||
Serial port the exoskeleton arm's sensor board is connected to, e.g. `/dev/ttyUSB0`. An empty
|
||||
string disables exoskeleton control for that arm.
|
||||
baud_rate (`int`, *optional*, defaults to 115200):
|
||||
Baud rate for the serial connection.
|
||||
"""
|
||||
"""Serial port configuration for individual exoskeleton arm."""
|
||||
|
||||
port: str = ""
|
||||
baud_rate: int = 115200
|
||||
@@ -38,26 +30,6 @@ class ExoskeletonArmPortConfig:
|
||||
@TeleoperatorConfig.register_subclass("unitree_g1")
|
||||
@dataclass
|
||||
class UnitreeG1TeleoperatorConfig(TeleoperatorConfig):
|
||||
"""Configuration for the Unitree G1 bimanual exoskeleton teleoperator.
|
||||
|
||||
Args:
|
||||
left_arm_config (`ExoskeletonArmPortConfig`, *optional*):
|
||||
Serial port settings for the left exoskeleton arm. Leave `port` empty to run without exoskeleton
|
||||
control on this side.
|
||||
right_arm_config (`ExoskeletonArmPortConfig`, *optional*):
|
||||
Serial port settings for the right exoskeleton arm. Leave `port` empty to run without
|
||||
exoskeleton control on this side.
|
||||
frozen_joints (`str`, *optional*, defaults to `""`):
|
||||
Comma-separated G1 arm joint names to exclude from the exoskeleton-driven inverse kinematics.
|
||||
These joints are held at their neutral pose instead of being tracked.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several teleoperators of the same
|
||||
type. It also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-teleoperator directory under
|
||||
the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
left_arm_config: ExoskeletonArmPortConfig = field(default_factory=ExoskeletonArmPortConfig)
|
||||
right_arm_config: ExoskeletonArmPortConfig = field(default_factory=ExoskeletonArmPortConfig)
|
||||
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""This module handles calibration of hall effect sensors used in the exoskeleton.
|
||||
|
||||
"""
|
||||
This module handles calibration of hall effect sensors used in the exoskeleton.
|
||||
Each joint has a pair of ADC channels outputting sin and cos values that trace an ellipse
|
||||
as the joint rotates due to imprecision in magnet/sensor placement. We fit this ellipse to a unit circle,
|
||||
and calculate arctan2 of the unit circle to get the joint angle.
|
||||
@@ -59,21 +59,6 @@ JOINTS = {
|
||||
|
||||
@dataclass
|
||||
class ExoskeletonJointCalibration:
|
||||
"""Per-joint calibration mapping raw sin/cos ADC pairs to an angle in radians.
|
||||
|
||||
Args:
|
||||
name (`str`):
|
||||
Joint name, matching a key in `JOINTS`.
|
||||
center_fit (`list[float]`):
|
||||
The `[x, y]` center of the ellipse fitted to this joint's raw sin/cos ADC readings.
|
||||
T (`list[list[float]]`):
|
||||
2x2 transformation matrix mapping a centered raw reading onto the unit circle, correcting for
|
||||
the fitted ellipse's scale and rotation.
|
||||
zero_offset (`float`, *optional*, defaults to 0.0):
|
||||
Angle, in radians, measured while the joint was held at its neutral pose. Subtracted from the
|
||||
raw angle so the neutral pose reads as zero.
|
||||
"""
|
||||
|
||||
name: str # joint name
|
||||
center_fit: list[float] # center of the ellipse
|
||||
T: list[list[float]] # 2x2 transformation matrix
|
||||
@@ -90,11 +75,6 @@ class ExoskeletonCalibration:
|
||||
joints: list[ExoskeletonJointCalibration] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Serialize this calibration to a plain dict suitable for JSON storage.
|
||||
|
||||
Returns:
|
||||
`dict`: The calibration with nested joint calibrations flattened to plain dicts.
|
||||
"""
|
||||
return {
|
||||
"version": self.version,
|
||||
"side": self.side,
|
||||
@@ -112,15 +92,6 @@ class ExoskeletonCalibration:
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> ExoskeletonCalibration:
|
||||
"""Reconstruct a calibration from the dict produced by `to_dict`.
|
||||
|
||||
Args:
|
||||
data (`dict`):
|
||||
Parsed JSON calibration data. Missing optional keys fall back to their defaults.
|
||||
|
||||
Returns:
|
||||
`ExoskeletonCalibration`: The reconstructed calibration.
|
||||
"""
|
||||
joints = [
|
||||
ExoskeletonJointCalibration(
|
||||
name=j["name"],
|
||||
@@ -140,32 +111,6 @@ class ExoskeletonCalibration:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CalibParams:
|
||||
"""Tuning knobs for the interactive ellipse-fitting calibration UI.
|
||||
|
||||
Args:
|
||||
fit_every (`float`, *optional*, defaults to 0.15):
|
||||
Minimum time, in seconds, between successive ellipse re-fits while mapping a joint's range.
|
||||
min_fit_points (`int`, *optional*, defaults to 60):
|
||||
Minimum number of buffered samples required before attempting an ellipse fit.
|
||||
fit_window (`int`, *optional*, defaults to 900):
|
||||
Number of most recent raw samples considered for each ellipse fit.
|
||||
max_fit_points (`int`, *optional*, defaults to 300):
|
||||
Maximum number of points passed to the ellipse fitter; the fit window is downsampled evenly
|
||||
above this count.
|
||||
trim_low (`float`, *optional*, defaults to 0.05):
|
||||
Lower radius quantile below which points are treated as outliers and discarded before fitting.
|
||||
trim_high (`float`, *optional*, defaults to 0.95):
|
||||
Upper radius quantile above which points are treated as outliers and discarded before fitting.
|
||||
median_window (`int`, *optional*, defaults to 5):
|
||||
Number of raw samples averaged (median) to smooth each sin/cos reading before it is buffered.
|
||||
history (`int`, *optional*, defaults to 3500):
|
||||
Maximum number of samples retained per plot, for visualization only.
|
||||
draw_hz (`float`, *optional*, defaults to 120.0):
|
||||
Maximum refresh rate of the calibration plot.
|
||||
sample_count (`int`, *optional*, defaults to 50):
|
||||
Number of samples averaged to compute a joint's zero-pose offset.
|
||||
"""
|
||||
|
||||
fit_every: float = 0.15
|
||||
min_fit_points: int = 60
|
||||
fit_window: int = 900
|
||||
@@ -184,7 +129,9 @@ def normalize_angle(angle: float) -> float:
|
||||
|
||||
|
||||
def joint_z_and_angle(raw16: list[int], j: ExoskeletonJointCalibration) -> tuple[np.ndarray, float]:
|
||||
"""Applies calibration to each joint: raw → centered → ellipse-to-circle → angle."""
|
||||
"""
|
||||
Applies calibration to each joint: raw → centered → ellipse-to-circle → angle.
|
||||
"""
|
||||
pair = JOINTS[j.name]
|
||||
s, c = raw16[pair[0]], raw16[pair[1]] # get sin and cos
|
||||
p = np.array([float(c) - ADC_HALF, float(s) - ADC_HALF]) # center the raw values
|
||||
@@ -206,7 +153,9 @@ def run_exo_calibration(
|
||||
save_path: Path,
|
||||
params: CalibParams | None = None,
|
||||
) -> ExoskeletonCalibration:
|
||||
"""Run interactive calibration for an exoskeleton arm."""
|
||||
"""
|
||||
Run interactive calibration for an exoskeleton arm.
|
||||
"""
|
||||
require_package("pyserial", extra="unitree_g1", import_name="serial")
|
||||
try:
|
||||
import cv2
|
||||
@@ -224,11 +173,9 @@ def run_exo_calibration(
|
||||
logger.info(f"Starting calibration for {side} exoskeleton arm")
|
||||
|
||||
def running_median(win: deque) -> float:
|
||||
"""Return the median of a buffered window of raw ADC samples, used to smooth sensor noise."""
|
||||
return float(np.median(np.fromiter(win, dtype=float)))
|
||||
|
||||
def read_joint_point(raw16: list[int], pair: tuple[int, int]):
|
||||
"""Extract one joint's centered (x, y) sin/cos point, plus its raw sin/cos values."""
|
||||
s, c = raw16[pair[0]], raw16[pair[1]]
|
||||
return float(c) - ADC_HALF, float(s) - ADC_HALF, float(s), float(c)
|
||||
|
||||
@@ -312,7 +259,6 @@ def run_exo_calibration(
|
||||
zero_samples = []
|
||||
|
||||
def on_key(event):
|
||||
"""Matplotlib key-press handler that requests advancing to the calibration's next phase."""
|
||||
nonlocal advance_requested
|
||||
if event.key in ("n", "N", "enter", " "):
|
||||
advance_requested = True
|
||||
@@ -320,7 +266,6 @@ def run_exo_calibration(
|
||||
fig.canvas.mpl_connect("key_press_event", on_key)
|
||||
|
||||
def reset_state():
|
||||
"""Build a fresh mutable state dict for tracking one joint's in-progress ellipse fit."""
|
||||
return {
|
||||
"xs": deque(maxlen=params.history),
|
||||
"ys": deque(maxlen=params.history),
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""IK helper for exoskeleton-to-G1 teleoperation.
|
||||
|
||||
We map Exoskeleton joint angles to end-effector pose in world frame, visualizing the result in meshcat
|
||||
after calibration.
|
||||
"""
|
||||
IK helper for exoskeleton-to-G1 teleoperation. We map Exoskeleton joint angles to end-effector pose in world frame,
|
||||
visualizing the result in meshcat after calibration.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -44,24 +43,6 @@ def _frame_id(model, name: str) -> int | None:
|
||||
|
||||
@dataclass
|
||||
class ArmCfg:
|
||||
"""Static per-arm configuration linking an exoskeleton URDF to its G1 counterpart.
|
||||
|
||||
Args:
|
||||
side (`str`):
|
||||
Which arm this describes, `"left"` or `"right"`.
|
||||
urdf (`str`):
|
||||
Path to the exoskeleton arm's URDF file.
|
||||
root (`str`):
|
||||
Name of the exoskeleton's root node in the meshcat scene tree.
|
||||
g1_ee (`str`):
|
||||
Name of the corresponding end-effector frame on the G1 URDF model.
|
||||
offset (`np.ndarray`):
|
||||
World-frame translation applied to the exoskeleton and its IK target, so the exoskeleton's
|
||||
visualization does not overlap the G1's.
|
||||
marker_prefix (`str`):
|
||||
Prefix used to namespace this arm's meshcat marker paths.
|
||||
"""
|
||||
|
||||
side: str # "left" | "right"
|
||||
urdf: str # exo_left.urdf / exo_right.urdf
|
||||
root: str # "exo_left" / "exo_right"
|
||||
@@ -71,28 +52,12 @@ class ArmCfg:
|
||||
|
||||
|
||||
class Markers:
|
||||
"""Creates meshcat visualization primitives, showing end-effector frames of exoskeleton and G1."""
|
||||
"""Creates meshcat visualization primitives, showing end-effector frames of exoskeleton and G1"""
|
||||
|
||||
def __init__(self, viewer):
|
||||
"""Store the meshcat viewer (or scene-tree node) markers will be attached under.
|
||||
|
||||
Args:
|
||||
viewer:
|
||||
The meshcat viewer markers are added to.
|
||||
"""
|
||||
self.v = viewer
|
||||
|
||||
def sphere(self, path: str, r: float, rgba: tuple[float, float, float, float]):
|
||||
"""Add a colored sphere marker to the meshcat scene.
|
||||
|
||||
Args:
|
||||
path (`str`):
|
||||
Meshcat scene-tree path for this marker, e.g. `"markers/left_exo_ee"`.
|
||||
r (`float`):
|
||||
Sphere radius, in meters.
|
||||
rgba (`tuple[float, float, float, float]`):
|
||||
Red, green, and blue components (each 0-1) followed by opacity (0-1).
|
||||
"""
|
||||
import meshcat.geometry as mg
|
||||
|
||||
c = (int(rgba[0] * 255) << 16) | (int(rgba[1] * 255) << 8) | int(rgba[2] * 255)
|
||||
@@ -102,16 +67,6 @@ class Markers:
|
||||
)
|
||||
|
||||
def axes(self, path: str, axis_len: float = 0.1, axis_w: int = 6):
|
||||
"""Add a red/green/blue XYZ axis-triad marker to the meshcat scene.
|
||||
|
||||
Args:
|
||||
path (`str`):
|
||||
Meshcat scene-tree path for this marker.
|
||||
axis_len (`float`, *optional*, defaults to 0.1):
|
||||
Length of each axis line, in meters.
|
||||
axis_w (`int`, *optional*, defaults to 6):
|
||||
Line width, in pixels.
|
||||
"""
|
||||
import meshcat.geometry as mg
|
||||
|
||||
pts = np.array(
|
||||
@@ -130,37 +85,21 @@ class Markers:
|
||||
)
|
||||
|
||||
def tf(self, path: str, mat: np.ndarray):
|
||||
"""Update the transform of an existing marker.
|
||||
|
||||
Args:
|
||||
path (`str`):
|
||||
Meshcat scene-tree path of the marker to move.
|
||||
mat (`np.ndarray`):
|
||||
New 4x4 homogeneous transform for the marker, in world frame.
|
||||
"""
|
||||
self.v[path].set_transform(mat)
|
||||
|
||||
|
||||
class ExoskeletonIKHelper:
|
||||
"""Maps exoskeleton joint angles to G1 arm joint angles via forward and inverse kinematics.
|
||||
|
||||
Loads the G1 robot and both exoskeleton arm URDF models via Pinocchio, computes forward kinematics on
|
||||
the exoskeleton to obtain end-effector poses in the world frame, then solves inverse kinematics on the
|
||||
G1 model to find joint angles reproducing those poses. Also provides an optional meshcat
|
||||
visualization showing both robots alongside their IK targets.
|
||||
"""
|
||||
- Loads G1 robot and exoskeleton URDF models via Pinocchio
|
||||
- Computes forward kinematics on exoskeleton to get end-effector poses
|
||||
- Solves inverse kinematics on G1 to match those poses
|
||||
- Provides meshcat visualization showing both robots and targets
|
||||
|
||||
Args:
|
||||
frozen_joints (`list[str] | None`, *optional*):
|
||||
G1 joint names to exclude from IK; these are held at their current pose instead of being
|
||||
solved for.
|
||||
frozen_joints: List of G1 joint names to exclude from IK (kept at neutral).
|
||||
"""
|
||||
|
||||
def __init__(self, frozen_joints: list[str] | None = None):
|
||||
"""Load the G1 and exoskeleton Pinocchio models and precompute frozen-joint indices.
|
||||
|
||||
Raises:
|
||||
ImportError: If `pinocchio` is not installed.
|
||||
"""
|
||||
try:
|
||||
import pinocchio as pin
|
||||
except ImportError as e:
|
||||
@@ -249,9 +188,9 @@ class ExoskeletonIKHelper:
|
||||
logger.info(f"loaded {a.side} exo urdf: {a.urdf}")
|
||||
|
||||
def init_visualization(self):
|
||||
"""Creates a browser-based visualization of exoskeleton and G1 robot.
|
||||
|
||||
Highlights end-effector frames and target positions.
|
||||
"""
|
||||
Creates a browser-based visualization of exoskeleton and G1 robot,
|
||||
highlighting end-effector frames and target positions.
|
||||
"""
|
||||
try:
|
||||
from pinocchio.visualize import MeshcatVisualizer
|
||||
@@ -298,7 +237,7 @@ class ExoskeletonIKHelper:
|
||||
print(f"\nmeshcat url: {self.viewer.url()}\n")
|
||||
|
||||
def _fk_target_world(self, side: str, angles: dict[str, float]) -> np.ndarray | None:
|
||||
"""Returns wrist frame target to be used for G1 IK in 4x4 homogeneous transform. Takes offset into account."""
|
||||
"""returns wrist frame target to be used for G1 IK in 4x4 homogeneous transform. Takes offset into account."""
|
||||
if side not in self.exo or not angles:
|
||||
return None
|
||||
|
||||
@@ -324,10 +263,6 @@ class ExoskeletonIKHelper:
|
||||
return target
|
||||
|
||||
def update_visualization(self):
|
||||
"""Refresh the meshcat scene with the G1's and both exoskeletons' current poses and IK targets.
|
||||
|
||||
No-op if `init_visualization` has not been called yet.
|
||||
"""
|
||||
if self.viewer is None or self.markers is None:
|
||||
return
|
||||
|
||||
@@ -376,9 +311,9 @@ class ExoskeletonIKHelper:
|
||||
left_angles: dict[str, float],
|
||||
right_angles: dict[str, float],
|
||||
) -> dict[str, float]:
|
||||
"""Performs FK on exoskeleton to get end-effector poses in world frame.
|
||||
|
||||
Solves IK on G1 to return joint angles matching those poses in G1 motor order.
|
||||
"""
|
||||
Performs FK on exoskeleton to get end-effector poses in world frame,
|
||||
after which it solves IK on G1 to return joint angles matching those poses in G1 motor order.
|
||||
"""
|
||||
pin = self.pin
|
||||
|
||||
|
||||
@@ -35,17 +35,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_raw16(line: bytes) -> list[int] | None:
|
||||
"""Parse one line of exoskeleton telemetry into 16 raw ADC channel readings.
|
||||
|
||||
Args:
|
||||
line (`bytes`):
|
||||
One raw line read from the exoskeleton's serial port, expected to contain 16
|
||||
whitespace-separated integers (sin/cos pairs for each sensed joint, plus joystick channels).
|
||||
|
||||
Returns:
|
||||
`list[int] | None`: The 16 raw ADC values in channel order, or `None` if the line is malformed or
|
||||
has fewer than 16 values.
|
||||
"""
|
||||
try:
|
||||
parts = line.decode("utf-8", errors="ignore").split()
|
||||
if len(parts) < 16:
|
||||
@@ -56,18 +45,7 @@ def parse_raw16(line: bytes) -> list[int] | None:
|
||||
|
||||
|
||||
def read_raw_from_serial(ser) -> list[int] | None:
|
||||
"""Read the latest sample from serial; if the input buffer is backed up, keep only the newest.
|
||||
|
||||
Draining the buffer down to the newest line keeps teleoperation responsive to the exoskeleton's
|
||||
current pose instead of replaying a queue of stale samples.
|
||||
|
||||
Args:
|
||||
ser (`serial.Serial`):
|
||||
Open serial connection to the exoskeleton's sensor board.
|
||||
|
||||
Returns:
|
||||
`list[int] | None`: The most recently parsed sample, or `None` if no valid line was available.
|
||||
"""
|
||||
"""Read latest sample from serial; if buffer is backed up, keep only the newest."""
|
||||
try:
|
||||
last = None
|
||||
while ser.in_waiting > 0:
|
||||
@@ -89,27 +67,6 @@ def read_raw_from_serial(ser) -> list[int] | None:
|
||||
|
||||
@dataclass
|
||||
class ExoskeletonArm:
|
||||
"""Serial link and calibration state for one exoskeleton arm (left or right).
|
||||
|
||||
Wraps the raw serial connection to the arm's sensor board and converts its hall-effect sensor readings
|
||||
into calibrated joint angles via `get_angles`, once a calibration has been loaded or produced by
|
||||
`calibrate`.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm's sensor board is connected to, e.g. `/dev/ttyUSB0`.
|
||||
calibration_fpath (`Path`):
|
||||
Path to the JSON file used to load and save this arm's calibration.
|
||||
side (`str`):
|
||||
Which arm this is, `"left"` or `"right"`. Used to label saved calibration data and log
|
||||
messages.
|
||||
baud_rate (`int`, *optional*, defaults to 115200):
|
||||
Baud rate for the serial connection.
|
||||
calibration (`ExoskeletonCalibration | None`, *optional*):
|
||||
Calibration data for this arm. Loaded automatically from `calibration_fpath` if that file
|
||||
exists; otherwise populated by calling `calibrate`.
|
||||
"""
|
||||
|
||||
port: str
|
||||
calibration_fpath: Path
|
||||
side: str
|
||||
@@ -119,39 +76,19 @@ class ExoskeletonArm:
|
||||
calibration: ExoskeletonCalibration | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Check that `pyserial` is installed and load an existing calibration file, if any."""
|
||||
require_package("pyserial", extra="unitree_g1", import_name="serial")
|
||||
if self.calibration_fpath.is_file():
|
||||
self._load_calibration()
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Whether the serial connection to the arm's sensor board is open.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if the serial port has been opened and not yet closed.
|
||||
"""
|
||||
return self._ser is not None and getattr(self._ser, "is_open", False)
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Whether calibration data is available for this arm.
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if a calibration has been loaded from disk or produced by `calibrate`.
|
||||
"""
|
||||
return self.calibration is not None
|
||||
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Open the serial connection to the arm's sensor board.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to run `calibrate` automatically after connecting if no calibration is loaded yet.
|
||||
|
||||
Raises:
|
||||
ConnectionError: If the serial port cannot be opened.
|
||||
"""
|
||||
if self.is_connected:
|
||||
return
|
||||
try:
|
||||
@@ -165,7 +102,6 @@ class ExoskeletonArm:
|
||||
self.calibrate()
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Close the serial connection to the arm's sensor board, if open."""
|
||||
if self._ser:
|
||||
try:
|
||||
self._ser.close()
|
||||
@@ -181,41 +117,17 @@ class ExoskeletonArm:
|
||||
logger.warning(f"failed to load calibration: {e}")
|
||||
|
||||
def read_raw(self) -> list[int] | None:
|
||||
"""Read the arm's latest raw ADC sample.
|
||||
|
||||
Returns:
|
||||
`list[int] | None`: The 16 raw ADC channel values, or `None` if the arm is not connected or no
|
||||
valid sample was available.
|
||||
"""
|
||||
if not self._ser:
|
||||
return None
|
||||
return read_raw_from_serial(self._ser)
|
||||
|
||||
def get_angles(self) -> dict[str, float]:
|
||||
"""Read the arm's current sensor sample and convert it to calibrated joint angles.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: Joint name to angle in radians, or an empty dict if no sample was
|
||||
available on the serial link.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the arm has not been calibrated yet.
|
||||
"""
|
||||
if not self.calibration:
|
||||
raise RuntimeError("exoskeleton not calibrated")
|
||||
raw = self.read_raw()
|
||||
return {} if raw is None else exo_raw_to_angles(raw, self.calibration)
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Run the interactive per-joint calibration procedure and store its result.
|
||||
|
||||
Delegates to `run_exo_calibration`, which walks the operator through moving each joint through
|
||||
its range and holding a zero pose, then saves the resulting ellipse fits and zero offsets to
|
||||
`calibration_fpath`.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the arm is not connected.
|
||||
"""
|
||||
if not self.is_connected:
|
||||
raise RuntimeError("Cannot calibrate: exoskeleton not connected")
|
||||
self.calibration = run_exo_calibration(self._ser, self.side, self.calibration_fpath)
|
||||
|
||||
@@ -28,18 +28,7 @@ if TYPE_CHECKING or _unitree_sdk_available:
|
||||
else:
|
||||
|
||||
class Joystick:
|
||||
"""Placeholder used when `unitree_sdk2py` is not installed.
|
||||
|
||||
Raises `ImportError` on instantiation instead of on import, so the module can still be imported
|
||||
(and its non-hardware members inspected) without the SDK present.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Raise `ImportError` because `unitree_sdk2py` is required and not installed.
|
||||
|
||||
Raises:
|
||||
ImportError: Always.
|
||||
"""
|
||||
raise ImportError(
|
||||
"unitree_sdk2py is required for RemoteController. Install with: pip install unitree_sdk2py"
|
||||
)
|
||||
@@ -85,7 +74,6 @@ class RemoteController:
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize joystick axes, button state, and joystick-center calibration to their defaults."""
|
||||
self.lx = 0.0
|
||||
self.ly = 0.0
|
||||
self.rx = 0.0
|
||||
@@ -114,19 +102,6 @@ class RemoteController:
|
||||
self.remote_action.update(zip(REMOTE_AXES, (self.lx, self.ly, self.rx, self.ry), strict=True))
|
||||
|
||||
def calibrate_center(self, raw16: list[int] | None, side: str) -> None:
|
||||
"""Detect and record the center position of one side's exoskeleton-mounted joystick.
|
||||
|
||||
Meant to be called once at connect time. If the joystick's button ADC channel reads above
|
||||
half-scale, an exoskeleton joystick is assumed present on that side, and its current X/Y ADC
|
||||
reading is stored as the neutral center used by `set_from_exo`.
|
||||
|
||||
Args:
|
||||
raw16 (`list[int] | None`):
|
||||
The 16 raw ADC channel values read from the exoskeleton's sensor board, or `None` if no
|
||||
sample was available.
|
||||
side (`str`):
|
||||
Which joystick to calibrate, `"left"` or `"right"`.
|
||||
"""
|
||||
if raw16 is None or len(raw16) < 16:
|
||||
logger.info(f"{side.capitalize()} exo joystick: no data available")
|
||||
return
|
||||
@@ -148,17 +123,6 @@ class RemoteController:
|
||||
logger.info(f"{side.capitalize()} exo joystick enabled, center: x={x}, y={y}")
|
||||
|
||||
def set_from_exo(self, raw16: list[int] | None, side: str) -> None:
|
||||
"""Update one side's joystick axes and button from the exoskeleton-mounted joystick, if calibrated.
|
||||
|
||||
No-op if `calibrate_center` did not detect an exoskeleton joystick on that side.
|
||||
|
||||
Args:
|
||||
raw16 (`list[int] | None`):
|
||||
The 16 raw ADC channel values read from the exoskeleton's sensor board, or `None` if no
|
||||
sample was available.
|
||||
side (`str`):
|
||||
Which joystick to update, `"left"` or `"right"`.
|
||||
"""
|
||||
if raw16 is None or len(raw16) < 16:
|
||||
return
|
||||
|
||||
@@ -193,39 +157,17 @@ class RemoteController:
|
||||
|
||||
|
||||
class UnitreeG1Teleoperator(Teleoperator):
|
||||
"""Bimanual exoskeleton-arm teleoperator for the Unitree G1 humanoid, plus its wireless remote.
|
||||
"""
|
||||
Bimanual exoskeleton arms teleoperator for Unitree G1 arms.
|
||||
|
||||
Two exoskeleton arms worn by the operator report joint angles, which are converted to a G1 arm action
|
||||
via forward kinematics on the exoskeleton followed by inverse kinematics on the G1 (see
|
||||
[`~teleoperators.unitree_g1.exo_ik.ExoskeletonIKHelper`]). A Unitree wireless remote (or an
|
||||
exoskeleton-mounted joystick, when the remote is idle) supplies additional axes, typically used for
|
||||
locomotion. If neither exoskeleton arm has a configured serial port, the teleoperator falls back to
|
||||
remote-controller-only mode and reports no arm joint actions.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.teleoperators.unitree_g1 import UnitreeG1Teleoperator, UnitreeG1TeleoperatorConfig
|
||||
>>> teleop = UnitreeG1Teleoperator(UnitreeG1TeleoperatorConfig()) # doctest: +SKIP
|
||||
>>> with teleop: # doctest: +SKIP
|
||||
... action = teleop.get_action()
|
||||
```
|
||||
Uses inverse kinematics: exoskeleton FK computes end-effector pose,
|
||||
G1 IK solves for joint angles.
|
||||
"""
|
||||
|
||||
config_class = UnitreeG1TeleoperatorConfig
|
||||
name = "unitree_g1"
|
||||
|
||||
def __init__(self, config: UnitreeG1TeleoperatorConfig):
|
||||
"""Build the teleoperator from its configuration.
|
||||
|
||||
Args:
|
||||
config (`UnitreeG1TeleoperatorConfig`):
|
||||
The teleoperator's configuration. Exoskeleton arm control is enabled only if both
|
||||
`left_arm_config.port` and `right_arm_config.port` are set; leaving both empty runs in
|
||||
remote-controller-only mode.
|
||||
|
||||
Raises:
|
||||
ValueError: If exactly one of the two arm ports is configured.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
left_exo_enabled = bool(config.left_arm_config.port.strip())
|
||||
@@ -266,15 +208,6 @@ class UnitreeG1Teleoperator(Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""Keys the teleoperator's actions are reported under.
|
||||
|
||||
Includes one `"<joint>.q"` key per G1 arm joint (radians) when both exoskeleton arms are
|
||||
configured, plus the remote controller's stick and button axes. See
|
||||
[`~teleoperators.Teleoperator.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: Action names mapped to `float`.
|
||||
"""
|
||||
remote_features = dict.fromkeys(self.remote_controller.remote_action, float)
|
||||
if not self._arm_control_enabled:
|
||||
return remote_features
|
||||
@@ -283,48 +216,21 @@ class UnitreeG1Teleoperator(Teleoperator):
|
||||
|
||||
@cached_property
|
||||
def feedback_features(self) -> dict[str, type]:
|
||||
"""Same as [`~teleoperators.Teleoperator.feedback_features`].
|
||||
|
||||
Returns:
|
||||
`dict[str, type]`: A single `"wireless_remote"` key mapped to `bytes`, the raw Unitree
|
||||
wireless remote packet to be parsed into joystick and button state.
|
||||
"""
|
||||
return {"wireless_remote": bytes}
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_connected`].
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if exoskeleton arm control is disabled (remote-only mode), or if both
|
||||
exoskeleton arms are connected.
|
||||
"""
|
||||
if not self._arm_control_enabled:
|
||||
return True
|
||||
return self.left_arm.is_connected and self.right_arm.is_connected
|
||||
|
||||
@property
|
||||
def is_calibrated(self) -> bool:
|
||||
"""Same as [`~teleoperators.Teleoperator.is_calibrated`].
|
||||
|
||||
Returns:
|
||||
`bool`: `True` if exoskeleton arm control is disabled (remote-only mode), or if both
|
||||
exoskeleton arms are calibrated.
|
||||
"""
|
||||
if not self._arm_control_enabled:
|
||||
return True
|
||||
return self.left_arm.is_calibrated and self.right_arm.is_calibrated
|
||||
|
||||
def connect(self, calibrate: bool = True) -> None:
|
||||
"""Connect both exoskeleton arms, build the IK helper, and calibrate the remote's joystick centers.
|
||||
|
||||
If neither exoskeleton arm has a configured serial port, this is a no-op and the teleoperator
|
||||
falls back to reporting only remote-controller actions.
|
||||
|
||||
Args:
|
||||
calibrate (`bool`, *optional*, defaults to `True`):
|
||||
Whether to calibrate each exoskeleton arm that is not yet calibrated.
|
||||
"""
|
||||
if not self._arm_control_enabled:
|
||||
logger.warning("Exo ports not fully configured; teleop will send joystick only (no arm actions)")
|
||||
return
|
||||
@@ -344,12 +250,6 @@ class UnitreeG1Teleoperator(Teleoperator):
|
||||
self.remote_controller.calibrate_center(right_raw, "right")
|
||||
|
||||
def calibrate(self) -> None:
|
||||
"""Calibrate each exoskeleton arm that is not already calibrated, then verify tracking visually.
|
||||
|
||||
See [`~teleoperators.Teleoperator.calibrate`]. After both arms are calibrated, this opens the
|
||||
interactive meshcat visualization (see `run_visualization_loop`) so the operator can confirm the
|
||||
G1 arms track the exoskeleton before recording data.
|
||||
"""
|
||||
if not self.left_arm.is_calibrated:
|
||||
logger.info("Starting calibration for left arm...")
|
||||
self.left_arm.calibrate()
|
||||
@@ -366,27 +266,9 @@ class UnitreeG1Teleoperator(Teleoperator):
|
||||
self.run_visualization_loop()
|
||||
|
||||
def configure(self) -> None:
|
||||
"""No-op: the exoskeleton arms require no runtime configuration beyond calibration.
|
||||
|
||||
See [`~teleoperators.Teleoperator.configure`].
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_action(self) -> dict[str, float]:
|
||||
"""Read both exoskeleton arms and the remote controller, and combine them into one action.
|
||||
|
||||
Exoskeleton joint angles are converted to G1 arm joint angles by forward kinematics on the
|
||||
exoskeleton followed by inverse kinematics on the G1, via
|
||||
[`~teleoperators.unitree_g1.exo_ik.ExoskeletonIKHelper.compute_g1_joints_from_exo`]. The wireless
|
||||
remote takes priority over the exoskeleton-mounted joystick for stick/button axes whenever it
|
||||
reports a non-zero stick or a pressed button; otherwise the exoskeleton-mounted joystick (if
|
||||
calibrated) is used instead.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: G1 arm joint angles (`"<joint>.q"`, radians) when exoskeleton control is
|
||||
enabled, merged with the remote controller's stick and button axes. Matches
|
||||
[`~teleoperators.Teleoperator.action_features`].
|
||||
"""
|
||||
joint_action = {}
|
||||
left_raw = None
|
||||
right_raw = None
|
||||
@@ -411,19 +293,11 @@ class UnitreeG1Teleoperator(Teleoperator):
|
||||
return {**joint_action, **rc.remote_action}
|
||||
|
||||
def send_feedback(self, feedback: dict[str, Any]) -> None:
|
||||
"""Update the remote controller's parsed state from a raw wireless remote packet.
|
||||
|
||||
Args:
|
||||
feedback (`dict[str, Any]`):
|
||||
Feedback dict; only the `"wireless_remote"` key (raw bytes) is used, matching
|
||||
[`~teleoperators.Teleoperator.feedback_features`]. Ignored if the key is absent.
|
||||
"""
|
||||
wireless_remote = feedback.get("wireless_remote")
|
||||
if wireless_remote is not None:
|
||||
self.remote_controller.set_from_wireless(wireless_remote)
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Disconnect both exoskeleton arms. See [`~teleoperators.Teleoperator.disconnect`]."""
|
||||
self.left_arm.disconnect()
|
||||
self.right_arm.disconnect()
|
||||
|
||||
|
||||
@@ -34,19 +34,6 @@ class TeleopEvents(Enum):
|
||||
|
||||
|
||||
def make_teleoperator_from_config(config: TeleoperatorConfig) -> "Teleoperator":
|
||||
"""Instantiate the [`~teleoperators.Teleoperator`] matching a config's registered [`~teleoperators.TeleoperatorConfig.type`].
|
||||
|
||||
Args:
|
||||
config (`TeleoperatorConfig`):
|
||||
Configuration of the teleoperator to build.
|
||||
|
||||
Returns:
|
||||
`Teleoperator`: The instantiated teleoperator, not yet connected.
|
||||
|
||||
Raises:
|
||||
ValueError: If the config's type is not a known teleoperator and building it via the generic
|
||||
device factory also fails.
|
||||
"""
|
||||
# TODO(Steven): Consider just using the make_device_from_device_class for all types
|
||||
if config.type == "keyboard":
|
||||
from .keyboard import KeyboardTeleop
|
||||
|
||||
@@ -32,13 +32,16 @@ class RandomSubsetApply(Transform):
|
||||
"""Apply a random subset of N transformations from a list of transformations.
|
||||
|
||||
Args:
|
||||
transforms: list of transformations.
|
||||
p: represents the multinomial probabilities (with no replacement) used for sampling the transform.
|
||||
If the sum of the weights is not 1, they will be normalized. If ``None`` (default), all transforms
|
||||
have the same probability.
|
||||
n_subset: number of transformations to apply. If ``None``, all transforms are applied.
|
||||
Must be in [1, len(transforms)].
|
||||
random_order: apply transformations in a random order.
|
||||
transforms (`Sequence`):
|
||||
List of transformations.
|
||||
p (`list[float] | None`, *optional*):
|
||||
Multinomial probabilities (with no replacement) used for sampling the transform. Normalized if
|
||||
they don't already sum to 1. `None` gives all transforms the same probability.
|
||||
n_subset (`int | None`, *optional*):
|
||||
Number of transformations to apply. Must be in `[1, len(transforms)]`. `None` applies all of
|
||||
them.
|
||||
random_order (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply the sampled transformations in a random order.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -48,6 +51,12 @@ class RandomSubsetApply(Transform):
|
||||
n_subset: int | None = None,
|
||||
random_order: bool = False,
|
||||
) -> None:
|
||||
"""Validate and store the transform pool, sampling weights, and subset size.
|
||||
|
||||
Raises:
|
||||
TypeError: If `transforms` is not a sequence, or `n_subset` is not an int or `None`.
|
||||
ValueError: If `p`'s length doesn't match `transforms`, or `n_subset` is out of range.
|
||||
"""
|
||||
super().__init__()
|
||||
if not isinstance(transforms, Sequence):
|
||||
raise TypeError("Argument transforms should be a sequence of callables")
|
||||
@@ -74,6 +83,7 @@ class RandomSubsetApply(Transform):
|
||||
self.selected_transforms: list[Callable[..., Any]] = []
|
||||
|
||||
def forward(self, *inputs: Any) -> Any:
|
||||
"""Sample a subset of `self.transforms` and apply them in sequence to `inputs`."""
|
||||
needs_unpacking = len(inputs) > 1
|
||||
|
||||
selected_indices = torch.multinomial(torch.tensor(self.p), self.n_subset)
|
||||
@@ -89,6 +99,7 @@ class RandomSubsetApply(Transform):
|
||||
return outputs
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
"""Return the constructor arguments shown in `repr(self)`."""
|
||||
return (
|
||||
f"transforms={self.transforms}, "
|
||||
f"p={self.p}, "
|
||||
@@ -108,16 +119,18 @@ class SharpnessJitter(Transform):
|
||||
A sharpness_factor of 0 gives a blurred image, 1 gives the original image while 2 increases the sharpness
|
||||
by a factor of 2.
|
||||
|
||||
If the input is a :class:`torch.Tensor`,
|
||||
it is expected to have [..., 1 or 3, H, W] shape, where ... means an arbitrary number of leading dimensions.
|
||||
If the input is a `torch.Tensor`, it is expected to have `[..., 1 or 3, H, W]` shape, where `...` means
|
||||
an arbitrary number of leading dimensions.
|
||||
|
||||
Args:
|
||||
sharpness: How much to jitter sharpness. sharpness_factor is chosen uniformly from
|
||||
[max(0, 1 - sharpness), 1 + sharpness] or the given
|
||||
[min, max]. Should be non negative numbers.
|
||||
sharpness (`float | collections.abc.Sequence[float]`):
|
||||
How much to jitter sharpness. `sharpness_factor` is chosen uniformly from
|
||||
`[max(0, 1 - sharpness), 1 + sharpness]`, or the given `[min, max]`. Values must be
|
||||
non-negative.
|
||||
"""
|
||||
|
||||
def __init__(self, sharpness: float | Sequence[float]) -> None:
|
||||
"""Normalize `sharpness` into a `(min, max)` range to sample from on each call."""
|
||||
super().__init__()
|
||||
self.sharpness = self._check_input(sharpness)
|
||||
|
||||
@@ -138,10 +151,12 @@ class SharpnessJitter(Transform):
|
||||
return float(sharpness[0]), float(sharpness[1])
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a `sharpness_factor` uniformly from `self.sharpness`."""
|
||||
sharpness_factor = torch.empty(1).uniform_(self.sharpness[0], self.sharpness[1]).item()
|
||||
return {"sharpness_factor": sharpness_factor}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Adjust `inpt`'s sharpness by `params["sharpness_factor"]`."""
|
||||
sharpness_factor = params["sharpness_factor"]
|
||||
return self._call_kernel(F.adjust_sharpness, inpt, sharpness_factor=sharpness_factor)
|
||||
|
||||
@@ -153,10 +168,17 @@ class GaussianNoise(Transform):
|
||||
Common in real-robot setups where wrist cameras operate in suboptimal lighting.
|
||||
|
||||
Args:
|
||||
std: Range (min, max) for noise standard deviation in pixel-value scale (0-255).
|
||||
std (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(5.0, 25.0)`):
|
||||
Range `(min, max)` for the noise standard deviation, in pixel-value scale (0-255).
|
||||
"""
|
||||
|
||||
def __init__(self, std: float | Sequence[float] = (5.0, 25.0)) -> None:
|
||||
"""Normalize `std` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `std` is not a number or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `0 <= min <= max`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(std, (int, float)):
|
||||
self.std = (0.0, float(std))
|
||||
@@ -168,12 +190,14 @@ class GaussianNoise(Transform):
|
||||
raise ValueError(f"std must satisfy 0 <= min <= max, but got {self.std}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a noise `std` uniformly from `self.std`, plus a seed for reproducible noise."""
|
||||
return {
|
||||
"std": torch.empty(1).uniform_(self.std[0], self.std[1]).item(),
|
||||
"seed": torch.randint(0, torch.iinfo(torch.int64).max, ()).item(),
|
||||
}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Add Gaussian noise with `params["std"]` (in pixel-value scale) to `inpt`, if it's a float tensor."""
|
||||
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
|
||||
generator = torch.Generator(device=inpt.device).manual_seed(params["seed"])
|
||||
noise = torch.randn(inpt.shape, device=inpt.device, dtype=inpt.dtype, generator=generator)
|
||||
@@ -187,10 +211,17 @@ class MotionBlur(Transform):
|
||||
Generates a 1D averaging kernel along a random direction, applied via depthwise convolution.
|
||||
|
||||
Args:
|
||||
kernel_size: An odd kernel size or a range containing at least one odd kernel size.
|
||||
kernel_size (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3, 11)`):
|
||||
An odd kernel size, or a `(min, max)` range containing at least one odd kernel size.
|
||||
"""
|
||||
|
||||
def __init__(self, kernel_size: int | Sequence[int] = (3, 11)) -> None:
|
||||
"""Normalize `kernel_size` into a `(min, max)` range containing at least one odd value.
|
||||
|
||||
Raises:
|
||||
TypeError: If `kernel_size` is not an int or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `1 <= min <= max`, or contains no odd value.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(kernel_size, int):
|
||||
self.kernel_size = (kernel_size, kernel_size)
|
||||
@@ -205,6 +236,7 @@ class MotionBlur(Transform):
|
||||
raise ValueError(f"kernel_size range must contain an odd value, but got {self.kernel_size}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample an odd kernel size from `self.kernel_size` and a random blur direction in degrees."""
|
||||
num_odd_sizes = (self.kernel_size[1] - self._first_odd_kernel_size) // 2 + 1
|
||||
size_index = int(torch.randint(0, num_odd_sizes, ()).item())
|
||||
ks = self._first_odd_kernel_size + 2 * size_index
|
||||
@@ -212,6 +244,11 @@ class MotionBlur(Transform):
|
||||
return {"kernel_size": ks, "angle": angle}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Convolve `inpt` with a directional averaging kernel per `params`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -241,10 +278,17 @@ class JPEGCompression(Transform):
|
||||
Models quality degradation from video compression in network-streamed camera feeds.
|
||||
|
||||
Args:
|
||||
quality: Range (min, max) for JPEG quality factor (lower = more artifacts).
|
||||
quality (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(15, 75)`):
|
||||
Range `(min, max)` for the JPEG quality factor. Lower values produce more artifacts.
|
||||
"""
|
||||
|
||||
def __init__(self, quality: int | Sequence[int] = (15, 75)) -> None:
|
||||
"""Normalize `quality` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `quality` is not an int or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `1 <= min <= max <= 100`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(quality, int):
|
||||
self.quality = (quality, quality)
|
||||
@@ -256,9 +300,16 @@ class JPEGCompression(Transform):
|
||||
raise ValueError(f"quality must satisfy 1 <= min <= max <= 100, but got {self.quality}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a JPEG `quality` factor uniformly (as an int) from `self.quality`."""
|
||||
return {"quality": int(torch.randint(self.quality[0], self.quality[1] + 1, (1,)).item())}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Re-encode and decode `inpt` as JPEG at `params["quality"]`, introducing compression artifacts.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions, or with a channel count
|
||||
other than 1 or 3.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -284,9 +335,12 @@ class GaussianPatchBrightness(Transform):
|
||||
encountered in real robot workspaces with multiple light sources.
|
||||
|
||||
Args:
|
||||
num_patches: Range (min, max) for number of brightness patches.
|
||||
sigma_range: Range for Gaussian sigma as fraction of image size.
|
||||
factor_range: Range for brightness factor (< 1 darkens, > 1 brightens).
|
||||
num_patches (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(1, 4)`):
|
||||
Range `(min, max)` for the number of brightness patches.
|
||||
sigma_range (`Sequence`, *optional*, defaults to `(0.05, 0.25)`):
|
||||
Range `(min, max)` for each patch's Gaussian sigma, as a fraction of image size.
|
||||
factor_range (`Sequence`, *optional*, defaults to `(0.4, 1.6)`):
|
||||
Range `(min, max)` for the brightness factor; below 1 darkens, above 1 brightens.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -295,6 +349,12 @@ class GaussianPatchBrightness(Transform):
|
||||
sigma_range: Sequence[float] = (0.05, 0.25),
|
||||
factor_range: Sequence[float] = (0.4, 1.6),
|
||||
) -> None:
|
||||
"""Validate and store the patch count, size, and brightness ranges.
|
||||
|
||||
Raises:
|
||||
TypeError: If any range argument is not the expected type or length.
|
||||
ValueError: If any range does not satisfy `min <= max` within its valid bounds.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(num_patches, int):
|
||||
self.num_patches = (num_patches, num_patches)
|
||||
@@ -316,6 +376,7 @@ class GaussianPatchBrightness(Transform):
|
||||
raise ValueError(f"factor_range must satisfy 0 <= min <= max, but got {self.factor_range}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a random number of patches, each with a random center, sigma, and brightness factor."""
|
||||
n = int(torch.randint(self.num_patches[0], self.num_patches[1] + 1, (1,)).item())
|
||||
return {
|
||||
"centers": torch.rand(n, 2).tolist(),
|
||||
@@ -324,6 +385,7 @@ class GaussianPatchBrightness(Transform):
|
||||
}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Multiply `inpt` by a mask of overlapping Gaussian brightness patches per `params`."""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
h, w = inpt.shape[-2:]
|
||||
@@ -347,10 +409,17 @@ class RandomShadow(Transform):
|
||||
Symmetric: randomly brightens or darkens to prevent BatchNorm stats shift.
|
||||
|
||||
Args:
|
||||
opacity: Range (min, max) for shadow/highlight opacity.
|
||||
opacity (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.3, 0.6)`):
|
||||
Range `(min, max)` for the shadow/highlight opacity.
|
||||
"""
|
||||
|
||||
def __init__(self, opacity: float | Sequence[float] = (0.3, 0.6)) -> None:
|
||||
"""Normalize `opacity` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `opacity` is not a number or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `0 <= min <= max <= 1`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(opacity, (int, float)):
|
||||
self.opacity = (float(opacity), float(opacity))
|
||||
@@ -362,6 +431,7 @@ class RandomShadow(Transform):
|
||||
raise ValueError(f"opacity must satisfy 0 <= min <= max <= 1, but got {self.opacity}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample the shadow band's opacity, horizontal position/width, and darken-vs-brighten direction."""
|
||||
return {
|
||||
"opacity": torch.empty(1).uniform_(self.opacity[0], self.opacity[1]).item(),
|
||||
"start": torch.rand(1).item(),
|
||||
@@ -370,6 +440,11 @@ class RandomShadow(Transform):
|
||||
}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Multiply `inpt` by a soft-edged vertical band mask per `params`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -401,10 +476,14 @@ class CoarseDropout(Transform):
|
||||
during robot manipulation.
|
||||
|
||||
Args:
|
||||
max_holes: Maximum number of rectangular patches to drop.
|
||||
max_height_frac: Maximum patch height as fraction of image height.
|
||||
max_width_frac: Maximum patch width as fraction of image width.
|
||||
fill_value: Value to fill dropped regions with.
|
||||
max_holes (`int`, *optional*, defaults to 8):
|
||||
Maximum number of rectangular patches to drop.
|
||||
max_height_frac (`float`, *optional*, defaults to 0.07):
|
||||
Maximum patch height, as a fraction of image height.
|
||||
max_width_frac (`float`, *optional*, defaults to 0.07):
|
||||
Maximum patch width, as a fraction of image width.
|
||||
fill_value (`float`, *optional*, defaults to 0.0):
|
||||
Value to fill dropped regions with.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -414,6 +493,12 @@ class CoarseDropout(Transform):
|
||||
max_width_frac: float = 0.07,
|
||||
fill_value: float = 0.0,
|
||||
) -> None:
|
||||
"""Validate and store the dropout patch count, size limits, and fill value.
|
||||
|
||||
Raises:
|
||||
TypeError: If `max_holes` is not an int.
|
||||
ValueError: If any argument is out of its valid range.
|
||||
"""
|
||||
super().__init__()
|
||||
if not isinstance(max_holes, int):
|
||||
raise TypeError("max_holes must be an int.")
|
||||
@@ -431,6 +516,7 @@ class CoarseDropout(Transform):
|
||||
self.fill_value = fill_value
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a random number of dropout patches, each with a random size and position."""
|
||||
n = int(torch.randint(1, self.max_holes + 1, (1,)).item())
|
||||
sizes = torch.rand(n, 2)
|
||||
sizes[:, 0] *= self.max_height_frac
|
||||
@@ -438,6 +524,11 @@ class CoarseDropout(Transform):
|
||||
return {"sizes": sizes.tolist(), "positions": torch.rand(n, 2).tolist()}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Fill the rectangular patches described by `params` in `inpt` with `self.fill_value`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -464,10 +555,18 @@ class GammaCorrection(Transform):
|
||||
preventing BatchNorm statistics shift.
|
||||
|
||||
Args:
|
||||
gamma: Range (min, max) for gamma value. Values < 1 brighten, > 1 darken.
|
||||
gamma (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.5, 2.0)`):
|
||||
Range `(min, max)` for the gamma value. Values below 1 brighten, above 1 darken.
|
||||
"""
|
||||
|
||||
def __init__(self, gamma: float | Sequence[float] = (0.5, 2.0)) -> None:
|
||||
"""Normalize `gamma` into a log-symmetric `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `gamma` is not a number or a length-2 sequence.
|
||||
ValueError: If a single `gamma` is not positive, or the resulting range does not satisfy
|
||||
`0 < min <= max`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(gamma, (int, float)):
|
||||
gamma = float(gamma)
|
||||
@@ -482,12 +581,14 @@ class GammaCorrection(Transform):
|
||||
raise ValueError(f"gamma must satisfy 0 < min <= max, but got {self.gamma}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a `gamma` value log-uniformly from `self.gamma`."""
|
||||
log_lo = math.log(self.gamma[0])
|
||||
log_hi = math.log(self.gamma[1])
|
||||
gamma = math.exp(torch.empty(1).uniform_(log_lo, log_hi).item())
|
||||
return {"gamma": gamma}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Raise `inpt` to the power `params["gamma"]`, if it's a float tensor."""
|
||||
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
|
||||
return inpt.pow(params["gamma"]).clamp(0.0, 1.0)
|
||||
return inpt
|
||||
@@ -537,11 +638,18 @@ class PlanckianJitter(Transform):
|
||||
Reference: Zini et al., "Planckian Jitter", CVPR 2022 Workshop.
|
||||
|
||||
Args:
|
||||
temperature: A fixed color temperature or range in Kelvin. Supported values
|
||||
are between 3000 K and 15000 K.
|
||||
temperature (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3000, 15000)`):
|
||||
A fixed color temperature, or a `(min, max)` range, in Kelvin. Supported values are between
|
||||
3000 K and 15000 K.
|
||||
"""
|
||||
|
||||
def __init__(self, temperature: int | Sequence[int] = (3_000, 15_000)) -> None:
|
||||
"""Normalize `temperature` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `temperature` is not an int or a length-2 sequence.
|
||||
ValueError: If the resulting range falls outside `[3000, 15000]` Kelvin.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(temperature, int):
|
||||
self.temperature = (temperature, temperature)
|
||||
@@ -562,10 +670,16 @@ class PlanckianJitter(Transform):
|
||||
)
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a color `temperature` in Kelvin uniformly from `self.temperature`."""
|
||||
temperature = int(torch.randint(self.temperature[0], self.temperature[1] + 1, ()).item())
|
||||
return {"temperature": temperature}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Scale `inpt`'s red/blue channels per the black-body coefficients at `params["temperature"]`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor that isn't 3-channel with at least 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3 or inpt.shape[-3] != 3:
|
||||
@@ -613,15 +727,18 @@ _CUSTOM_TRANSFORMS: dict[str, type[Transform]] = {
|
||||
|
||||
@dataclass
|
||||
class ImageTransformConfig:
|
||||
"""
|
||||
For each transform, the following parameters are available:
|
||||
weight: This represents the multinomial probability (with no replacement)
|
||||
used for sampling the transform. If the sum of the weights is not 1,
|
||||
they will be normalized.
|
||||
type: The name of the class used. This is either a class available under torchvision.transforms.v2 or a
|
||||
custom transform defined here.
|
||||
kwargs: Lower & upper bound respectively used for sampling the transform's parameter
|
||||
(following uniform distribution) when it's applied.
|
||||
"""Configuration for one entry in an [`~transforms.ImageTransformsConfig`]'s `tfs` mapping.
|
||||
|
||||
Args:
|
||||
weight (`float`, *optional*, defaults to 1.0):
|
||||
Multinomial probability (with no replacement) of sampling this transform. Normalized against
|
||||
the other transforms' weights if they don't already sum to 1.
|
||||
type (`str`, *optional*, defaults to `"Identity"`):
|
||||
Name of the transform class to build — either a class under `torchvision.transforms.v2` or one
|
||||
of the custom transforms in this module. Passed to
|
||||
[`~transforms.make_transform_from_config`].
|
||||
kwargs (`dict[str, Any]`, *optional*):
|
||||
Keyword arguments passed to the transform's constructor.
|
||||
"""
|
||||
|
||||
weight: float = 1.0
|
||||
@@ -631,11 +748,21 @@ class ImageTransformConfig:
|
||||
|
||||
@dataclass
|
||||
class ImageTransformsConfig:
|
||||
"""
|
||||
These transforms are all using standard torchvision.transforms.v2
|
||||
You can find out how these transformations affect images here:
|
||||
https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html
|
||||
We use a custom RandomSubsetApply container to sample them.
|
||||
"""Configuration for [`~transforms.ImageTransforms`], a random subset of image augmentations.
|
||||
|
||||
Transforms are standard [`torchvision.transforms.v2`](https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html)
|
||||
or custom transforms from this module, sampled via [`~transforms.RandomSubsetApply`].
|
||||
|
||||
Args:
|
||||
enable (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply transforms at all. `False` disables augmentation entirely.
|
||||
max_num_transforms (`int`, *optional*, defaults to 3):
|
||||
Maximum number of transforms (sampled from `tfs`) applied to each frame. Must be in
|
||||
`[1, len(tfs)]`.
|
||||
random_order (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply the sampled transforms in a random order, instead of the order in `tfs`.
|
||||
tfs (`dict[str, ImageTransformConfig]`, *optional*):
|
||||
The available transforms, keyed by name, with their sampling weight and constructor arguments.
|
||||
"""
|
||||
|
||||
# Set this flag to `true` to enable transforms during training
|
||||
@@ -683,6 +810,19 @@ class ImageTransformsConfig:
|
||||
|
||||
|
||||
def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
|
||||
"""Instantiate the transform named by `cfg.type`, from `torchvision.transforms.v2` or this module.
|
||||
|
||||
Args:
|
||||
cfg (`ImageTransformConfig`):
|
||||
Configuration naming the transform class and its constructor arguments.
|
||||
|
||||
Returns:
|
||||
`Transform`: The instantiated transform.
|
||||
|
||||
Raises:
|
||||
ValueError: If `cfg.type` is not a `torchvision.transforms.v2` transform or one of this module's
|
||||
custom transforms.
|
||||
"""
|
||||
if cfg.type in _CUSTOM_TRANSFORMS:
|
||||
return _CUSTOM_TRANSFORMS[cfg.type](**cfg.kwargs)
|
||||
|
||||
@@ -698,9 +838,20 @@ def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
|
||||
|
||||
|
||||
class ImageTransforms(Transform):
|
||||
"""A class to compose image transforms based on configuration."""
|
||||
"""Composes a random subset of image augmentations from an [`~transforms.ImageTransformsConfig`].
|
||||
|
||||
Builds each enabled transform (weight > 0) named in `cfg.tfs`, then wraps them in a
|
||||
[`~transforms.RandomSubsetApply`] so a random subset is applied on each call. If `cfg.enable` is
|
||||
`False` or no transforms are enabled, this is equivalent to the identity transform.
|
||||
"""
|
||||
|
||||
def __init__(self, cfg: ImageTransformsConfig) -> None:
|
||||
"""Build the enabled transforms from `cfg` and wrap them in a random-subset sampler.
|
||||
|
||||
Args:
|
||||
cfg (`ImageTransformsConfig`):
|
||||
Configuration listing the available transforms and how many to sample per call.
|
||||
"""
|
||||
super().__init__()
|
||||
self._cfg = cfg
|
||||
|
||||
@@ -725,4 +876,5 @@ class ImageTransforms(Transform):
|
||||
)
|
||||
|
||||
def forward(self, *inputs: Any) -> Any:
|
||||
"""Apply the sampled subset of transforms (or the identity, if none are enabled) to `inputs`."""
|
||||
return self.tf(*inputs)
|
||||
|
||||
@@ -60,7 +60,7 @@ PATH_TO_LEROBOT = PATH_TO_REPO / "src" / "lerobot"
|
||||
# Modules whose public objects are checked. Add a module here once its docstrings follow the standard.
|
||||
MODULES_TO_CHECK = [
|
||||
"lerobot.robots",
|
||||
"lerobot.teleoperators",
|
||||
"lerobot.transforms",
|
||||
]
|
||||
|
||||
# Objects that do not yet follow the standard, so the check can be green from day one. Removing an entry
|
||||
|
||||
@@ -15,6 +15,4 @@ src/lerobot/robots/robot.py
|
||||
src/lerobot/robots/so_follower/config_so_follower.py
|
||||
src/lerobot/robots/so_follower/so_follower.py
|
||||
src/lerobot/robots/utils.py
|
||||
src/lerobot/teleoperators/phone/config_phone.py
|
||||
src/lerobot/teleoperators/teleoperator.py
|
||||
src/lerobot/teleoperators/unitree_g1/unitree_g1.py
|
||||
|
||||
Reference in New Issue
Block a user