admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-24 18:26:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(depth): add depth support for follower robots

Browse Source
This commit is contained in:
CarolinePascal
2026-06-24 21:29:02 +02:00
parent 5efeef3c2c
commit 28d8fe8858
7 changed files with 126 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lerobot/robots/hope_jr/hope_jr_arm.py
+19 -7
View File
@@ -66,9 +66,14 @@ class HopeJrArm(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -139,10 +144,17 @@ class HopeJrArm(Robot):
# Capture images from cameras # Capture images from cameras
for cam_key, cam in self.cameras.items(): for cam_key, cam in self.cameras.items():
start = time.perf_counter() if getattr(cam, "use_rgb", True):
obs_dict[cam_key] = cam.read_latest() start = time.perf_counter()
dt_ms = (time.perf_counter() - start) * 1e3 obs_dict[cam_key] = cam.read_latest()
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms") dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms")
if getattr(cam, "use_depth", False):
start = time.perf_counter()
obs_dict[f"{cam_key}_depth"] = cam.read_latest_depth()
dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key} depth: {dt_ms:.1f}ms")
return obs_dict return obs_dict
src/lerobot/robots/hope_jr/hope_jr_hand.py
+19 -7
View File
@@ -102,9 +102,14 @@ class HopeJrHand(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -170,10 +175,17 @@ class HopeJrHand(Robot):
# Capture images from cameras # Capture images from cameras
for cam_key, cam in self.cameras.items(): for cam_key, cam in self.cameras.items():
start = time.perf_counter() if getattr(cam, "use_rgb", True):
obs_dict[cam_key] = cam.read_latest() start = time.perf_counter()
dt_ms = (time.perf_counter() - start) * 1e3 obs_dict[cam_key] = cam.read_latest()
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms") dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms")
if getattr(cam, "use_depth", False):
start = time.perf_counter()
obs_dict[f"{cam_key}_depth"] = cam.read_latest_depth()
dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key} depth: {dt_ms:.1f}ms")
return obs_dict return obs_dict
src/lerobot/robots/koch_follower/koch_follower.py
+19 -7
View File
@@ -68,9 +68,14 @@ class KochFollower(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -192,10 +197,17 @@ class KochFollower(Robot):
# Capture images from cameras # Capture images from cameras
for cam_key, cam in self.cameras.items(): for cam_key, cam in self.cameras.items():
start = time.perf_counter() if getattr(cam, "use_rgb", True):
obs_dict[cam_key] = cam.read_latest() start = time.perf_counter()
dt_ms = (time.perf_counter() - start) * 1e3 obs_dict[cam_key] = cam.read_latest()
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms") dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms")
if getattr(cam, "use_depth", False):
start = time.perf_counter()
obs_dict[f"{cam_key}_depth"] = cam.read_latest_depth()
dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key} depth: {dt_ms:.1f}ms")
return obs_dict return obs_dict
src/lerobot/robots/omx_follower/omx_follower.py
+19 -7
View File
@@ -68,9 +68,14 @@ class OmxFollower(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -175,10 +180,17 @@ class OmxFollower(Robot):
# Capture images from cameras # Capture images from cameras
for cam_key, cam in self.cameras.items(): for cam_key, cam in self.cameras.items():
start = time.perf_counter() if getattr(cam, "use_rgb", True):
obs_dict[cam_key] = cam.read_latest() start = time.perf_counter()
dt_ms = (time.perf_counter() - start) * 1e3 obs_dict[cam_key] = cam.read_latest()
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms") dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms")
if getattr(cam, "use_depth", False):
start = time.perf_counter()
obs_dict[f"{cam_key}_depth"] = cam.read_latest_depth()
dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key} depth: {dt_ms:.1f}ms")
return obs_dict return obs_dict
src/lerobot/robots/openarm_follower/openarm_follower.py
+19 -7
View File
@@ -101,9 +101,14 @@ class OpenArmFollower(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
"""Camera features for observation space.""" """Camera features for observation space."""
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -242,10 +247,17 @@ class OpenArmFollower(Robot):
# Capture images from cameras # Capture images from cameras
for cam_key, cam in self.cameras.items(): for cam_key, cam in self.cameras.items():
start = time.perf_counter() if getattr(cam, "use_rgb", True):
obs_dict[cam_key] = cam.read_latest() start = time.perf_counter()
dt_ms = (time.perf_counter() - start) * 1e3 obs_dict[cam_key] = cam.read_latest()
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms") dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms")
if getattr(cam, "use_depth", False):
start = time.perf_counter()
obs_dict[f"{cam_key}_depth"] = cam.read_latest_depth()
dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key} depth: {dt_ms:.1f}ms")
dt_ms = (time.perf_counter() - start) * 1e3 dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} get_observation took: {dt_ms:.1f}ms") logger.debug(f"{self} get_observation took: {dt_ms:.1f}ms")
src/lerobot/robots/rebot_b601_follower/rebot_b601_follower.py
+19 -7
View File
@@ -80,9 +80,14 @@ class RebotB601Follower(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -213,10 +218,17 @@ class RebotB601Follower(Robot):
logger.debug(f"{self} read state: {dt_ms:.1f}ms") logger.debug(f"{self} read state: {dt_ms:.1f}ms")
for cam_key, cam in self.cameras.items(): for cam_key, cam in self.cameras.items():
start = time.perf_counter() if getattr(cam, "use_rgb", True):
obs_dict[cam_key] = cam.read_latest() start = time.perf_counter()
dt_ms = (time.perf_counter() - start) * 1e3 obs_dict[cam_key] = cam.read_latest()
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms") dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key}: {dt_ms:.1f}ms")
if getattr(cam, "use_depth", False):
start = time.perf_counter()
obs_dict[f"{cam_key}_depth"] = cam.read_latest_depth()
dt_ms = (time.perf_counter() - start) * 1e3
logger.debug(f"{self} read {cam_key} depth: {dt_ms:.1f}ms")
return obs_dict return obs_dict
src/lerobot/robots/unitree_g1/unitree_g1.py
+12 -4
View File
@@ -222,9 +222,14 @@ class UnitreeG1(Robot):
@property @property
def _cameras_ft(self) -> dict[str, tuple]: def _cameras_ft(self) -> dict[str, tuple]:
return { features: dict[str, tuple] = {}
cam: (self.config.cameras[cam].height, self.config.cameras[cam].width, 3) for cam in self.cameras for cam in self.cameras:
} cfg = self.config.cameras[cam]
if getattr(cfg, "use_rgb", True):
features[cam] = (cfg.height, cfg.width, 3)
if getattr(cfg, "use_depth", False):
features[f"{cam}_depth"] = (cfg.height, cfg.width, 1)
return features
@cached_property @cached_property
def observation_features(self) -> dict[str, type | tuple]: def observation_features(self) -> dict[str, type | tuple]:
@@ -458,7 +463,10 @@ class UnitreeG1(Robot):
# Cameras - read images from ZMQ cameras # Cameras - read images from ZMQ cameras
for cam_name, cam in self._cameras.items(): for cam_name, cam in self._cameras.items():
obs[cam_name] = cam.read_latest() if getattr(cam, "use_rgb", True):
obs[cam_name] = cam.read_latest()
if getattr(cam, "use_depth", False):
obs[f"{cam_name}_depth"] = cam.read_latest_depth()
return obs return obs