Compare commits

..

2 Commits

Author SHA1 Message Date
CarolinePascal 4f1e61b9ac docs(cameras): remove agents_memory/ from the PR
Scratch coordination notes for review, not repo history — this convention was already tried and
deliberately removed once before (8146bd828). The open questions for Person A are inlined into the PR
description instead.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-07 00:06:57 +02:00
CarolinePascal 2178e11eaa docs(cameras): write the API reference docstrings
Takes src/lerobot/cameras/ to 100% public docstring coverage, the first module of Wave 1 after Person A's
robots/ pilot and infrastructure PR. Converts the three config classes that already had prose but used a
bold **Attributes**: block (invisible to check_docstrings.py, not the standard's Args: pattern for config
dataclasses) and documents the previously-bare CameraConfig, its three enums, ZMQCameraConfig, and the
module's utility functions. Adds per-backend sections to docs/source/api/cameras.mdx and removes the
module's ruff D-ignore.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-07 00:01:37 +02:00
18 changed files with 402 additions and 423 deletions
+49
View File
@@ -22,3 +22,52 @@ See the [Cameras guide](../cameras) for choosing and configuring a camera, and
## make_cameras_from_configs
[[autodoc]] lerobot.cameras.make_cameras_from_configs
## ColorMode
[[autodoc]] lerobot.cameras.ColorMode
## Cv2Rotation
[[autodoc]] lerobot.cameras.Cv2Rotation
## Cv2Backends
[[autodoc]] lerobot.cameras.Cv2Backends
## OpenCVCamera
The default backend for USB webcams and video files, built on OpenCV.
[[autodoc]] lerobot.cameras.opencv.OpenCVCamera
- all
[[autodoc]] lerobot.cameras.opencv.OpenCVCameraConfig
## RealSenseCamera
Intel RealSense cameras, with optional depth sensing.
[[autodoc]] lerobot.cameras.realsense.RealSenseCamera
- all
[[autodoc]] lerobot.cameras.realsense.RealSenseCameraConfig
## Reachy2Camera
Cameras exposed by a Reachy 2 robot's own camera manager.
[[autodoc]] lerobot.cameras.reachy2_camera.Reachy2Camera
- all
[[autodoc]] lerobot.cameras.reachy2_camera.Reachy2CameraConfig
## ZMQCamera
Reads frames published over a ZeroMQ socket by `ImageServer`, for cameras attached to a different machine
than the one running the policy.
[[autodoc]] lerobot.cameras.zmq.ZMQCamera
- all
[[autodoc]] lerobot.cameras.zmq.ZMQCameraConfig
-71
View File
@@ -1,71 +0,0 @@
# Image Transforms
Data-augmentation transforms applied to camera observations during training. [`~transforms.ImageTransforms`]
composes a random subset of them, configured via [`~transforms.ImageTransformsConfig`].
## ImageTransforms
[[autodoc]] lerobot.transforms.ImageTransforms
- all
## ImageTransformsConfig
[[autodoc]] lerobot.transforms.ImageTransformsConfig
## ImageTransformConfig
[[autodoc]] lerobot.transforms.ImageTransformConfig
## make_transform_from_config
[[autodoc]] lerobot.transforms.make_transform_from_config
## RandomSubsetApply
[[autodoc]] lerobot.transforms.RandomSubsetApply
- all
## SharpnessJitter
[[autodoc]] lerobot.transforms.SharpnessJitter
- all
## GaussianNoise
[[autodoc]] lerobot.transforms.GaussianNoise
- all
## MotionBlur
[[autodoc]] lerobot.transforms.MotionBlur
- all
## JPEGCompression
[[autodoc]] lerobot.transforms.JPEGCompression
- all
## GaussianPatchBrightness
[[autodoc]] lerobot.transforms.GaussianPatchBrightness
- all
## RandomShadow
[[autodoc]] lerobot.transforms.RandomShadow
- all
## CoarseDropout
[[autodoc]] lerobot.transforms.CoarseDropout
- all
## GammaCorrection
[[autodoc]] lerobot.transforms.GammaCorrection
- all
## PlanckianJitter
[[autodoc]] lerobot.transforms.PlanckianJitter
- all
+1 -1
View File
@@ -437,7 +437,6 @@ ignore = [
# Awaiting conversion, one PR per module.
"src/lerobot/annotations/**" = ["D"]
"src/lerobot/async_inference/**" = ["D"]
"src/lerobot/cameras/**" = ["D"]
"src/lerobot/common/**" = ["D"]
"src/lerobot/configs/**" = ["D"]
"src/lerobot/data_processing/**" = ["D"]
@@ -455,6 +454,7 @@ ignore = [
"src/lerobot/rollout/**" = ["D"]
"src/lerobot/scripts/**" = ["D"]
"src/lerobot/teleoperators/**" = ["D"]
"src/lerobot/transforms/**" = ["D"]
"src/lerobot/transport/**" = ["D"]
"src/lerobot/utils/**" = ["D"]
"src/lerobot/lerobot_types.py" = ["D"]
+7 -6
View File
@@ -50,23 +50,23 @@ class Camera(abc.ABC):
self.height: int | None = config.height
def __enter__(self):
"""
Context manager entry.
"""Context manager entry.
Automatically connects to the camera.
"""
self.connect()
return self
def __exit__(self, exc_type, exc_value, traceback) -> None:
"""
Context manager exit.
"""Context manager exit.
Automatically disconnects, ensuring resources are released even on error.
"""
self.disconnect()
def __del__(self) -> None:
"""
Destructor safety net.
"""Destructor safety net.
Attempts to disconnect if the object is garbage collected without cleanup.
"""
try:
@@ -90,6 +90,7 @@ class Camera(abc.ABC):
@abc.abstractmethod
def find_cameras() -> list[dict[str, Any]]:
"""Detects available cameras connected to the system.
Returns:
List[Dict[str, Any]]: A list of dictionaries,
where each dictionary contains information about a detected camera.
+66
View File
@@ -22,15 +22,36 @@ import draccus # type: ignore # TODO: add type stubs for draccus
class ColorMode(str, Enum):
"""Color channel order for frames returned by a camera.
**Attributes**:
- **RGB** -- Red-green-blue channel order.
- **BGR** -- Blue-green-red channel order, OpenCV's native order.
"""
RGB = "rgb"
BGR = "bgr"
@classmethod
def _missing_(cls, value: object) -> None:
"""Reject a value that is not a valid `ColorMode`.
Raises:
ValueError: Always, naming the invalid value and the valid choices.
"""
raise ValueError(f"`color_mode` is expected to be in {list(cls)}, but {value} is provided.")
class Cv2Rotation(int, Enum):
"""Clockwise rotation to apply to a frame after capture, in degrees.
**Attributes**:
- **NO_ROTATION** -- No rotation.
- **ROTATE_90** -- Rotate 90° clockwise.
- **ROTATE_180** -- Rotate 180°.
- **ROTATE_270** -- Rotate 270° clockwise (90° counter-clockwise).
"""
NO_ROTATION = 0
ROTATE_90 = 90
ROTATE_180 = 180
@@ -38,11 +59,31 @@ class Cv2Rotation(int, Enum):
@classmethod
def _missing_(cls, value: object) -> None:
"""Reject a value that is not a valid `Cv2Rotation`.
Raises:
ValueError: Always, naming the invalid value and the valid choices.
"""
raise ValueError(f"`rotation` is expected to be in {list(cls)}, but {value} is provided.")
# Subset from https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html
class Cv2Backends(int, Enum):
"""OpenCV capture backend to request when opening a device.
See the [OpenCV `VideoCaptureAPIs` reference](https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html)
for the full list this is a subset of.
**Attributes**:
- **ANY** -- Let OpenCV auto-detect the backend.
- **V4L2** -- Video4Linux2, the usual choice on Linux.
- **DSHOW** -- DirectShow, a Windows backend.
- **PVAPI** -- PvAPI, for Prosilica GigE cameras.
- **ANDROID** -- Android's native camera API.
- **AVFOUNDATION** -- AVFoundation, the usual choice on macOS.
- **MSMF** -- Microsoft Media Foundation, a Windows backend.
"""
ANY = 0
V4L2 = 200
DSHOW = 700
@@ -53,15 +94,40 @@ class Cv2Backends(int, Enum):
@classmethod
def _missing_(cls, value: object) -> None:
"""Reject a value that is not a valid `Cv2Backends`.
Raises:
ValueError: Always, naming the invalid value and the valid choices.
"""
raise ValueError(f"`backend` is expected to be in {list(cls)}, but {value} is provided.")
@dataclass(kw_only=True)
class CameraConfig(draccus.ChoiceRegistry, abc.ABC): # type: ignore # TODO: add type stubs for draccus
"""Base configuration shared by every camera backend.
Concrete backends subclass this and register themselves with
`@CameraConfig.register_subclass("name")`, which is what makes `--camera.type=name` work on the
command line. Subclasses inherit the three fields below and must document them alongside their own.
Args:
fps (`int`, *optional*):
Requested frames per second for the color stream. `None` leaves it at the backend's default.
width (`int`, *optional*):
Requested frame width in pixels. `None` leaves it at the backend's default.
height (`int`, *optional*):
Requested frame height in pixels. `None` leaves it at the backend's default.
"""
fps: int | None = None
width: int | None = None
height: int | None = None
@property
def type(self) -> str:
"""Return the registered name this config was registered under.
Returns:
`str`: The name passed to `@CameraConfig.register_subclass`, e.g. `"opencv"`.
"""
return str(self.get_choice_name(self.__class__))
+12 -32
View File
@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Provides the OpenCVCamera class for capturing frames from cameras using OpenCV.
"""
"""Provides the OpenCVCamera class for capturing frames from cameras using OpenCV."""
import logging
import math
@@ -50,8 +48,7 @@ logger = logging.getLogger(__name__)
class OpenCVCamera(Camera):
"""
Manages camera interactions using OpenCV for efficient frame recording.
"""Manages camera interactions using OpenCV for efficient frame recording.
This class provides a high-level interface to connect to, configure, and read
frames from cameras compatible with OpenCV's VideoCapture. It supports both
@@ -93,8 +90,7 @@ class OpenCVCamera(Camera):
"""
def __init__(self, config: OpenCVCameraConfig):
"""
Initializes the OpenCVCamera instance.
"""Initializes the OpenCVCamera instance.
Args:
config: The configuration settings for the camera.
@@ -125,6 +121,7 @@ class OpenCVCamera(Camera):
self._reset_connection_settings()
def __str__(self) -> str:
"""Return a short representation naming the class and its `index_or_path`."""
return f"{self.__class__.__name__}({self.index_or_path})"
def _reset_connection_settings(self) -> None:
@@ -143,8 +140,7 @@ class OpenCVCamera(Camera):
@check_if_already_connected
def connect(self, warmup: bool = True) -> None:
"""
Connects to the OpenCV camera specified in the configuration.
"""Connects to the OpenCV camera specified in the configuration.
Initializes the OpenCV VideoCapture object, sets desired camera properties
(FPS, width, height), starts the background reading thread and performs initial checks.
@@ -158,7 +154,6 @@ class OpenCVCamera(Camera):
ConnectionError: If the specified camera index/path is not found or fails to open.
RuntimeError: If the camera opens but fails to apply requested settings.
"""
# Use 1 thread for OpenCV operations to avoid potential conflicts or
# blocking in multi-threaded applications, especially during data collection.
cv2.setNumThreads(1)
@@ -196,8 +191,7 @@ class OpenCVCamera(Camera):
@check_if_not_connected
def _configure_capture_settings(self) -> None:
"""
Applies the specified FOURCC, FPS, width, and height settings to the connected camera.
"""Applies the specified FOURCC, FPS, width, and height settings to the connected camera.
This method attempts to set the camera properties via OpenCV. It checks if
the camera successfully applied the settings and raises an error if not.
@@ -214,7 +208,6 @@ class OpenCVCamera(Camera):
to the requested value.
DeviceNotConnectedError: If the camera is not connected.
"""
if self.videocapture is None:
raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
@@ -246,7 +239,6 @@ class OpenCVCamera(Camera):
def _validate_fps(self) -> None:
"""Validates and sets the camera's frames per second (FPS)."""
if self.videocapture is None:
raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
@@ -261,7 +253,6 @@ class OpenCVCamera(Camera):
def _validate_fourcc(self) -> None:
"""Validates and sets the camera's FOURCC code."""
fourcc_code = cv2.VideoWriter_fourcc(*self.config.fourcc)
if self.videocapture is None:
@@ -282,7 +273,6 @@ class OpenCVCamera(Camera):
def _validate_width_and_height(self) -> None:
"""Validates and sets the camera's frame capture width and height."""
if self.videocapture is None:
raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
@@ -306,8 +296,7 @@ class OpenCVCamera(Camera):
@staticmethod
def find_cameras() -> list[dict[str, Any]]:
"""
Detects available OpenCV cameras connected to the system.
"""Detects available OpenCV cameras connected to the system.
On Linux, it scans '/dev/video*' paths. On other systems (like macOS, Windows),
it checks indices from 0 up to `MAX_OPENCV_INDEX`.
@@ -375,8 +364,7 @@ class OpenCVCamera(Camera):
@check_if_not_connected
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
"""
Reads a single frame synchronously from the camera.
"""Reads a single frame synchronously from the camera.
This is a blocking call. It waits for the next available frame from the
camera hardware via OpenCV.
@@ -392,7 +380,6 @@ class OpenCVCamera(Camera):
received frame dimensions don't match expectations before rotation.
ValueError: If an invalid `color_mode` is requested.
"""
start_time = time.perf_counter()
if color_mode is not None:
@@ -412,8 +399,7 @@ class OpenCVCamera(Camera):
return frame
def _postprocess_image(self, image: NDArray[Any]) -> NDArray[Any]:
"""
Applies color conversion, dimension validation, and rotation to a raw frame.
"""Applies color conversion, dimension validation, and rotation to a raw frame.
Args:
image (np.ndarray): The raw image frame (expected BGR format from OpenCV).
@@ -426,7 +412,6 @@ class OpenCVCamera(Camera):
RuntimeError: If the raw frame dimensions do not match the configured
`width` and `height`.
"""
if self.color_mode not in (ColorMode.RGB, ColorMode.BGR):
raise ValueError(
f"Invalid color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}."
@@ -452,8 +437,7 @@ class OpenCVCamera(Camera):
return processed_image
def _read_loop(self) -> None:
"""
Internal loop run by the background thread for asynchronous reading.
"""Internal loop run by the background thread for asynchronous reading.
On each iteration:
1. Reads a color frame (blocking call)
@@ -538,8 +522,7 @@ class OpenCVCamera(Camera):
@check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
"""
Reads the latest available frame asynchronously.
"""Reads the latest available frame asynchronously.
This method retrieves the most recent frame captured by the background
read thread. It does not block waiting for the camera hardware directly,
@@ -559,7 +542,6 @@ class OpenCVCamera(Camera):
TimeoutError: If no frame becomes available within the specified timeout.
RuntimeError: If an unexpected error occurs.
"""
if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.")
@@ -594,7 +576,6 @@ class OpenCVCamera(Camera):
DeviceNotConnectedError: If the camera is not connected.
RuntimeError: If the camera is connected but has not captured any frames yet.
"""
if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.")
@@ -614,8 +595,7 @@ class OpenCVCamera(Camera):
return frame
def disconnect(self) -> None:
"""
Disconnects from the camera and cleans up resources.
"""Disconnects from the camera and cleans up resources.
Stops the background read thread (if running) and releases the OpenCV
VideoCapture object.
@@ -40,20 +40,28 @@ class OpenCVCameraConfig(CameraConfig):
OpenCVCameraConfig(0, 30, 1280, 720, fourcc="YUYV") # With YUYV format
```
**Attributes**:
- **index_or_path** (`int | Path`) -- Either an integer representing the camera device index, or a
Path object pointing to a video file.
- **fps** -- Requested frames per second for the color stream.
- **width** -- Requested frame width in pixels for the color stream.
- **height** -- Requested frame height in pixels for the color stream.
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no
rotation.
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds)
- **fourcc** (`str | None`) -- FOURCC code for video format (e.g., "MJPG", "YUYV", "I420"). Defaults
to None (auto-detect).
- **backend** (`Cv2Backends`) -- OpenCV backend identifier
(https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html). Defaults to ANY.
Args:
index_or_path (`int | Path`):
Either an integer representing the camera device index, or a Path object pointing to a video
file.
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
Color mode for image output.
rotation (`Cv2Rotation`, *optional*, defaults to `Cv2Rotation.NO_ROTATION`):
Image rotation setting (0°, 90°, 180°, or 270°).
warmup_s (`int`, *optional*, defaults to 1):
Time reading frames before returning from connect (in seconds).
fourcc (`str`, *optional*):
FOURCC code for video format (e.g., `"MJPG"`, `"YUYV"`, `"I420"`). `None` auto-detects.
backend (`Cv2Backends`, *optional*, defaults to `Cv2Backends.ANY`):
OpenCV backend identifier. See [`~cameras.Cv2Backends`] for the supported values.
fps (`int`, *optional*):
Requested frames per second for the color stream. `None` leaves it at the backend's default.
width (`int`, *optional*):
Requested frame width in pixels for the color stream. `None` leaves it at the backend's
default.
height (`int`, *optional*):
Requested frame height in pixels for the color stream. `None` leaves it at the backend's
default.
Note:
- Only 3-channel color output (RGB/BGR) is currently supported.
@@ -69,6 +77,11 @@ class OpenCVCameraConfig(CameraConfig):
backend: Cv2Backends = Cv2Backends.ANY
def __post_init__(self) -> None:
"""Normalize `color_mode`, `rotation`, and `backend`, and validate `fourcc`.
Raises:
ValueError: If `fourcc` is set and is not a 4-character string.
"""
self.color_mode = ColorMode(self.color_mode)
self.rotation = Cv2Rotation(self.rotation)
self.backend = Cv2Backends(self.backend)
@@ -43,16 +43,24 @@ class Reachy2CameraConfig(CameraConfig):
) # Left teleop camera, 640x480 @ 30FPS
```
**Attributes**:
- **name** (`str`) -- Name of the camera device. Can be "teleop" or "depth".
- **image_type** (`str`) -- Type of image stream. For "teleop" camera, can be "left" or "right". For
"depth" camera, can be "rgb" or "depth". (depth is not supported yet)
- **fps** -- Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
- **width** -- Requested frame width in pixels for the color stream.
- **height** -- Requested frame height in pixels for the color stream.
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
- **ip_address** (`str | None`) -- IP address of the robot. Defaults to "localhost".
- **port** (`int`) -- Port number for the camera server. Defaults to 50065.
Args:
name (`str`):
Name of the camera device. Either `"teleop"` or `"depth"`.
image_type (`str`):
Type of image stream. For the `"teleop"` camera, either `"left"` or `"right"`. For the
`"depth"` camera, either `"rgb"` or `"depth"` (depth is not supported yet).
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
Color mode for image output.
ip_address (`str`, *optional*, defaults to `"localhost"`):
IP address of the robot.
port (`int`, *optional*, defaults to 50065):
Port number for the camera server.
fps (`int`, *optional*):
Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
width (`int`, *optional*):
Requested frame width in pixels for the color stream.
height (`int`, *optional*):
Requested frame height in pixels for the color stream.
Note:
- Only 3-channel color output (RGB/BGR) is currently supported.
@@ -65,6 +73,11 @@ class Reachy2CameraConfig(CameraConfig):
port: int = 50065
def __post_init__(self) -> None:
"""Normalize `color_mode` and validate `name`/`image_type`.
Raises:
ValueError: If `name` is not `"teleop"`/`"depth"`, or `image_type` is not valid for `name`.
"""
if self.name not in ["teleop", "depth"]:
raise ValueError(f"`name` is expected to be 'teleop' or 'depth', but {self.name} is provided.")
if (self.name == "teleop" and self.image_type not in ["left", "right"]) or (
@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Provides the Reachy2Camera class for capturing frames from Reachy 2 cameras using Reachy 2's CameraManager.
"""
"""Provides the Reachy2Camera class for capturing frames from Reachy 2 cameras using Reachy 2's CameraManager."""
from __future__ import annotations
@@ -42,6 +40,8 @@ else:
CameraManager = None
class CameraView:
"""Fallback stand-in for `reachy2_sdk`'s `CameraView` when the SDK is not installed."""
LEFT = 0
RIGHT = 1
@@ -55,8 +55,7 @@ logger = logging.getLogger(__name__)
class Reachy2Camera(Camera):
"""
Manages Reachy 2 camera using Reachy 2 CameraManager.
"""Manages Reachy 2 camera using Reachy 2 CameraManager.
This class provides a high-level interface to connect to, configure, and read
frames from Reachy 2 cameras. It supports both synchronous and asynchronous
@@ -70,8 +69,7 @@ class Reachy2Camera(Camera):
"""
def __init__(self, config: Reachy2CameraConfig):
"""
Initializes the Reachy2Camera instance.
"""Initializes the Reachy2Camera instance.
Args:
config: The configuration settings for the camera.
@@ -88,6 +86,7 @@ class Reachy2Camera(Camera):
self.cam_manager: CameraManager | None = None
def __str__(self) -> str:
"""Return a short representation naming the class, camera name, and image type."""
return f"{self.__class__.__name__}({self.config.name}, {self.config.image_type})"
@property
@@ -105,8 +104,7 @@ class Reachy2Camera(Camera):
raise ValueError(f"Invalid camera name '{self.config.name}'. Expected 'teleop' or 'depth'.")
def connect(self, warmup: bool = True) -> None:
"""
Connects to the Reachy2 CameraManager as specified in the configuration.
"""Connects to the Reachy2 CameraManager as specified in the configuration.
Raises:
DeviceNotConnectedError: If the camera is not connected.
@@ -120,15 +118,12 @@ class Reachy2Camera(Camera):
@staticmethod
def find_cameras() -> list[dict[str, Any]]:
"""
Detection not implemented for Reachy2 cameras.
"""
"""Detection not implemented for Reachy2 cameras."""
raise NotImplementedError("Camera detection is not implemented for Reachy2 cameras.")
@check_if_not_connected
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
"""
Reads a single frame synchronously from the camera.
"""Reads a single frame synchronously from the camera.
This method retrieves the most recent frame available in Reachy 2's low-level software.
@@ -187,8 +182,7 @@ class Reachy2Camera(Camera):
@check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
"""
Same as read()
"""Same as [`~cameras.reachy2_camera.Reachy2Camera.read`]; Reachy 2 has no separate async path.
Returns:
np.ndarray: The latest captured frame as a NumPy array in the format
@@ -199,7 +193,6 @@ class Reachy2Camera(Camera):
TimeoutError: If no frame becomes available within the specified timeout.
RuntimeError: If an unexpected error occurs.
"""
return self.read()
@check_if_not_connected
@@ -220,7 +213,6 @@ class Reachy2Camera(Camera):
DeviceNotConnectedError: If the camera is not connected.
RuntimeError: If the camera is connected but has not captured any frames yet.
"""
if self.latest_frame is None or self.latest_timestamp is None:
raise RuntimeError(f"{self} has not captured any frames yet.")
@@ -234,13 +226,11 @@ class Reachy2Camera(Camera):
@check_if_not_connected
def disconnect(self) -> None:
"""
Stops the background read thread (if running).
"""Stops the background read thread (if running).
Raises:
DeviceNotConnectedError: If the camera is already disconnected.
"""
if self.cam_manager is not None:
self.cam_manager.disconnect()
@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Provides the RealSenseCamera class for capturing frames from Intel RealSense cameras.
"""
"""Provides the RealSenseCamera class for capturing frames from Intel RealSense cameras."""
import logging
import sys
@@ -46,8 +44,7 @@ pkg_name = "pyrealsense2-macosx" if sys.platform == "darwin" else "pyrealsense2"
class RealSenseCamera(Camera):
"""
Manages interactions with Intel RealSense cameras for frame and depth recording.
"""Manages interactions with Intel RealSense cameras for frame and depth recording.
This class provides an interface similar to `OpenCVCamera` but tailored for
RealSense devices, leveraging the `pyrealsense2` library. It uses the camera's
@@ -115,8 +112,7 @@ class RealSenseCamera(Camera):
_MAX_CONNECT_ATTEMPTS = 3
def __init__(self, config: RealSenseCameraConfig):
"""
Initializes the RealSenseCamera instance.
"""Initializes the RealSenseCamera instance.
Args:
config: The configuration settings for the camera.
@@ -161,6 +157,7 @@ class RealSenseCamera(Camera):
self._reset_connection_settings()
def __str__(self) -> str:
"""Return a short representation naming the class and its `serial_number`."""
return f"{self.__class__.__name__}({self.serial_number})"
def _reset_connection_settings(self) -> None:
@@ -250,8 +247,7 @@ class RealSenseCamera(Camera):
@check_if_already_connected
def connect(self, warmup: bool = True) -> None:
"""
Connects to the RealSense camera specified in the configuration.
"""Connects to the RealSense camera specified in the configuration.
Initializes the RealSense pipeline, configures the required streams (color
and optionally depth), starts the pipeline, and validates the actual stream settings.
@@ -270,7 +266,6 @@ class RealSenseCamera(Camera):
ConnectionError: If the camera is found but fails to start the pipeline or no RealSense devices are detected at all.
RuntimeError: If the pipeline starts but fails to apply requested settings.
"""
if not warmup:
self._open_pipeline()
logger.info(f"{self} connected.")
@@ -306,8 +301,7 @@ class RealSenseCamera(Camera):
@staticmethod
def find_cameras() -> list[dict[str, Any]]:
"""
Detects available Intel RealSense cameras connected to the system.
"""Detects available Intel RealSense cameras connected to the system.
Returns:
List[Dict[str, Any]]: A list of dictionaries,
@@ -406,7 +400,6 @@ class RealSenseCamera(Camera):
Raises:
DeviceNotConnectedError: If device is not connected.
"""
if self.rs_profile is None:
raise RuntimeError(f"{self}: rs_profile must be initialized before use.")
@@ -544,8 +537,7 @@ class RealSenseCamera(Camera):
@check_if_not_connected
def read_depth(self, timeout_ms: int = 200) -> NDArray[Any]:
"""
Reads a single frame (depth) synchronously from the camera.
"""Reads a single frame (depth) synchronously from the camera.
This is a blocking call. It waits for a coherent set of frames (depth)
from the camera hardware via the RealSense pipeline.
@@ -583,8 +575,7 @@ class RealSenseCamera(Camera):
@check_if_not_connected
def read(self, color_mode: ColorMode | None = None, timeout_ms: int = 0) -> NDArray[Any]:
"""
Reads a single frame (color) synchronously from the camera.
"""Reads a single frame (color) synchronously from the camera.
This is a blocking call. It waits for a coherent set of frames (color)
from the camera hardware via the RealSense pipeline.
@@ -598,7 +589,6 @@ class RealSenseCamera(Camera):
RuntimeError: If reading frames from the pipeline fails or frames are invalid.
ValueError: If an invalid `color_mode` is requested.
"""
start_time = time.perf_counter()
if color_mode is not None:
@@ -622,11 +612,12 @@ class RealSenseCamera(Camera):
return frame
def _postprocess_image(self, image: NDArray[Any], depth_frame: bool = False) -> NDArray[Any]:
"""
Applies color conversion, dimension validation, and rotation to a raw color frame.
"""Applies color conversion, dimension validation, and rotation to a raw color frame.
Args:
image (np.ndarray): The raw image frame (expected RGB format from RealSense).
depth_frame (bool): Whether `image` is a single-channel depth frame rather than a 3-channel
color frame.
Returns:
np.ndarray: The processed image frame according to `self.color_mode` and `self.rotation`.
@@ -636,7 +627,6 @@ class RealSenseCamera(Camera):
RuntimeError: If the raw frame dimensions do not match the configured
`width` and `height`.
"""
if self.color_mode and self.color_mode not in (ColorMode.RGB, ColorMode.BGR):
raise ValueError(
f"Invalid requested color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}."
@@ -665,8 +655,7 @@ class RealSenseCamera(Camera):
return processed_image
def _read_loop(self) -> None:
"""
Internal loop run by the background thread for asynchronous reading.
"""Internal loop run by the background thread for asynchronous reading.
On each iteration:
1. Reads a color/depth frame (blocking call with 10s timeout)
@@ -790,8 +779,7 @@ class RealSenseCamera(Camera):
@check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
"""
Reads the latest available frame data (color) asynchronously.
"""Reads the latest available frame data (color) asynchronously.
This method retrieves the most recent color frame captured by the background
read thread. It does not block waiting for the camera hardware directly,
@@ -811,7 +799,6 @@ class RealSenseCamera(Camera):
TimeoutError: If no frame data becomes available within the specified timeout.
RuntimeError: If the background thread died unexpectedly or another error occurs.
"""
if not self.use_rgb:
raise RuntimeError(f"{self}: cannot read color — camera was configured with use_rgb=False.")
@@ -897,15 +884,13 @@ class RealSenseCamera(Camera):
return self._read_latest(max_age_ms=max_age_ms, read_depth=True)
def disconnect(self) -> None:
"""
Disconnects from the camera, stops the pipeline, and cleans up resources.
"""Disconnects from the camera, stops the pipeline, and cleans up resources.
Stops the background read thread (if running) and stops the RealSense pipeline.
Raises:
DeviceNotConnectedError: If the camera is already disconnected (pipeline not running).
"""
if not self.is_connected and self.thread is None:
raise DeviceNotConnectedError(
f"Attempted to disconnect {self}, but it appears already disconnected."
@@ -36,28 +36,39 @@ class RealSenseCameraConfig(CameraConfig):
RealSenseCameraConfig("0123456789", 30, 640, 480, rotation=Cv2Rotation.ROTATE_90) # With 90° rotation
```
**Attributes**:
- **fps** -- Requested frames per second for the color stream.
- **width** -- Requested frame width in pixels for the color stream.
- **height** -- Requested frame height in pixels for the color stream.
- **serial_number_or_name** (`str`) -- Unique serial number or human-readable name to identify the
camera.
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
- **use_rgb** (`bool`) -- Whether to enable the color stream. Defaults to True.
- **use_depth** (`bool`) -- Whether to enable depth stream. Defaults to False.
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no
rotation.
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds)
- **exposure** (`int | None`) -- Manual exposure value for the color sensor. When set, auto-exposure
is disabled and this fixed value is used. Valid ranges are camera-model specific and reported if the
value is rejected. Defaults to None (leave unchanged).
- **gain** (`int | None`) -- Manual gain value for the color sensor. When set, auto-exposure is
disabled and this fixed gain is used, which also freezes exposure at its current value when no
exposure is configured. Valid ranges are camera-model specific and reported if the value is
rejected. Defaults to None (leave unchanged).
- **white_balance** (`int | None`) -- Manual white balance value for the color sensor. When set, auto
white balance is disabled and this fixed value is used. Valid ranges are camera-model specific and
reported if the value is rejected. Defaults to None (leave unchanged).
Args:
serial_number_or_name (`str`):
Unique serial number or human-readable name to identify the camera.
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
Color mode for image output.
use_rgb (`bool`, *optional*, defaults to `True`):
Whether to enable the color stream.
use_depth (`bool`, *optional*, defaults to `False`):
Whether to enable the depth stream.
rotation (`Cv2Rotation`, *optional*, defaults to `Cv2Rotation.NO_ROTATION`):
Image rotation setting (0°, 90°, 180°, or 270°).
warmup_s (`int`, *optional*, defaults to 1):
Time reading frames before returning from connect (in seconds).
exposure (`int`, *optional*):
Manual exposure value for the color sensor. When set, auto-exposure is disabled and this fixed
value is used. Valid ranges are camera-model specific and reported if the value is rejected.
`None` leaves auto-exposure unchanged.
gain (`int`, *optional*):
Manual gain value for the color sensor. When set, auto-exposure is disabled and this fixed
gain is used, which also freezes exposure at its current value when no exposure is configured.
Valid ranges are camera-model specific and reported if the value is rejected. `None` leaves it
unchanged.
white_balance (`int`, *optional*):
Manual white balance value for the color sensor. When set, auto white balance is disabled and
this fixed value is used. Valid ranges are camera-model specific and reported if the value is
rejected. `None` leaves it unchanged.
fps (`int`, *optional*):
Requested frames per second for the color stream. The depth stream, when enabled, uses the
same FPS. Either all of `fps`, `width`, and `height` must be set, or none of them.
width (`int`, *optional*):
Requested frame width in pixels for the color stream.
height (`int`, *optional*):
Requested frame height in pixels for the color stream.
Note:
- Either name or serial_number must be specified.
@@ -78,6 +89,12 @@ class RealSenseCameraConfig(CameraConfig):
white_balance: int | None = None
def __post_init__(self) -> None:
"""Normalize `color_mode`/`rotation` and validate the stream and sensor-option settings.
Raises:
ValueError: If neither `use_rgb` nor `use_depth` is enabled, if a manual color sensor option
is set without `use_rgb=True`, or if only some of `fps`/`width`/`height` are set.
"""
self.color_mode = ColorMode(self.color_mode)
self.rotation = Cv2Rotation(self.rotation)
+36
View File
@@ -23,6 +23,33 @@ from .configs import CameraConfig, Cv2Rotation
def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[str, Camera]:
"""Instantiate one [`~cameras.Camera`] per entry in a mapping of configs.
Dispatches on each config's registered `type` to build the matching backend class. This only
constructs the camera objects; call [`~cameras.Camera.connect`] on each before use.
Args:
camera_configs (`dict[str, CameraConfig]`):
Camera configs keyed by the name each camera should be identified by, e.g. in a robot's
observation features.
Returns:
`dict[str, Camera]`: A camera instance per key, in the same order as `camera_configs`.
Raises:
ValueError: If a config's type is not a known backend and building it via the generic device
factory also fails.
Example:
```python
>>> from lerobot.cameras.opencv import OpenCVCameraConfig
>>> from lerobot.cameras.utils import make_cameras_from_configs
>>> configs = {"top": OpenCVCameraConfig(index_or_path=0, fps=30, width=640, height=480)}
>>> cameras = make_cameras_from_configs(configs)
>>> list(cameras.keys())
['top']
```
"""
cameras: dict[str, Camera] = {}
for key, cfg in camera_configs.items():
@@ -57,6 +84,15 @@ def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[s
def get_cv2_rotation(rotation: Cv2Rotation) -> int | None:
"""Map a [`~cameras.Cv2Rotation`] to the OpenCV rotation flag `cv2.rotate` expects.
Args:
rotation (`Cv2Rotation`):
The configured rotation.
Returns:
`int | None`: The matching `cv2.ROTATE_*` constant, or `None` for [`~cameras.Cv2Rotation.NO_ROTATION`].
"""
import cv2 # type: ignore # TODO: add type stubs for OpenCV
if rotation == Cv2Rotation.ROTATE_90:
+15 -21
View File
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
ZMQCamera - Captures frames from remote cameras via ZeroMQ using JSON protocol in the
following format:
"""Captures frames from remote cameras via ZeroMQ.
Uses a JSON protocol of the following form:
{
"timestamps": {"camera_name": float},
"images": {"camera_name": "<base64-jpeg>"}
@@ -52,8 +52,7 @@ logger = logging.getLogger(__name__)
class ZMQCamera(Camera):
"""
Manages camera interactions via ZeroMQ for receiving frames from a remote server.
"""Manages camera interactions via ZeroMQ for receiving frames from a remote server.
This class connects to a ZMQ Publisher, subscribes to frame topics, and decodes
incoming JSON messages containing Base64 encoded images. It supports both
@@ -81,6 +80,11 @@ class ZMQCamera(Camera):
"""
def __init__(self, config: ZMQCameraConfig):
"""Initialize the camera with the given configuration.
Args:
config: Camera configuration, including the image server's address and port.
"""
require_package("pyzmq", extra="pyzmq-dep", import_name="zmq")
super().__init__(config)
@@ -105,6 +109,7 @@ class ZMQCamera(Camera):
self.new_frame_event: Event = Event()
def __str__(self) -> str:
"""Return a short representation naming the camera and its server address."""
return f"ZMQCamera({self.camera_name}@{self.server_address}:{self.port})"
@property
@@ -120,7 +125,6 @@ class ZMQCamera(Camera):
warmup (bool): If True, waits for the camera to provide at least one
valid frame before returning. Defaults to True.
"""
logger.info(f"Connecting to {self}...")
try:
@@ -171,15 +175,11 @@ class ZMQCamera(Camera):
@staticmethod
def find_cameras() -> list[dict[str, Any]]:
"""
Detection not implemented for ZMQ cameras. These cameras require manual configuration (server address/port).
"""
"""Detection not implemented for ZMQ cameras; they require manual server address/port configuration."""
raise NotImplementedError("Camera detection is not implemented for ZMQ cameras.")
def _read_from_hardware(self) -> NDArray[Any]:
"""
Reads a single frame directly from the ZMQ socket.
"""
"""Reads a single frame directly from the ZMQ socket."""
if not self.is_connected or self.socket is None:
raise DeviceNotConnectedError(f"{self} is not connected.")
@@ -215,8 +215,7 @@ class ZMQCamera(Camera):
@check_if_not_connected
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
"""
Reads a single frame synchronously from the camera.
"""Reads a single frame synchronously from the camera.
This is a blocking call. It waits for the next available frame from the
camera background thread.
@@ -243,9 +242,7 @@ class ZMQCamera(Camera):
return frame
def _read_loop(self) -> None:
"""
Internal loop run by the background thread for asynchronous reading.
"""
"""Internal loop run by the background thread for asynchronous reading."""
stop_event = self.stop_event
if stop_event is None:
raise RuntimeError(f"{self}: stop_event is not initialized.")
@@ -306,8 +303,7 @@ class ZMQCamera(Camera):
@check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
"""
Reads the latest available frame asynchronously.
"""Reads the latest available frame asynchronously.
Args:
timeout_ms (float): Maximum time in milliseconds to wait for a frame
@@ -321,7 +317,6 @@ class ZMQCamera(Camera):
TimeoutError: If no frame data becomes available within the specified timeout.
RuntimeError: If the background thread is not running.
"""
if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.")
@@ -353,7 +348,6 @@ class ZMQCamera(Camera):
DeviceNotConnectedError: If the camera is not connected.
RuntimeError: If the camera is connected but has not captured any frames yet.
"""
if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.")
@@ -24,6 +24,32 @@ __all__ = ["ZMQCameraConfig", "ColorMode"]
@CameraConfig.register_subclass("zmq")
@dataclass
class ZMQCameraConfig(CameraConfig):
"""Configuration for a camera served over a ZeroMQ socket by `ImageServer`.
Use this to read frames from a camera attached to a different machine (e.g. a Raspberry Pi on a
robot), which streams JPEG-encoded frames to this config's `server_address`/`port` over ZMQ.
Args:
server_address (`str`):
Address of the machine running the image server, e.g. `"192.168.1.50"`.
port (`int`, *optional*, defaults to 5555):
TCP port the image server is publishing on. Must be between 1 and 65535.
camera_name (`str`, *optional*, defaults to `"zmq_camera"`):
Name used to identify this camera in logs and observation keys.
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
Color mode for the decoded frames.
timeout_ms (`int`, *optional*, defaults to 5000):
How long to wait for a frame before raising a timeout, in milliseconds. Must be positive.
warmup_s (`int`, *optional*, defaults to 1):
Time spent reading frames before returning from connect, in seconds.
fps (`int`, *optional*):
Requested frames per second. `None` leaves it at the server's own rate.
width (`int`, *optional*):
Requested frame width in pixels. `None` leaves it at the server's own resolution.
height (`int`, *optional*):
Requested frame height in pixels. `None` leaves it at the server's own resolution.
"""
server_address: str
port: int = 5555
camera_name: str = "zmq_camera"
@@ -32,6 +58,12 @@ class ZMQCameraConfig(CameraConfig):
warmup_s: int = 1
def __post_init__(self) -> None:
"""Normalize `color_mode` and validate the socket settings.
Raises:
ValueError: If `timeout_ms` is not positive, `server_address` is empty, or `port` is outside
the 1-65535 range.
"""
self.color_mode = ColorMode(self.color_mode)
if self.timeout_ms <= 0:
+27 -2
View File
@@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Streams camera images over ZMQ.
"""Streams camera images over ZMQ.
Uses lerobot's OpenCVCamera for capture, encodes images to base64 and sends them over ZMQ.
"""
@@ -47,6 +47,12 @@ class CameraCaptureThread:
"""Background thread that continuously captures and encodes frames from a camera."""
def __init__(self, camera: OpenCVCamera, name: str):
"""Initialize the capture thread for one camera.
Args:
camera: The already-connected camera to read frames from.
name: Name used to identify this camera in log messages.
"""
self.camera = camera
self.name = name
self.latest_encoded: str | None = None # Pre-encoded JPEG as base64
@@ -89,7 +95,21 @@ class CameraCaptureThread:
class ImageServer:
"""Reads from one or more local OpenCV cameras and publishes their frames over a ZMQ `PUB` socket.
Runs on the machine physically connected to the cameras (e.g. a Raspberry Pi on a robot); pair with
[`~cameras.zmq.ZMQCamera`] on the machine that consumes the stream.
"""
def __init__(self, config: dict, port: int = 5555):
"""Connect every camera listed in `config` and open the publishing socket.
Args:
config: A mapping with an optional `"fps"` (the publish loop rate, not the camera capture
rate) and a `"cameras"` mapping of camera name to a dict with `"device_id"`, `"shape"`
(`[height, width]`), and `"fourcc"` keys.
port: TCP port to publish on.
"""
# fps controls the publish loop rate (how often frames are sent over ZMQ), not the camera capture rate
self.fps = config.get("fps", 30)
self.cameras: dict[str, OpenCVCamera] = {}
@@ -124,6 +144,11 @@ class ImageServer:
logger.info(f"ImageServer running on port {port}")
def run(self):
"""Start each camera's capture thread and publish frames until interrupted.
Blocks until `KeyboardInterrupt`, then stops the capture threads, disconnects the cameras, and
closes the socket.
"""
frame_count = 0
frame_times = deque(maxlen=60)
last_published_ts: dict[str, float] = {}
+41 -193
View File
@@ -32,16 +32,13 @@ class RandomSubsetApply(Transform):
"""Apply a random subset of N transformations from a list of transformations.
Args:
transforms (`Sequence`):
List of transformations.
p (`list[float] | None`, *optional*):
Multinomial probabilities (with no replacement) used for sampling the transform. Normalized if
they don't already sum to 1. `None` gives all transforms the same probability.
n_subset (`int | None`, *optional*):
Number of transformations to apply. Must be in `[1, len(transforms)]`. `None` applies all of
them.
random_order (`bool`, *optional*, defaults to `False`):
Whether to apply the sampled transformations in a random order.
transforms: list of transformations.
p: represents the multinomial probabilities (with no replacement) used for sampling the transform.
If the sum of the weights is not 1, they will be normalized. If ``None`` (default), all transforms
have the same probability.
n_subset: number of transformations to apply. If ``None``, all transforms are applied.
Must be in [1, len(transforms)].
random_order: apply transformations in a random order.
"""
def __init__(
@@ -51,12 +48,6 @@ class RandomSubsetApply(Transform):
n_subset: int | None = None,
random_order: bool = False,
) -> None:
"""Validate and store the transform pool, sampling weights, and subset size.
Raises:
TypeError: If `transforms` is not a sequence, or `n_subset` is not an int or `None`.
ValueError: If `p`'s length doesn't match `transforms`, or `n_subset` is out of range.
"""
super().__init__()
if not isinstance(transforms, Sequence):
raise TypeError("Argument transforms should be a sequence of callables")
@@ -83,7 +74,6 @@ class RandomSubsetApply(Transform):
self.selected_transforms: list[Callable[..., Any]] = []
def forward(self, *inputs: Any) -> Any:
"""Sample a subset of `self.transforms` and apply them in sequence to `inputs`."""
needs_unpacking = len(inputs) > 1
selected_indices = torch.multinomial(torch.tensor(self.p), self.n_subset)
@@ -99,7 +89,6 @@ class RandomSubsetApply(Transform):
return outputs
def extra_repr(self) -> str:
"""Return the constructor arguments shown in `repr(self)`."""
return (
f"transforms={self.transforms}, "
f"p={self.p}, "
@@ -119,18 +108,16 @@ class SharpnessJitter(Transform):
A sharpness_factor of 0 gives a blurred image, 1 gives the original image while 2 increases the sharpness
by a factor of 2.
If the input is a `torch.Tensor`, it is expected to have `[..., 1 or 3, H, W]` shape, where `...` means
an arbitrary number of leading dimensions.
If the input is a :class:`torch.Tensor`,
it is expected to have [..., 1 or 3, H, W] shape, where ... means an arbitrary number of leading dimensions.
Args:
sharpness (`float | collections.abc.Sequence[float]`):
How much to jitter sharpness. `sharpness_factor` is chosen uniformly from
`[max(0, 1 - sharpness), 1 + sharpness]`, or the given `[min, max]`. Values must be
non-negative.
sharpness: How much to jitter sharpness. sharpness_factor is chosen uniformly from
[max(0, 1 - sharpness), 1 + sharpness] or the given
[min, max]. Should be non negative numbers.
"""
def __init__(self, sharpness: float | Sequence[float]) -> None:
"""Normalize `sharpness` into a `(min, max)` range to sample from on each call."""
super().__init__()
self.sharpness = self._check_input(sharpness)
@@ -151,12 +138,10 @@ class SharpnessJitter(Transform):
return float(sharpness[0]), float(sharpness[1])
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a `sharpness_factor` uniformly from `self.sharpness`."""
sharpness_factor = torch.empty(1).uniform_(self.sharpness[0], self.sharpness[1]).item()
return {"sharpness_factor": sharpness_factor}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Adjust `inpt`'s sharpness by `params["sharpness_factor"]`."""
sharpness_factor = params["sharpness_factor"]
return self._call_kernel(F.adjust_sharpness, inpt, sharpness_factor=sharpness_factor)
@@ -168,17 +153,10 @@ class GaussianNoise(Transform):
Common in real-robot setups where wrist cameras operate in suboptimal lighting.
Args:
std (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(5.0, 25.0)`):
Range `(min, max)` for the noise standard deviation, in pixel-value scale (0-255).
std: Range (min, max) for noise standard deviation in pixel-value scale (0-255).
"""
def __init__(self, std: float | Sequence[float] = (5.0, 25.0)) -> None:
"""Normalize `std` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `std` is not a number or a length-2 sequence.
ValueError: If the resulting range does not satisfy `0 <= min <= max`.
"""
super().__init__()
if isinstance(std, (int, float)):
self.std = (0.0, float(std))
@@ -190,14 +168,12 @@ class GaussianNoise(Transform):
raise ValueError(f"std must satisfy 0 <= min <= max, but got {self.std}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a noise `std` uniformly from `self.std`, plus a seed for reproducible noise."""
return {
"std": torch.empty(1).uniform_(self.std[0], self.std[1]).item(),
"seed": torch.randint(0, torch.iinfo(torch.int64).max, ()).item(),
}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Add Gaussian noise with `params["std"]` (in pixel-value scale) to `inpt`, if it's a float tensor."""
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
generator = torch.Generator(device=inpt.device).manual_seed(params["seed"])
noise = torch.randn(inpt.shape, device=inpt.device, dtype=inpt.dtype, generator=generator)
@@ -211,17 +187,10 @@ class MotionBlur(Transform):
Generates a 1D averaging kernel along a random direction, applied via depthwise convolution.
Args:
kernel_size (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3, 11)`):
An odd kernel size, or a `(min, max)` range containing at least one odd kernel size.
kernel_size: An odd kernel size or a range containing at least one odd kernel size.
"""
def __init__(self, kernel_size: int | Sequence[int] = (3, 11)) -> None:
"""Normalize `kernel_size` into a `(min, max)` range containing at least one odd value.
Raises:
TypeError: If `kernel_size` is not an int or a length-2 sequence.
ValueError: If the resulting range does not satisfy `1 <= min <= max`, or contains no odd value.
"""
super().__init__()
if isinstance(kernel_size, int):
self.kernel_size = (kernel_size, kernel_size)
@@ -236,7 +205,6 @@ class MotionBlur(Transform):
raise ValueError(f"kernel_size range must contain an odd value, but got {self.kernel_size}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample an odd kernel size from `self.kernel_size` and a random blur direction in degrees."""
num_odd_sizes = (self.kernel_size[1] - self._first_odd_kernel_size) // 2 + 1
size_index = int(torch.randint(0, num_odd_sizes, ()).item())
ks = self._first_odd_kernel_size + 2 * size_index
@@ -244,11 +212,6 @@ class MotionBlur(Transform):
return {"kernel_size": ks, "angle": angle}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Convolve `inpt` with a directional averaging kernel per `params`.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt
if inpt.ndim < 3:
@@ -278,17 +241,10 @@ class JPEGCompression(Transform):
Models quality degradation from video compression in network-streamed camera feeds.
Args:
quality (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(15, 75)`):
Range `(min, max)` for the JPEG quality factor. Lower values produce more artifacts.
quality: Range (min, max) for JPEG quality factor (lower = more artifacts).
"""
def __init__(self, quality: int | Sequence[int] = (15, 75)) -> None:
"""Normalize `quality` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `quality` is not an int or a length-2 sequence.
ValueError: If the resulting range does not satisfy `1 <= min <= max <= 100`.
"""
super().__init__()
if isinstance(quality, int):
self.quality = (quality, quality)
@@ -300,16 +256,9 @@ class JPEGCompression(Transform):
raise ValueError(f"quality must satisfy 1 <= min <= max <= 100, but got {self.quality}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a JPEG `quality` factor uniformly (as an int) from `self.quality`."""
return {"quality": int(torch.randint(self.quality[0], self.quality[1] + 1, (1,)).item())}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Re-encode and decode `inpt` as JPEG at `params["quality"]`, introducing compression artifacts.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions, or with a channel count
other than 1 or 3.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt
if inpt.ndim < 3:
@@ -335,12 +284,9 @@ class GaussianPatchBrightness(Transform):
encountered in real robot workspaces with multiple light sources.
Args:
num_patches (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(1, 4)`):
Range `(min, max)` for the number of brightness patches.
sigma_range (`Sequence`, *optional*, defaults to `(0.05, 0.25)`):
Range `(min, max)` for each patch's Gaussian sigma, as a fraction of image size.
factor_range (`Sequence`, *optional*, defaults to `(0.4, 1.6)`):
Range `(min, max)` for the brightness factor; below 1 darkens, above 1 brightens.
num_patches: Range (min, max) for number of brightness patches.
sigma_range: Range for Gaussian sigma as fraction of image size.
factor_range: Range for brightness factor (< 1 darkens, > 1 brightens).
"""
def __init__(
@@ -349,12 +295,6 @@ class GaussianPatchBrightness(Transform):
sigma_range: Sequence[float] = (0.05, 0.25),
factor_range: Sequence[float] = (0.4, 1.6),
) -> None:
"""Validate and store the patch count, size, and brightness ranges.
Raises:
TypeError: If any range argument is not the expected type or length.
ValueError: If any range does not satisfy `min <= max` within its valid bounds.
"""
super().__init__()
if isinstance(num_patches, int):
self.num_patches = (num_patches, num_patches)
@@ -376,7 +316,6 @@ class GaussianPatchBrightness(Transform):
raise ValueError(f"factor_range must satisfy 0 <= min <= max, but got {self.factor_range}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a random number of patches, each with a random center, sigma, and brightness factor."""
n = int(torch.randint(self.num_patches[0], self.num_patches[1] + 1, (1,)).item())
return {
"centers": torch.rand(n, 2).tolist(),
@@ -385,7 +324,6 @@ class GaussianPatchBrightness(Transform):
}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Multiply `inpt` by a mask of overlapping Gaussian brightness patches per `params`."""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt
h, w = inpt.shape[-2:]
@@ -409,17 +347,10 @@ class RandomShadow(Transform):
Symmetric: randomly brightens or darkens to prevent BatchNorm stats shift.
Args:
opacity (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.3, 0.6)`):
Range `(min, max)` for the shadow/highlight opacity.
opacity: Range (min, max) for shadow/highlight opacity.
"""
def __init__(self, opacity: float | Sequence[float] = (0.3, 0.6)) -> None:
"""Normalize `opacity` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `opacity` is not a number or a length-2 sequence.
ValueError: If the resulting range does not satisfy `0 <= min <= max <= 1`.
"""
super().__init__()
if isinstance(opacity, (int, float)):
self.opacity = (float(opacity), float(opacity))
@@ -431,7 +362,6 @@ class RandomShadow(Transform):
raise ValueError(f"opacity must satisfy 0 <= min <= max <= 1, but got {self.opacity}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample the shadow band's opacity, horizontal position/width, and darken-vs-brighten direction."""
return {
"opacity": torch.empty(1).uniform_(self.opacity[0], self.opacity[1]).item(),
"start": torch.rand(1).item(),
@@ -440,11 +370,6 @@ class RandomShadow(Transform):
}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Multiply `inpt` by a soft-edged vertical band mask per `params`.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt
if inpt.ndim < 3:
@@ -476,14 +401,10 @@ class CoarseDropout(Transform):
during robot manipulation.
Args:
max_holes (`int`, *optional*, defaults to 8):
Maximum number of rectangular patches to drop.
max_height_frac (`float`, *optional*, defaults to 0.07):
Maximum patch height, as a fraction of image height.
max_width_frac (`float`, *optional*, defaults to 0.07):
Maximum patch width, as a fraction of image width.
fill_value (`float`, *optional*, defaults to 0.0):
Value to fill dropped regions with.
max_holes: Maximum number of rectangular patches to drop.
max_height_frac: Maximum patch height as fraction of image height.
max_width_frac: Maximum patch width as fraction of image width.
fill_value: Value to fill dropped regions with.
"""
def __init__(
@@ -493,12 +414,6 @@ class CoarseDropout(Transform):
max_width_frac: float = 0.07,
fill_value: float = 0.0,
) -> None:
"""Validate and store the dropout patch count, size limits, and fill value.
Raises:
TypeError: If `max_holes` is not an int.
ValueError: If any argument is out of its valid range.
"""
super().__init__()
if not isinstance(max_holes, int):
raise TypeError("max_holes must be an int.")
@@ -516,7 +431,6 @@ class CoarseDropout(Transform):
self.fill_value = fill_value
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a random number of dropout patches, each with a random size and position."""
n = int(torch.randint(1, self.max_holes + 1, (1,)).item())
sizes = torch.rand(n, 2)
sizes[:, 0] *= self.max_height_frac
@@ -524,11 +438,6 @@ class CoarseDropout(Transform):
return {"sizes": sizes.tolist(), "positions": torch.rand(n, 2).tolist()}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Fill the rectangular patches described by `params` in `inpt` with `self.fill_value`.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt
if inpt.ndim < 3:
@@ -555,18 +464,10 @@ class GammaCorrection(Transform):
preventing BatchNorm statistics shift.
Args:
gamma (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.5, 2.0)`):
Range `(min, max)` for the gamma value. Values below 1 brighten, above 1 darken.
gamma: Range (min, max) for gamma value. Values < 1 brighten, > 1 darken.
"""
def __init__(self, gamma: float | Sequence[float] = (0.5, 2.0)) -> None:
"""Normalize `gamma` into a log-symmetric `(min, max)` range to sample from on each call.
Raises:
TypeError: If `gamma` is not a number or a length-2 sequence.
ValueError: If a single `gamma` is not positive, or the resulting range does not satisfy
`0 < min <= max`.
"""
super().__init__()
if isinstance(gamma, (int, float)):
gamma = float(gamma)
@@ -581,14 +482,12 @@ class GammaCorrection(Transform):
raise ValueError(f"gamma must satisfy 0 < min <= max, but got {self.gamma}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a `gamma` value log-uniformly from `self.gamma`."""
log_lo = math.log(self.gamma[0])
log_hi = math.log(self.gamma[1])
gamma = math.exp(torch.empty(1).uniform_(log_lo, log_hi).item())
return {"gamma": gamma}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Raise `inpt` to the power `params["gamma"]`, if it's a float tensor."""
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
return inpt.pow(params["gamma"]).clamp(0.0, 1.0)
return inpt
@@ -638,18 +537,11 @@ class PlanckianJitter(Transform):
Reference: Zini et al., "Planckian Jitter", CVPR 2022 Workshop.
Args:
temperature (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3000, 15000)`):
A fixed color temperature, or a `(min, max)` range, in Kelvin. Supported values are between
3000 K and 15000 K.
temperature: A fixed color temperature or range in Kelvin. Supported values
are between 3000 K and 15000 K.
"""
def __init__(self, temperature: int | Sequence[int] = (3_000, 15_000)) -> None:
"""Normalize `temperature` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `temperature` is not an int or a length-2 sequence.
ValueError: If the resulting range falls outside `[3000, 15000]` Kelvin.
"""
super().__init__()
if isinstance(temperature, int):
self.temperature = (temperature, temperature)
@@ -670,16 +562,10 @@ class PlanckianJitter(Transform):
)
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a color `temperature` in Kelvin uniformly from `self.temperature`."""
temperature = int(torch.randint(self.temperature[0], self.temperature[1] + 1, ()).item())
return {"temperature": temperature}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Scale `inpt`'s red/blue channels per the black-body coefficients at `params["temperature"]`.
Raises:
ValueError: If `inpt` is a float tensor that isn't 3-channel with at least 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt
if inpt.ndim < 3 or inpt.shape[-3] != 3:
@@ -727,18 +613,15 @@ _CUSTOM_TRANSFORMS: dict[str, type[Transform]] = {
@dataclass
class ImageTransformConfig:
"""Configuration for one entry in an [`~transforms.ImageTransformsConfig`]'s `tfs` mapping.
Args:
weight (`float`, *optional*, defaults to 1.0):
Multinomial probability (with no replacement) of sampling this transform. Normalized against
the other transforms' weights if they don't already sum to 1.
type (`str`, *optional*, defaults to `"Identity"`):
Name of the transform class to build — either a class under `torchvision.transforms.v2` or one
of the custom transforms in this module. Passed to
[`~transforms.make_transform_from_config`].
kwargs (`dict[str, Any]`, *optional*):
Keyword arguments passed to the transform's constructor.
"""
For each transform, the following parameters are available:
weight: This represents the multinomial probability (with no replacement)
used for sampling the transform. If the sum of the weights is not 1,
they will be normalized.
type: The name of the class used. This is either a class available under torchvision.transforms.v2 or a
custom transform defined here.
kwargs: Lower & upper bound respectively used for sampling the transform's parameter
(following uniform distribution) when it's applied.
"""
weight: float = 1.0
@@ -748,21 +631,11 @@ class ImageTransformConfig:
@dataclass
class ImageTransformsConfig:
"""Configuration for [`~transforms.ImageTransforms`], a random subset of image augmentations.
Transforms are standard [`torchvision.transforms.v2`](https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html)
or custom transforms from this module, sampled via [`~transforms.RandomSubsetApply`].
Args:
enable (`bool`, *optional*, defaults to `False`):
Whether to apply transforms at all. `False` disables augmentation entirely.
max_num_transforms (`int`, *optional*, defaults to 3):
Maximum number of transforms (sampled from `tfs`) applied to each frame. Must be in
`[1, len(tfs)]`.
random_order (`bool`, *optional*, defaults to `False`):
Whether to apply the sampled transforms in a random order, instead of the order in `tfs`.
tfs (`dict[str, ImageTransformConfig]`, *optional*):
The available transforms, keyed by name, with their sampling weight and constructor arguments.
"""
These transforms are all using standard torchvision.transforms.v2
You can find out how these transformations affect images here:
https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html
We use a custom RandomSubsetApply container to sample them.
"""
# Set this flag to `true` to enable transforms during training
@@ -810,19 +683,6 @@ class ImageTransformsConfig:
def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
"""Instantiate the transform named by `cfg.type`, from `torchvision.transforms.v2` or this module.
Args:
cfg (`ImageTransformConfig`):
Configuration naming the transform class and its constructor arguments.
Returns:
`Transform`: The instantiated transform.
Raises:
ValueError: If `cfg.type` is not a `torchvision.transforms.v2` transform or one of this module's
custom transforms.
"""
if cfg.type in _CUSTOM_TRANSFORMS:
return _CUSTOM_TRANSFORMS[cfg.type](**cfg.kwargs)
@@ -838,20 +698,9 @@ def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
class ImageTransforms(Transform):
"""Composes a random subset of image augmentations from an [`~transforms.ImageTransformsConfig`].
Builds each enabled transform (weight > 0) named in `cfg.tfs`, then wraps them in a
[`~transforms.RandomSubsetApply`] so a random subset is applied on each call. If `cfg.enable` is
`False` or no transforms are enabled, this is equivalent to the identity transform.
"""
"""A class to compose image transforms based on configuration."""
def __init__(self, cfg: ImageTransformsConfig) -> None:
"""Build the enabled transforms from `cfg` and wrap them in a random-subset sampler.
Args:
cfg (`ImageTransformsConfig`):
Configuration listing the available transforms and how many to sample per call.
"""
super().__init__()
self._cfg = cfg
@@ -876,5 +725,4 @@ class ImageTransforms(Transform):
)
def forward(self, *inputs: Any) -> Any:
"""Apply the sampled subset of transforms (or the identity, if none are enabled) to `inputs`."""
return self.tf(*inputs)
+1 -1
View File
@@ -60,7 +60,7 @@ PATH_TO_LEROBOT = PATH_TO_REPO / "src" / "lerobot"
# Modules whose public objects are checked. Add a module here once its docstrings follow the standard.
MODULES_TO_CHECK = [
"lerobot.robots",
"lerobot.transforms",
"lerobot.cameras",
]
# Objects that do not yet follow the standard, so the check can be green from day one. Removing an entry
+1
View File
@@ -10,6 +10,7 @@
# `utils/check_doctest_list.py` does not care which way round it is.
#
# Keep alphabetically sorted: `make check-doctest-list` enforces it, `make fix-docstrings` sorts it.
src/lerobot/cameras/utils.py
src/lerobot/motors/motors_bus.py
src/lerobot/robots/robot.py
src/lerobot/robots/so_follower/config_so_follower.py