Compare commits

..

2 Commits

Author SHA1 Message Date
CarolinePascal 4f1e61b9ac docs(cameras): remove agents_memory/ from the PR
Scratch coordination notes for review, not repo history — this convention was already tried and
deliberately removed once before (8146bd828). The open questions for Person A are inlined into the PR
description instead.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-07 00:06:57 +02:00
CarolinePascal 2178e11eaa docs(cameras): write the API reference docstrings
Takes src/lerobot/cameras/ to 100% public docstring coverage, the first module of Wave 1 after Person A's
robots/ pilot and infrastructure PR. Converts the three config classes that already had prose but used a
bold **Attributes**: block (invisible to check_docstrings.py, not the standard's Args: pattern for config
dataclasses) and documents the previously-bare CameraConfig, its three enums, ZMQCameraConfig, and the
module's utility functions. Adds per-backend sections to docs/source/api/cameras.mdx and removes the
module's ruff D-ignore.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-07 00:01:37 +02:00
18 changed files with 402 additions and 423 deletions
+49
View File
@@ -22,3 +22,52 @@ See the [Cameras guide](../cameras) for choosing and configuring a camera, and
## make_cameras_from_configs ## make_cameras_from_configs
[[autodoc]] lerobot.cameras.make_cameras_from_configs [[autodoc]] lerobot.cameras.make_cameras_from_configs
## ColorMode
[[autodoc]] lerobot.cameras.ColorMode
## Cv2Rotation
[[autodoc]] lerobot.cameras.Cv2Rotation
## Cv2Backends
[[autodoc]] lerobot.cameras.Cv2Backends
## OpenCVCamera
The default backend for USB webcams and video files, built on OpenCV.
[[autodoc]] lerobot.cameras.opencv.OpenCVCamera
- all
[[autodoc]] lerobot.cameras.opencv.OpenCVCameraConfig
## RealSenseCamera
Intel RealSense cameras, with optional depth sensing.
[[autodoc]] lerobot.cameras.realsense.RealSenseCamera
- all
[[autodoc]] lerobot.cameras.realsense.RealSenseCameraConfig
## Reachy2Camera
Cameras exposed by a Reachy 2 robot's own camera manager.
[[autodoc]] lerobot.cameras.reachy2_camera.Reachy2Camera
- all
[[autodoc]] lerobot.cameras.reachy2_camera.Reachy2CameraConfig
## ZMQCamera
Reads frames published over a ZeroMQ socket by `ImageServer`, for cameras attached to a different machine
than the one running the policy.
[[autodoc]] lerobot.cameras.zmq.ZMQCamera
- all
[[autodoc]] lerobot.cameras.zmq.ZMQCameraConfig
-71
View File
@@ -1,71 +0,0 @@
# Image Transforms
Data-augmentation transforms applied to camera observations during training. [`~transforms.ImageTransforms`]
composes a random subset of them, configured via [`~transforms.ImageTransformsConfig`].
## ImageTransforms
[[autodoc]] lerobot.transforms.ImageTransforms
- all
## ImageTransformsConfig
[[autodoc]] lerobot.transforms.ImageTransformsConfig
## ImageTransformConfig
[[autodoc]] lerobot.transforms.ImageTransformConfig
## make_transform_from_config
[[autodoc]] lerobot.transforms.make_transform_from_config
## RandomSubsetApply
[[autodoc]] lerobot.transforms.RandomSubsetApply
- all
## SharpnessJitter
[[autodoc]] lerobot.transforms.SharpnessJitter
- all
## GaussianNoise
[[autodoc]] lerobot.transforms.GaussianNoise
- all
## MotionBlur
[[autodoc]] lerobot.transforms.MotionBlur
- all
## JPEGCompression
[[autodoc]] lerobot.transforms.JPEGCompression
- all
## GaussianPatchBrightness
[[autodoc]] lerobot.transforms.GaussianPatchBrightness
- all
## RandomShadow
[[autodoc]] lerobot.transforms.RandomShadow
- all
## CoarseDropout
[[autodoc]] lerobot.transforms.CoarseDropout
- all
## GammaCorrection
[[autodoc]] lerobot.transforms.GammaCorrection
- all
## PlanckianJitter
[[autodoc]] lerobot.transforms.PlanckianJitter
- all
+1 -1
View File
@@ -437,7 +437,6 @@ ignore = [
# Awaiting conversion, one PR per module. # Awaiting conversion, one PR per module.
"src/lerobot/annotations/**" = ["D"] "src/lerobot/annotations/**" = ["D"]
"src/lerobot/async_inference/**" = ["D"] "src/lerobot/async_inference/**" = ["D"]
"src/lerobot/cameras/**" = ["D"]
"src/lerobot/common/**" = ["D"] "src/lerobot/common/**" = ["D"]
"src/lerobot/configs/**" = ["D"] "src/lerobot/configs/**" = ["D"]
"src/lerobot/data_processing/**" = ["D"] "src/lerobot/data_processing/**" = ["D"]
@@ -455,6 +454,7 @@ ignore = [
"src/lerobot/rollout/**" = ["D"] "src/lerobot/rollout/**" = ["D"]
"src/lerobot/scripts/**" = ["D"] "src/lerobot/scripts/**" = ["D"]
"src/lerobot/teleoperators/**" = ["D"] "src/lerobot/teleoperators/**" = ["D"]
"src/lerobot/transforms/**" = ["D"]
"src/lerobot/transport/**" = ["D"] "src/lerobot/transport/**" = ["D"]
"src/lerobot/utils/**" = ["D"] "src/lerobot/utils/**" = ["D"]
"src/lerobot/lerobot_types.py" = ["D"] "src/lerobot/lerobot_types.py" = ["D"]
+7 -6
View File
@@ -50,23 +50,23 @@ class Camera(abc.ABC):
self.height: int | None = config.height self.height: int | None = config.height
def __enter__(self): def __enter__(self):
""" """Context manager entry.
Context manager entry.
Automatically connects to the camera. Automatically connects to the camera.
""" """
self.connect() self.connect()
return self return self
def __exit__(self, exc_type, exc_value, traceback) -> None: def __exit__(self, exc_type, exc_value, traceback) -> None:
""" """Context manager exit.
Context manager exit.
Automatically disconnects, ensuring resources are released even on error. Automatically disconnects, ensuring resources are released even on error.
""" """
self.disconnect() self.disconnect()
def __del__(self) -> None: def __del__(self) -> None:
""" """Destructor safety net.
Destructor safety net.
Attempts to disconnect if the object is garbage collected without cleanup. Attempts to disconnect if the object is garbage collected without cleanup.
""" """
try: try:
@@ -90,6 +90,7 @@ class Camera(abc.ABC):
@abc.abstractmethod @abc.abstractmethod
def find_cameras() -> list[dict[str, Any]]: def find_cameras() -> list[dict[str, Any]]:
"""Detects available cameras connected to the system. """Detects available cameras connected to the system.
Returns: Returns:
List[Dict[str, Any]]: A list of dictionaries, List[Dict[str, Any]]: A list of dictionaries,
where each dictionary contains information about a detected camera. where each dictionary contains information about a detected camera.
+66
View File
@@ -22,15 +22,36 @@ import draccus # type: ignore # TODO: add type stubs for draccus
class ColorMode(str, Enum): class ColorMode(str, Enum):
"""Color channel order for frames returned by a camera.
**Attributes**:
- **RGB** -- Red-green-blue channel order.
- **BGR** -- Blue-green-red channel order, OpenCV's native order.
"""
RGB = "rgb" RGB = "rgb"
BGR = "bgr" BGR = "bgr"
@classmethod @classmethod
def _missing_(cls, value: object) -> None: def _missing_(cls, value: object) -> None:
"""Reject a value that is not a valid `ColorMode`.
Raises:
ValueError: Always, naming the invalid value and the valid choices.
"""
raise ValueError(f"`color_mode` is expected to be in {list(cls)}, but {value} is provided.") raise ValueError(f"`color_mode` is expected to be in {list(cls)}, but {value} is provided.")
class Cv2Rotation(int, Enum): class Cv2Rotation(int, Enum):
"""Clockwise rotation to apply to a frame after capture, in degrees.
**Attributes**:
- **NO_ROTATION** -- No rotation.
- **ROTATE_90** -- Rotate 90° clockwise.
- **ROTATE_180** -- Rotate 180°.
- **ROTATE_270** -- Rotate 270° clockwise (90° counter-clockwise).
"""
NO_ROTATION = 0 NO_ROTATION = 0
ROTATE_90 = 90 ROTATE_90 = 90
ROTATE_180 = 180 ROTATE_180 = 180
@@ -38,11 +59,31 @@ class Cv2Rotation(int, Enum):
@classmethod @classmethod
def _missing_(cls, value: object) -> None: def _missing_(cls, value: object) -> None:
"""Reject a value that is not a valid `Cv2Rotation`.
Raises:
ValueError: Always, naming the invalid value and the valid choices.
"""
raise ValueError(f"`rotation` is expected to be in {list(cls)}, but {value} is provided.") raise ValueError(f"`rotation` is expected to be in {list(cls)}, but {value} is provided.")
# Subset from https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html # Subset from https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html
class Cv2Backends(int, Enum): class Cv2Backends(int, Enum):
"""OpenCV capture backend to request when opening a device.
See the [OpenCV `VideoCaptureAPIs` reference](https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html)
for the full list this is a subset of.
**Attributes**:
- **ANY** -- Let OpenCV auto-detect the backend.
- **V4L2** -- Video4Linux2, the usual choice on Linux.
- **DSHOW** -- DirectShow, a Windows backend.
- **PVAPI** -- PvAPI, for Prosilica GigE cameras.
- **ANDROID** -- Android's native camera API.
- **AVFOUNDATION** -- AVFoundation, the usual choice on macOS.
- **MSMF** -- Microsoft Media Foundation, a Windows backend.
"""
ANY = 0 ANY = 0
V4L2 = 200 V4L2 = 200
DSHOW = 700 DSHOW = 700
@@ -53,15 +94,40 @@ class Cv2Backends(int, Enum):
@classmethod @classmethod
def _missing_(cls, value: object) -> None: def _missing_(cls, value: object) -> None:
"""Reject a value that is not a valid `Cv2Backends`.
Raises:
ValueError: Always, naming the invalid value and the valid choices.
"""
raise ValueError(f"`backend` is expected to be in {list(cls)}, but {value} is provided.") raise ValueError(f"`backend` is expected to be in {list(cls)}, but {value} is provided.")
@dataclass(kw_only=True) @dataclass(kw_only=True)
class CameraConfig(draccus.ChoiceRegistry, abc.ABC): # type: ignore # TODO: add type stubs for draccus class CameraConfig(draccus.ChoiceRegistry, abc.ABC): # type: ignore # TODO: add type stubs for draccus
"""Base configuration shared by every camera backend.
Concrete backends subclass this and register themselves with
`@CameraConfig.register_subclass("name")`, which is what makes `--camera.type=name` work on the
command line. Subclasses inherit the three fields below and must document them alongside their own.
Args:
fps (`int`, *optional*):
Requested frames per second for the color stream. `None` leaves it at the backend's default.
width (`int`, *optional*):
Requested frame width in pixels. `None` leaves it at the backend's default.
height (`int`, *optional*):
Requested frame height in pixels. `None` leaves it at the backend's default.
"""
fps: int | None = None fps: int | None = None
width: int | None = None width: int | None = None
height: int | None = None height: int | None = None
@property @property
def type(self) -> str: def type(self) -> str:
"""Return the registered name this config was registered under.
Returns:
`str`: The name passed to `@CameraConfig.register_subclass`, e.g. `"opencv"`.
"""
return str(self.get_choice_name(self.__class__)) return str(self.get_choice_name(self.__class__))
+12 -32
View File
@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
""" """Provides the OpenCVCamera class for capturing frames from cameras using OpenCV."""
Provides the OpenCVCamera class for capturing frames from cameras using OpenCV.
"""
import logging import logging
import math import math
@@ -50,8 +48,7 @@ logger = logging.getLogger(__name__)
class OpenCVCamera(Camera): class OpenCVCamera(Camera):
""" """Manages camera interactions using OpenCV for efficient frame recording.
Manages camera interactions using OpenCV for efficient frame recording.
This class provides a high-level interface to connect to, configure, and read This class provides a high-level interface to connect to, configure, and read
frames from cameras compatible with OpenCV's VideoCapture. It supports both frames from cameras compatible with OpenCV's VideoCapture. It supports both
@@ -93,8 +90,7 @@ class OpenCVCamera(Camera):
""" """
def __init__(self, config: OpenCVCameraConfig): def __init__(self, config: OpenCVCameraConfig):
""" """Initializes the OpenCVCamera instance.
Initializes the OpenCVCamera instance.
Args: Args:
config: The configuration settings for the camera. config: The configuration settings for the camera.
@@ -125,6 +121,7 @@ class OpenCVCamera(Camera):
self._reset_connection_settings() self._reset_connection_settings()
def __str__(self) -> str: def __str__(self) -> str:
"""Return a short representation naming the class and its `index_or_path`."""
return f"{self.__class__.__name__}({self.index_or_path})" return f"{self.__class__.__name__}({self.index_or_path})"
def _reset_connection_settings(self) -> None: def _reset_connection_settings(self) -> None:
@@ -143,8 +140,7 @@ class OpenCVCamera(Camera):
@check_if_already_connected @check_if_already_connected
def connect(self, warmup: bool = True) -> None: def connect(self, warmup: bool = True) -> None:
""" """Connects to the OpenCV camera specified in the configuration.
Connects to the OpenCV camera specified in the configuration.
Initializes the OpenCV VideoCapture object, sets desired camera properties Initializes the OpenCV VideoCapture object, sets desired camera properties
(FPS, width, height), starts the background reading thread and performs initial checks. (FPS, width, height), starts the background reading thread and performs initial checks.
@@ -158,7 +154,6 @@ class OpenCVCamera(Camera):
ConnectionError: If the specified camera index/path is not found or fails to open. ConnectionError: If the specified camera index/path is not found or fails to open.
RuntimeError: If the camera opens but fails to apply requested settings. RuntimeError: If the camera opens but fails to apply requested settings.
""" """
# Use 1 thread for OpenCV operations to avoid potential conflicts or # Use 1 thread for OpenCV operations to avoid potential conflicts or
# blocking in multi-threaded applications, especially during data collection. # blocking in multi-threaded applications, especially during data collection.
cv2.setNumThreads(1) cv2.setNumThreads(1)
@@ -196,8 +191,7 @@ class OpenCVCamera(Camera):
@check_if_not_connected @check_if_not_connected
def _configure_capture_settings(self) -> None: def _configure_capture_settings(self) -> None:
""" """Applies the specified FOURCC, FPS, width, and height settings to the connected camera.
Applies the specified FOURCC, FPS, width, and height settings to the connected camera.
This method attempts to set the camera properties via OpenCV. It checks if This method attempts to set the camera properties via OpenCV. It checks if
the camera successfully applied the settings and raises an error if not. the camera successfully applied the settings and raises an error if not.
@@ -214,7 +208,6 @@ class OpenCVCamera(Camera):
to the requested value. to the requested value.
DeviceNotConnectedError: If the camera is not connected. DeviceNotConnectedError: If the camera is not connected.
""" """
if self.videocapture is None: if self.videocapture is None:
raise DeviceNotConnectedError(f"{self} videocapture is not initialized") raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
@@ -246,7 +239,6 @@ class OpenCVCamera(Camera):
def _validate_fps(self) -> None: def _validate_fps(self) -> None:
"""Validates and sets the camera's frames per second (FPS).""" """Validates and sets the camera's frames per second (FPS)."""
if self.videocapture is None: if self.videocapture is None:
raise DeviceNotConnectedError(f"{self} videocapture is not initialized") raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
@@ -261,7 +253,6 @@ class OpenCVCamera(Camera):
def _validate_fourcc(self) -> None: def _validate_fourcc(self) -> None:
"""Validates and sets the camera's FOURCC code.""" """Validates and sets the camera's FOURCC code."""
fourcc_code = cv2.VideoWriter_fourcc(*self.config.fourcc) fourcc_code = cv2.VideoWriter_fourcc(*self.config.fourcc)
if self.videocapture is None: if self.videocapture is None:
@@ -282,7 +273,6 @@ class OpenCVCamera(Camera):
def _validate_width_and_height(self) -> None: def _validate_width_and_height(self) -> None:
"""Validates and sets the camera's frame capture width and height.""" """Validates and sets the camera's frame capture width and height."""
if self.videocapture is None: if self.videocapture is None:
raise DeviceNotConnectedError(f"{self} videocapture is not initialized") raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
@@ -306,8 +296,7 @@ class OpenCVCamera(Camera):
@staticmethod @staticmethod
def find_cameras() -> list[dict[str, Any]]: def find_cameras() -> list[dict[str, Any]]:
""" """Detects available OpenCV cameras connected to the system.
Detects available OpenCV cameras connected to the system.
On Linux, it scans '/dev/video*' paths. On other systems (like macOS, Windows), On Linux, it scans '/dev/video*' paths. On other systems (like macOS, Windows),
it checks indices from 0 up to `MAX_OPENCV_INDEX`. it checks indices from 0 up to `MAX_OPENCV_INDEX`.
@@ -375,8 +364,7 @@ class OpenCVCamera(Camera):
@check_if_not_connected @check_if_not_connected
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]: def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
""" """Reads a single frame synchronously from the camera.
Reads a single frame synchronously from the camera.
This is a blocking call. It waits for the next available frame from the This is a blocking call. It waits for the next available frame from the
camera hardware via OpenCV. camera hardware via OpenCV.
@@ -392,7 +380,6 @@ class OpenCVCamera(Camera):
received frame dimensions don't match expectations before rotation. received frame dimensions don't match expectations before rotation.
ValueError: If an invalid `color_mode` is requested. ValueError: If an invalid `color_mode` is requested.
""" """
start_time = time.perf_counter() start_time = time.perf_counter()
if color_mode is not None: if color_mode is not None:
@@ -412,8 +399,7 @@ class OpenCVCamera(Camera):
return frame return frame
def _postprocess_image(self, image: NDArray[Any]) -> NDArray[Any]: def _postprocess_image(self, image: NDArray[Any]) -> NDArray[Any]:
""" """Applies color conversion, dimension validation, and rotation to a raw frame.
Applies color conversion, dimension validation, and rotation to a raw frame.
Args: Args:
image (np.ndarray): The raw image frame (expected BGR format from OpenCV). image (np.ndarray): The raw image frame (expected BGR format from OpenCV).
@@ -426,7 +412,6 @@ class OpenCVCamera(Camera):
RuntimeError: If the raw frame dimensions do not match the configured RuntimeError: If the raw frame dimensions do not match the configured
`width` and `height`. `width` and `height`.
""" """
if self.color_mode not in (ColorMode.RGB, ColorMode.BGR): if self.color_mode not in (ColorMode.RGB, ColorMode.BGR):
raise ValueError( raise ValueError(
f"Invalid color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}." f"Invalid color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}."
@@ -452,8 +437,7 @@ class OpenCVCamera(Camera):
return processed_image return processed_image
def _read_loop(self) -> None: def _read_loop(self) -> None:
""" """Internal loop run by the background thread for asynchronous reading.
Internal loop run by the background thread for asynchronous reading.
On each iteration: On each iteration:
1. Reads a color frame (blocking call) 1. Reads a color frame (blocking call)
@@ -538,8 +522,7 @@ class OpenCVCamera(Camera):
@check_if_not_connected @check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]: def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
""" """Reads the latest available frame asynchronously.
Reads the latest available frame asynchronously.
This method retrieves the most recent frame captured by the background This method retrieves the most recent frame captured by the background
read thread. It does not block waiting for the camera hardware directly, read thread. It does not block waiting for the camera hardware directly,
@@ -559,7 +542,6 @@ class OpenCVCamera(Camera):
TimeoutError: If no frame becomes available within the specified timeout. TimeoutError: If no frame becomes available within the specified timeout.
RuntimeError: If an unexpected error occurs. RuntimeError: If an unexpected error occurs.
""" """
if self.thread is None or not self.thread.is_alive(): if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.") raise RuntimeError(f"{self} read thread is not running.")
@@ -594,7 +576,6 @@ class OpenCVCamera(Camera):
DeviceNotConnectedError: If the camera is not connected. DeviceNotConnectedError: If the camera is not connected.
RuntimeError: If the camera is connected but has not captured any frames yet. RuntimeError: If the camera is connected but has not captured any frames yet.
""" """
if self.thread is None or not self.thread.is_alive(): if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.") raise RuntimeError(f"{self} read thread is not running.")
@@ -614,8 +595,7 @@ class OpenCVCamera(Camera):
return frame return frame
def disconnect(self) -> None: def disconnect(self) -> None:
""" """Disconnects from the camera and cleans up resources.
Disconnects from the camera and cleans up resources.
Stops the background read thread (if running) and releases the OpenCV Stops the background read thread (if running) and releases the OpenCV
VideoCapture object. VideoCapture object.
@@ -40,20 +40,28 @@ class OpenCVCameraConfig(CameraConfig):
OpenCVCameraConfig(0, 30, 1280, 720, fourcc="YUYV") # With YUYV format OpenCVCameraConfig(0, 30, 1280, 720, fourcc="YUYV") # With YUYV format
``` ```
**Attributes**: Args:
- **index_or_path** (`int | Path`) -- Either an integer representing the camera device index, or a index_or_path (`int | Path`):
Path object pointing to a video file. Either an integer representing the camera device index, or a Path object pointing to a video
- **fps** -- Requested frames per second for the color stream. file.
- **width** -- Requested frame width in pixels for the color stream. color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
- **height** -- Requested frame height in pixels for the color stream. Color mode for image output.
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB. rotation (`Cv2Rotation`, *optional*, defaults to `Cv2Rotation.NO_ROTATION`):
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no Image rotation setting (0°, 90°, 180°, or 270°).
rotation. warmup_s (`int`, *optional*, defaults to 1):
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds) Time reading frames before returning from connect (in seconds).
- **fourcc** (`str | None`) -- FOURCC code for video format (e.g., "MJPG", "YUYV", "I420"). Defaults fourcc (`str`, *optional*):
to None (auto-detect). FOURCC code for video format (e.g., `"MJPG"`, `"YUYV"`, `"I420"`). `None` auto-detects.
- **backend** (`Cv2Backends`) -- OpenCV backend identifier backend (`Cv2Backends`, *optional*, defaults to `Cv2Backends.ANY`):
(https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html). Defaults to ANY. OpenCV backend identifier. See [`~cameras.Cv2Backends`] for the supported values.
fps (`int`, *optional*):
Requested frames per second for the color stream. `None` leaves it at the backend's default.
width (`int`, *optional*):
Requested frame width in pixels for the color stream. `None` leaves it at the backend's
default.
height (`int`, *optional*):
Requested frame height in pixels for the color stream. `None` leaves it at the backend's
default.
Note: Note:
- Only 3-channel color output (RGB/BGR) is currently supported. - Only 3-channel color output (RGB/BGR) is currently supported.
@@ -69,6 +77,11 @@ class OpenCVCameraConfig(CameraConfig):
backend: Cv2Backends = Cv2Backends.ANY backend: Cv2Backends = Cv2Backends.ANY
def __post_init__(self) -> None: def __post_init__(self) -> None:
"""Normalize `color_mode`, `rotation`, and `backend`, and validate `fourcc`.
Raises:
ValueError: If `fourcc` is set and is not a 4-character string.
"""
self.color_mode = ColorMode(self.color_mode) self.color_mode = ColorMode(self.color_mode)
self.rotation = Cv2Rotation(self.rotation) self.rotation = Cv2Rotation(self.rotation)
self.backend = Cv2Backends(self.backend) self.backend = Cv2Backends(self.backend)
@@ -43,16 +43,24 @@ class Reachy2CameraConfig(CameraConfig):
) # Left teleop camera, 640x480 @ 30FPS ) # Left teleop camera, 640x480 @ 30FPS
``` ```
**Attributes**: Args:
- **name** (`str`) -- Name of the camera device. Can be "teleop" or "depth". name (`str`):
- **image_type** (`str`) -- Type of image stream. For "teleop" camera, can be "left" or "right". For Name of the camera device. Either `"teleop"` or `"depth"`.
"depth" camera, can be "rgb" or "depth". (depth is not supported yet) image_type (`str`):
- **fps** -- Requested frames per second for the color stream. Not configurable for Reachy 2 cameras. Type of image stream. For the `"teleop"` camera, either `"left"` or `"right"`. For the
- **width** -- Requested frame width in pixels for the color stream. `"depth"` camera, either `"rgb"` or `"depth"` (depth is not supported yet).
- **height** -- Requested frame height in pixels for the color stream. color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB. Color mode for image output.
- **ip_address** (`str | None`) -- IP address of the robot. Defaults to "localhost". ip_address (`str`, *optional*, defaults to `"localhost"`):
- **port** (`int`) -- Port number for the camera server. Defaults to 50065. IP address of the robot.
port (`int`, *optional*, defaults to 50065):
Port number for the camera server.
fps (`int`, *optional*):
Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
width (`int`, *optional*):
Requested frame width in pixels for the color stream.
height (`int`, *optional*):
Requested frame height in pixels for the color stream.
Note: Note:
- Only 3-channel color output (RGB/BGR) is currently supported. - Only 3-channel color output (RGB/BGR) is currently supported.
@@ -65,6 +73,11 @@ class Reachy2CameraConfig(CameraConfig):
port: int = 50065 port: int = 50065
def __post_init__(self) -> None: def __post_init__(self) -> None:
"""Normalize `color_mode` and validate `name`/`image_type`.
Raises:
ValueError: If `name` is not `"teleop"`/`"depth"`, or `image_type` is not valid for `name`.
"""
if self.name not in ["teleop", "depth"]: if self.name not in ["teleop", "depth"]:
raise ValueError(f"`name` is expected to be 'teleop' or 'depth', but {self.name} is provided.") raise ValueError(f"`name` is expected to be 'teleop' or 'depth', but {self.name} is provided.")
if (self.name == "teleop" and self.image_type not in ["left", "right"]) or ( if (self.name == "teleop" and self.image_type not in ["left", "right"]) or (
@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
""" """Provides the Reachy2Camera class for capturing frames from Reachy 2 cameras using Reachy 2's CameraManager."""
Provides the Reachy2Camera class for capturing frames from Reachy 2 cameras using Reachy 2's CameraManager.
"""
from __future__ import annotations from __future__ import annotations
@@ -42,6 +40,8 @@ else:
CameraManager = None CameraManager = None
class CameraView: class CameraView:
"""Fallback stand-in for `reachy2_sdk`'s `CameraView` when the SDK is not installed."""
LEFT = 0 LEFT = 0
RIGHT = 1 RIGHT = 1
@@ -55,8 +55,7 @@ logger = logging.getLogger(__name__)
class Reachy2Camera(Camera): class Reachy2Camera(Camera):
""" """Manages Reachy 2 camera using Reachy 2 CameraManager.
Manages Reachy 2 camera using Reachy 2 CameraManager.
This class provides a high-level interface to connect to, configure, and read This class provides a high-level interface to connect to, configure, and read
frames from Reachy 2 cameras. It supports both synchronous and asynchronous frames from Reachy 2 cameras. It supports both synchronous and asynchronous
@@ -70,8 +69,7 @@ class Reachy2Camera(Camera):
""" """
def __init__(self, config: Reachy2CameraConfig): def __init__(self, config: Reachy2CameraConfig):
""" """Initializes the Reachy2Camera instance.
Initializes the Reachy2Camera instance.
Args: Args:
config: The configuration settings for the camera. config: The configuration settings for the camera.
@@ -88,6 +86,7 @@ class Reachy2Camera(Camera):
self.cam_manager: CameraManager | None = None self.cam_manager: CameraManager | None = None
def __str__(self) -> str: def __str__(self) -> str:
"""Return a short representation naming the class, camera name, and image type."""
return f"{self.__class__.__name__}({self.config.name}, {self.config.image_type})" return f"{self.__class__.__name__}({self.config.name}, {self.config.image_type})"
@property @property
@@ -105,8 +104,7 @@ class Reachy2Camera(Camera):
raise ValueError(f"Invalid camera name '{self.config.name}'. Expected 'teleop' or 'depth'.") raise ValueError(f"Invalid camera name '{self.config.name}'. Expected 'teleop' or 'depth'.")
def connect(self, warmup: bool = True) -> None: def connect(self, warmup: bool = True) -> None:
""" """Connects to the Reachy2 CameraManager as specified in the configuration.
Connects to the Reachy2 CameraManager as specified in the configuration.
Raises: Raises:
DeviceNotConnectedError: If the camera is not connected. DeviceNotConnectedError: If the camera is not connected.
@@ -120,15 +118,12 @@ class Reachy2Camera(Camera):
@staticmethod @staticmethod
def find_cameras() -> list[dict[str, Any]]: def find_cameras() -> list[dict[str, Any]]:
""" """Detection not implemented for Reachy2 cameras."""
Detection not implemented for Reachy2 cameras.
"""
raise NotImplementedError("Camera detection is not implemented for Reachy2 cameras.") raise NotImplementedError("Camera detection is not implemented for Reachy2 cameras.")
@check_if_not_connected @check_if_not_connected
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]: def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
""" """Reads a single frame synchronously from the camera.
Reads a single frame synchronously from the camera.
This method retrieves the most recent frame available in Reachy 2's low-level software. This method retrieves the most recent frame available in Reachy 2's low-level software.
@@ -187,8 +182,7 @@ class Reachy2Camera(Camera):
@check_if_not_connected @check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]: def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
""" """Same as [`~cameras.reachy2_camera.Reachy2Camera.read`]; Reachy 2 has no separate async path.
Same as read()
Returns: Returns:
np.ndarray: The latest captured frame as a NumPy array in the format np.ndarray: The latest captured frame as a NumPy array in the format
@@ -199,7 +193,6 @@ class Reachy2Camera(Camera):
TimeoutError: If no frame becomes available within the specified timeout. TimeoutError: If no frame becomes available within the specified timeout.
RuntimeError: If an unexpected error occurs. RuntimeError: If an unexpected error occurs.
""" """
return self.read() return self.read()
@check_if_not_connected @check_if_not_connected
@@ -220,7 +213,6 @@ class Reachy2Camera(Camera):
DeviceNotConnectedError: If the camera is not connected. DeviceNotConnectedError: If the camera is not connected.
RuntimeError: If the camera is connected but has not captured any frames yet. RuntimeError: If the camera is connected but has not captured any frames yet.
""" """
if self.latest_frame is None or self.latest_timestamp is None: if self.latest_frame is None or self.latest_timestamp is None:
raise RuntimeError(f"{self} has not captured any frames yet.") raise RuntimeError(f"{self} has not captured any frames yet.")
@@ -234,13 +226,11 @@ class Reachy2Camera(Camera):
@check_if_not_connected @check_if_not_connected
def disconnect(self) -> None: def disconnect(self) -> None:
""" """Stops the background read thread (if running).
Stops the background read thread (if running).
Raises: Raises:
DeviceNotConnectedError: If the camera is already disconnected. DeviceNotConnectedError: If the camera is already disconnected.
""" """
if self.cam_manager is not None: if self.cam_manager is not None:
self.cam_manager.disconnect() self.cam_manager.disconnect()
@@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
""" """Provides the RealSenseCamera class for capturing frames from Intel RealSense cameras."""
Provides the RealSenseCamera class for capturing frames from Intel RealSense cameras.
"""
import logging import logging
import sys import sys
@@ -46,8 +44,7 @@ pkg_name = "pyrealsense2-macosx" if sys.platform == "darwin" else "pyrealsense2"
class RealSenseCamera(Camera): class RealSenseCamera(Camera):
""" """Manages interactions with Intel RealSense cameras for frame and depth recording.
Manages interactions with Intel RealSense cameras for frame and depth recording.
This class provides an interface similar to `OpenCVCamera` but tailored for This class provides an interface similar to `OpenCVCamera` but tailored for
RealSense devices, leveraging the `pyrealsense2` library. It uses the camera's RealSense devices, leveraging the `pyrealsense2` library. It uses the camera's
@@ -115,8 +112,7 @@ class RealSenseCamera(Camera):
_MAX_CONNECT_ATTEMPTS = 3 _MAX_CONNECT_ATTEMPTS = 3
def __init__(self, config: RealSenseCameraConfig): def __init__(self, config: RealSenseCameraConfig):
""" """Initializes the RealSenseCamera instance.
Initializes the RealSenseCamera instance.
Args: Args:
config: The configuration settings for the camera. config: The configuration settings for the camera.
@@ -161,6 +157,7 @@ class RealSenseCamera(Camera):
self._reset_connection_settings() self._reset_connection_settings()
def __str__(self) -> str: def __str__(self) -> str:
"""Return a short representation naming the class and its `serial_number`."""
return f"{self.__class__.__name__}({self.serial_number})" return f"{self.__class__.__name__}({self.serial_number})"
def _reset_connection_settings(self) -> None: def _reset_connection_settings(self) -> None:
@@ -250,8 +247,7 @@ class RealSenseCamera(Camera):
@check_if_already_connected @check_if_already_connected
def connect(self, warmup: bool = True) -> None: def connect(self, warmup: bool = True) -> None:
""" """Connects to the RealSense camera specified in the configuration.
Connects to the RealSense camera specified in the configuration.
Initializes the RealSense pipeline, configures the required streams (color Initializes the RealSense pipeline, configures the required streams (color
and optionally depth), starts the pipeline, and validates the actual stream settings. and optionally depth), starts the pipeline, and validates the actual stream settings.
@@ -270,7 +266,6 @@ class RealSenseCamera(Camera):
ConnectionError: If the camera is found but fails to start the pipeline or no RealSense devices are detected at all. ConnectionError: If the camera is found but fails to start the pipeline or no RealSense devices are detected at all.
RuntimeError: If the pipeline starts but fails to apply requested settings. RuntimeError: If the pipeline starts but fails to apply requested settings.
""" """
if not warmup: if not warmup:
self._open_pipeline() self._open_pipeline()
logger.info(f"{self} connected.") logger.info(f"{self} connected.")
@@ -306,8 +301,7 @@ class RealSenseCamera(Camera):
@staticmethod @staticmethod
def find_cameras() -> list[dict[str, Any]]: def find_cameras() -> list[dict[str, Any]]:
""" """Detects available Intel RealSense cameras connected to the system.
Detects available Intel RealSense cameras connected to the system.
Returns: Returns:
List[Dict[str, Any]]: A list of dictionaries, List[Dict[str, Any]]: A list of dictionaries,
@@ -406,7 +400,6 @@ class RealSenseCamera(Camera):
Raises: Raises:
DeviceNotConnectedError: If device is not connected. DeviceNotConnectedError: If device is not connected.
""" """
if self.rs_profile is None: if self.rs_profile is None:
raise RuntimeError(f"{self}: rs_profile must be initialized before use.") raise RuntimeError(f"{self}: rs_profile must be initialized before use.")
@@ -544,8 +537,7 @@ class RealSenseCamera(Camera):
@check_if_not_connected @check_if_not_connected
def read_depth(self, timeout_ms: int = 200) -> NDArray[Any]: def read_depth(self, timeout_ms: int = 200) -> NDArray[Any]:
""" """Reads a single frame (depth) synchronously from the camera.
Reads a single frame (depth) synchronously from the camera.
This is a blocking call. It waits for a coherent set of frames (depth) This is a blocking call. It waits for a coherent set of frames (depth)
from the camera hardware via the RealSense pipeline. from the camera hardware via the RealSense pipeline.
@@ -583,8 +575,7 @@ class RealSenseCamera(Camera):
@check_if_not_connected @check_if_not_connected
def read(self, color_mode: ColorMode | None = None, timeout_ms: int = 0) -> NDArray[Any]: def read(self, color_mode: ColorMode | None = None, timeout_ms: int = 0) -> NDArray[Any]:
""" """Reads a single frame (color) synchronously from the camera.
Reads a single frame (color) synchronously from the camera.
This is a blocking call. It waits for a coherent set of frames (color) This is a blocking call. It waits for a coherent set of frames (color)
from the camera hardware via the RealSense pipeline. from the camera hardware via the RealSense pipeline.
@@ -598,7 +589,6 @@ class RealSenseCamera(Camera):
RuntimeError: If reading frames from the pipeline fails or frames are invalid. RuntimeError: If reading frames from the pipeline fails or frames are invalid.
ValueError: If an invalid `color_mode` is requested. ValueError: If an invalid `color_mode` is requested.
""" """
start_time = time.perf_counter() start_time = time.perf_counter()
if color_mode is not None: if color_mode is not None:
@@ -622,11 +612,12 @@ class RealSenseCamera(Camera):
return frame return frame
def _postprocess_image(self, image: NDArray[Any], depth_frame: bool = False) -> NDArray[Any]: def _postprocess_image(self, image: NDArray[Any], depth_frame: bool = False) -> NDArray[Any]:
""" """Applies color conversion, dimension validation, and rotation to a raw color frame.
Applies color conversion, dimension validation, and rotation to a raw color frame.
Args: Args:
image (np.ndarray): The raw image frame (expected RGB format from RealSense). image (np.ndarray): The raw image frame (expected RGB format from RealSense).
depth_frame (bool): Whether `image` is a single-channel depth frame rather than a 3-channel
color frame.
Returns: Returns:
np.ndarray: The processed image frame according to `self.color_mode` and `self.rotation`. np.ndarray: The processed image frame according to `self.color_mode` and `self.rotation`.
@@ -636,7 +627,6 @@ class RealSenseCamera(Camera):
RuntimeError: If the raw frame dimensions do not match the configured RuntimeError: If the raw frame dimensions do not match the configured
`width` and `height`. `width` and `height`.
""" """
if self.color_mode and self.color_mode not in (ColorMode.RGB, ColorMode.BGR): if self.color_mode and self.color_mode not in (ColorMode.RGB, ColorMode.BGR):
raise ValueError( raise ValueError(
f"Invalid requested color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}." f"Invalid requested color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}."
@@ -665,8 +655,7 @@ class RealSenseCamera(Camera):
return processed_image return processed_image
def _read_loop(self) -> None: def _read_loop(self) -> None:
""" """Internal loop run by the background thread for asynchronous reading.
Internal loop run by the background thread for asynchronous reading.
On each iteration: On each iteration:
1. Reads a color/depth frame (blocking call with 10s timeout) 1. Reads a color/depth frame (blocking call with 10s timeout)
@@ -790,8 +779,7 @@ class RealSenseCamera(Camera):
@check_if_not_connected @check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]: def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
""" """Reads the latest available frame data (color) asynchronously.
Reads the latest available frame data (color) asynchronously.
This method retrieves the most recent color frame captured by the background This method retrieves the most recent color frame captured by the background
read thread. It does not block waiting for the camera hardware directly, read thread. It does not block waiting for the camera hardware directly,
@@ -811,7 +799,6 @@ class RealSenseCamera(Camera):
TimeoutError: If no frame data becomes available within the specified timeout. TimeoutError: If no frame data becomes available within the specified timeout.
RuntimeError: If the background thread died unexpectedly or another error occurs. RuntimeError: If the background thread died unexpectedly or another error occurs.
""" """
if not self.use_rgb: if not self.use_rgb:
raise RuntimeError(f"{self}: cannot read color — camera was configured with use_rgb=False.") raise RuntimeError(f"{self}: cannot read color — camera was configured with use_rgb=False.")
@@ -897,15 +884,13 @@ class RealSenseCamera(Camera):
return self._read_latest(max_age_ms=max_age_ms, read_depth=True) return self._read_latest(max_age_ms=max_age_ms, read_depth=True)
def disconnect(self) -> None: def disconnect(self) -> None:
""" """Disconnects from the camera, stops the pipeline, and cleans up resources.
Disconnects from the camera, stops the pipeline, and cleans up resources.
Stops the background read thread (if running) and stops the RealSense pipeline. Stops the background read thread (if running) and stops the RealSense pipeline.
Raises: Raises:
DeviceNotConnectedError: If the camera is already disconnected (pipeline not running). DeviceNotConnectedError: If the camera is already disconnected (pipeline not running).
""" """
if not self.is_connected and self.thread is None: if not self.is_connected and self.thread is None:
raise DeviceNotConnectedError( raise DeviceNotConnectedError(
f"Attempted to disconnect {self}, but it appears already disconnected." f"Attempted to disconnect {self}, but it appears already disconnected."
@@ -36,28 +36,39 @@ class RealSenseCameraConfig(CameraConfig):
RealSenseCameraConfig("0123456789", 30, 640, 480, rotation=Cv2Rotation.ROTATE_90) # With 90° rotation RealSenseCameraConfig("0123456789", 30, 640, 480, rotation=Cv2Rotation.ROTATE_90) # With 90° rotation
``` ```
**Attributes**: Args:
- **fps** -- Requested frames per second for the color stream. serial_number_or_name (`str`):
- **width** -- Requested frame width in pixels for the color stream. Unique serial number or human-readable name to identify the camera.
- **height** -- Requested frame height in pixels for the color stream. color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
- **serial_number_or_name** (`str`) -- Unique serial number or human-readable name to identify the Color mode for image output.
camera. use_rgb (`bool`, *optional*, defaults to `True`):
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB. Whether to enable the color stream.
- **use_rgb** (`bool`) -- Whether to enable the color stream. Defaults to True. use_depth (`bool`, *optional*, defaults to `False`):
- **use_depth** (`bool`) -- Whether to enable depth stream. Defaults to False. Whether to enable the depth stream.
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no rotation (`Cv2Rotation`, *optional*, defaults to `Cv2Rotation.NO_ROTATION`):
rotation. Image rotation setting (0°, 90°, 180°, or 270°).
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds) warmup_s (`int`, *optional*, defaults to 1):
- **exposure** (`int | None`) -- Manual exposure value for the color sensor. When set, auto-exposure Time reading frames before returning from connect (in seconds).
is disabled and this fixed value is used. Valid ranges are camera-model specific and reported if the exposure (`int`, *optional*):
value is rejected. Defaults to None (leave unchanged). Manual exposure value for the color sensor. When set, auto-exposure is disabled and this fixed
- **gain** (`int | None`) -- Manual gain value for the color sensor. When set, auto-exposure is value is used. Valid ranges are camera-model specific and reported if the value is rejected.
disabled and this fixed gain is used, which also freezes exposure at its current value when no `None` leaves auto-exposure unchanged.
exposure is configured. Valid ranges are camera-model specific and reported if the value is gain (`int`, *optional*):
rejected. Defaults to None (leave unchanged). Manual gain value for the color sensor. When set, auto-exposure is disabled and this fixed
- **white_balance** (`int | None`) -- Manual white balance value for the color sensor. When set, auto gain is used, which also freezes exposure at its current value when no exposure is configured.
white balance is disabled and this fixed value is used. Valid ranges are camera-model specific and Valid ranges are camera-model specific and reported if the value is rejected. `None` leaves it
reported if the value is rejected. Defaults to None (leave unchanged). unchanged.
white_balance (`int`, *optional*):
Manual white balance value for the color sensor. When set, auto white balance is disabled and
this fixed value is used. Valid ranges are camera-model specific and reported if the value is
rejected. `None` leaves it unchanged.
fps (`int`, *optional*):
Requested frames per second for the color stream. The depth stream, when enabled, uses the
same FPS. Either all of `fps`, `width`, and `height` must be set, or none of them.
width (`int`, *optional*):
Requested frame width in pixels for the color stream.
height (`int`, *optional*):
Requested frame height in pixels for the color stream.
Note: Note:
- Either name or serial_number must be specified. - Either name or serial_number must be specified.
@@ -78,6 +89,12 @@ class RealSenseCameraConfig(CameraConfig):
white_balance: int | None = None white_balance: int | None = None
def __post_init__(self) -> None: def __post_init__(self) -> None:
"""Normalize `color_mode`/`rotation` and validate the stream and sensor-option settings.
Raises:
ValueError: If neither `use_rgb` nor `use_depth` is enabled, if a manual color sensor option
is set without `use_rgb=True`, or if only some of `fps`/`width`/`height` are set.
"""
self.color_mode = ColorMode(self.color_mode) self.color_mode = ColorMode(self.color_mode)
self.rotation = Cv2Rotation(self.rotation) self.rotation = Cv2Rotation(self.rotation)
+36
View File
@@ -23,6 +23,33 @@ from .configs import CameraConfig, Cv2Rotation
def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[str, Camera]: def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[str, Camera]:
"""Instantiate one [`~cameras.Camera`] per entry in a mapping of configs.
Dispatches on each config's registered `type` to build the matching backend class. This only
constructs the camera objects; call [`~cameras.Camera.connect`] on each before use.
Args:
camera_configs (`dict[str, CameraConfig]`):
Camera configs keyed by the name each camera should be identified by, e.g. in a robot's
observation features.
Returns:
`dict[str, Camera]`: A camera instance per key, in the same order as `camera_configs`.
Raises:
ValueError: If a config's type is not a known backend and building it via the generic device
factory also fails.
Example:
```python
>>> from lerobot.cameras.opencv import OpenCVCameraConfig
>>> from lerobot.cameras.utils import make_cameras_from_configs
>>> configs = {"top": OpenCVCameraConfig(index_or_path=0, fps=30, width=640, height=480)}
>>> cameras = make_cameras_from_configs(configs)
>>> list(cameras.keys())
['top']
```
"""
cameras: dict[str, Camera] = {} cameras: dict[str, Camera] = {}
for key, cfg in camera_configs.items(): for key, cfg in camera_configs.items():
@@ -57,6 +84,15 @@ def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[s
def get_cv2_rotation(rotation: Cv2Rotation) -> int | None: def get_cv2_rotation(rotation: Cv2Rotation) -> int | None:
"""Map a [`~cameras.Cv2Rotation`] to the OpenCV rotation flag `cv2.rotate` expects.
Args:
rotation (`Cv2Rotation`):
The configured rotation.
Returns:
`int | None`: The matching `cv2.ROTATE_*` constant, or `None` for [`~cameras.Cv2Rotation.NO_ROTATION`].
"""
import cv2 # type: ignore # TODO: add type stubs for OpenCV import cv2 # type: ignore # TODO: add type stubs for OpenCV
if rotation == Cv2Rotation.ROTATE_90: if rotation == Cv2Rotation.ROTATE_90:
+15 -21
View File
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
""" """Captures frames from remote cameras via ZeroMQ.
ZMQCamera - Captures frames from remote cameras via ZeroMQ using JSON protocol in the
following format: Uses a JSON protocol of the following form:
{ {
"timestamps": {"camera_name": float}, "timestamps": {"camera_name": float},
"images": {"camera_name": "<base64-jpeg>"} "images": {"camera_name": "<base64-jpeg>"}
@@ -52,8 +52,7 @@ logger = logging.getLogger(__name__)
class ZMQCamera(Camera): class ZMQCamera(Camera):
""" """Manages camera interactions via ZeroMQ for receiving frames from a remote server.
Manages camera interactions via ZeroMQ for receiving frames from a remote server.
This class connects to a ZMQ Publisher, subscribes to frame topics, and decodes This class connects to a ZMQ Publisher, subscribes to frame topics, and decodes
incoming JSON messages containing Base64 encoded images. It supports both incoming JSON messages containing Base64 encoded images. It supports both
@@ -81,6 +80,11 @@ class ZMQCamera(Camera):
""" """
def __init__(self, config: ZMQCameraConfig): def __init__(self, config: ZMQCameraConfig):
"""Initialize the camera with the given configuration.
Args:
config: Camera configuration, including the image server's address and port.
"""
require_package("pyzmq", extra="pyzmq-dep", import_name="zmq") require_package("pyzmq", extra="pyzmq-dep", import_name="zmq")
super().__init__(config) super().__init__(config)
@@ -105,6 +109,7 @@ class ZMQCamera(Camera):
self.new_frame_event: Event = Event() self.new_frame_event: Event = Event()
def __str__(self) -> str: def __str__(self) -> str:
"""Return a short representation naming the camera and its server address."""
return f"ZMQCamera({self.camera_name}@{self.server_address}:{self.port})" return f"ZMQCamera({self.camera_name}@{self.server_address}:{self.port})"
@property @property
@@ -120,7 +125,6 @@ class ZMQCamera(Camera):
warmup (bool): If True, waits for the camera to provide at least one warmup (bool): If True, waits for the camera to provide at least one
valid frame before returning. Defaults to True. valid frame before returning. Defaults to True.
""" """
logger.info(f"Connecting to {self}...") logger.info(f"Connecting to {self}...")
try: try:
@@ -171,15 +175,11 @@ class ZMQCamera(Camera):
@staticmethod @staticmethod
def find_cameras() -> list[dict[str, Any]]: def find_cameras() -> list[dict[str, Any]]:
""" """Detection not implemented for ZMQ cameras; they require manual server address/port configuration."""
Detection not implemented for ZMQ cameras. These cameras require manual configuration (server address/port).
"""
raise NotImplementedError("Camera detection is not implemented for ZMQ cameras.") raise NotImplementedError("Camera detection is not implemented for ZMQ cameras.")
def _read_from_hardware(self) -> NDArray[Any]: def _read_from_hardware(self) -> NDArray[Any]:
""" """Reads a single frame directly from the ZMQ socket."""
Reads a single frame directly from the ZMQ socket.
"""
if not self.is_connected or self.socket is None: if not self.is_connected or self.socket is None:
raise DeviceNotConnectedError(f"{self} is not connected.") raise DeviceNotConnectedError(f"{self} is not connected.")
@@ -215,8 +215,7 @@ class ZMQCamera(Camera):
@check_if_not_connected @check_if_not_connected
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]: def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
""" """Reads a single frame synchronously from the camera.
Reads a single frame synchronously from the camera.
This is a blocking call. It waits for the next available frame from the This is a blocking call. It waits for the next available frame from the
camera background thread. camera background thread.
@@ -243,9 +242,7 @@ class ZMQCamera(Camera):
return frame return frame
def _read_loop(self) -> None: def _read_loop(self) -> None:
""" """Internal loop run by the background thread for asynchronous reading."""
Internal loop run by the background thread for asynchronous reading.
"""
stop_event = self.stop_event stop_event = self.stop_event
if stop_event is None: if stop_event is None:
raise RuntimeError(f"{self}: stop_event is not initialized.") raise RuntimeError(f"{self}: stop_event is not initialized.")
@@ -306,8 +303,7 @@ class ZMQCamera(Camera):
@check_if_not_connected @check_if_not_connected
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]: def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
""" """Reads the latest available frame asynchronously.
Reads the latest available frame asynchronously.
Args: Args:
timeout_ms (float): Maximum time in milliseconds to wait for a frame timeout_ms (float): Maximum time in milliseconds to wait for a frame
@@ -321,7 +317,6 @@ class ZMQCamera(Camera):
TimeoutError: If no frame data becomes available within the specified timeout. TimeoutError: If no frame data becomes available within the specified timeout.
RuntimeError: If the background thread is not running. RuntimeError: If the background thread is not running.
""" """
if self.thread is None or not self.thread.is_alive(): if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.") raise RuntimeError(f"{self} read thread is not running.")
@@ -353,7 +348,6 @@ class ZMQCamera(Camera):
DeviceNotConnectedError: If the camera is not connected. DeviceNotConnectedError: If the camera is not connected.
RuntimeError: If the camera is connected but has not captured any frames yet. RuntimeError: If the camera is connected but has not captured any frames yet.
""" """
if self.thread is None or not self.thread.is_alive(): if self.thread is None or not self.thread.is_alive():
raise RuntimeError(f"{self} read thread is not running.") raise RuntimeError(f"{self} read thread is not running.")
@@ -24,6 +24,32 @@ __all__ = ["ZMQCameraConfig", "ColorMode"]
@CameraConfig.register_subclass("zmq") @CameraConfig.register_subclass("zmq")
@dataclass @dataclass
class ZMQCameraConfig(CameraConfig): class ZMQCameraConfig(CameraConfig):
"""Configuration for a camera served over a ZeroMQ socket by `ImageServer`.
Use this to read frames from a camera attached to a different machine (e.g. a Raspberry Pi on a
robot), which streams JPEG-encoded frames to this config's `server_address`/`port` over ZMQ.
Args:
server_address (`str`):
Address of the machine running the image server, e.g. `"192.168.1.50"`.
port (`int`, *optional*, defaults to 5555):
TCP port the image server is publishing on. Must be between 1 and 65535.
camera_name (`str`, *optional*, defaults to `"zmq_camera"`):
Name used to identify this camera in logs and observation keys.
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
Color mode for the decoded frames.
timeout_ms (`int`, *optional*, defaults to 5000):
How long to wait for a frame before raising a timeout, in milliseconds. Must be positive.
warmup_s (`int`, *optional*, defaults to 1):
Time spent reading frames before returning from connect, in seconds.
fps (`int`, *optional*):
Requested frames per second. `None` leaves it at the server's own rate.
width (`int`, *optional*):
Requested frame width in pixels. `None` leaves it at the server's own resolution.
height (`int`, *optional*):
Requested frame height in pixels. `None` leaves it at the server's own resolution.
"""
server_address: str server_address: str
port: int = 5555 port: int = 5555
camera_name: str = "zmq_camera" camera_name: str = "zmq_camera"
@@ -32,6 +58,12 @@ class ZMQCameraConfig(CameraConfig):
warmup_s: int = 1 warmup_s: int = 1
def __post_init__(self) -> None: def __post_init__(self) -> None:
"""Normalize `color_mode` and validate the socket settings.
Raises:
ValueError: If `timeout_ms` is not positive, `server_address` is empty, or `port` is outside
the 1-65535 range.
"""
self.color_mode = ColorMode(self.color_mode) self.color_mode = ColorMode(self.color_mode)
if self.timeout_ms <= 0: if self.timeout_ms <= 0:
+27 -2
View File
@@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
""" """Streams camera images over ZMQ.
Streams camera images over ZMQ.
Uses lerobot's OpenCVCamera for capture, encodes images to base64 and sends them over ZMQ. Uses lerobot's OpenCVCamera for capture, encodes images to base64 and sends them over ZMQ.
""" """
@@ -47,6 +47,12 @@ class CameraCaptureThread:
"""Background thread that continuously captures and encodes frames from a camera.""" """Background thread that continuously captures and encodes frames from a camera."""
def __init__(self, camera: OpenCVCamera, name: str): def __init__(self, camera: OpenCVCamera, name: str):
"""Initialize the capture thread for one camera.
Args:
camera: The already-connected camera to read frames from.
name: Name used to identify this camera in log messages.
"""
self.camera = camera self.camera = camera
self.name = name self.name = name
self.latest_encoded: str | None = None # Pre-encoded JPEG as base64 self.latest_encoded: str | None = None # Pre-encoded JPEG as base64
@@ -89,7 +95,21 @@ class CameraCaptureThread:
class ImageServer: class ImageServer:
"""Reads from one or more local OpenCV cameras and publishes their frames over a ZMQ `PUB` socket.
Runs on the machine physically connected to the cameras (e.g. a Raspberry Pi on a robot); pair with
[`~cameras.zmq.ZMQCamera`] on the machine that consumes the stream.
"""
def __init__(self, config: dict, port: int = 5555): def __init__(self, config: dict, port: int = 5555):
"""Connect every camera listed in `config` and open the publishing socket.
Args:
config: A mapping with an optional `"fps"` (the publish loop rate, not the camera capture
rate) and a `"cameras"` mapping of camera name to a dict with `"device_id"`, `"shape"`
(`[height, width]`), and `"fourcc"` keys.
port: TCP port to publish on.
"""
# fps controls the publish loop rate (how often frames are sent over ZMQ), not the camera capture rate # fps controls the publish loop rate (how often frames are sent over ZMQ), not the camera capture rate
self.fps = config.get("fps", 30) self.fps = config.get("fps", 30)
self.cameras: dict[str, OpenCVCamera] = {} self.cameras: dict[str, OpenCVCamera] = {}
@@ -124,6 +144,11 @@ class ImageServer:
logger.info(f"ImageServer running on port {port}") logger.info(f"ImageServer running on port {port}")
def run(self): def run(self):
"""Start each camera's capture thread and publish frames until interrupted.
Blocks until `KeyboardInterrupt`, then stops the capture threads, disconnects the cameras, and
closes the socket.
"""
frame_count = 0 frame_count = 0
frame_times = deque(maxlen=60) frame_times = deque(maxlen=60)
last_published_ts: dict[str, float] = {} last_published_ts: dict[str, float] = {}
+41 -193
View File
@@ -32,16 +32,13 @@ class RandomSubsetApply(Transform):
"""Apply a random subset of N transformations from a list of transformations. """Apply a random subset of N transformations from a list of transformations.
Args: Args:
transforms (`Sequence`): transforms: list of transformations.
List of transformations. p: represents the multinomial probabilities (with no replacement) used for sampling the transform.
p (`list[float] | None`, *optional*): If the sum of the weights is not 1, they will be normalized. If ``None`` (default), all transforms
Multinomial probabilities (with no replacement) used for sampling the transform. Normalized if have the same probability.
they don't already sum to 1. `None` gives all transforms the same probability. n_subset: number of transformations to apply. If ``None``, all transforms are applied.
n_subset (`int | None`, *optional*): Must be in [1, len(transforms)].
Number of transformations to apply. Must be in `[1, len(transforms)]`. `None` applies all of random_order: apply transformations in a random order.
them.
random_order (`bool`, *optional*, defaults to `False`):
Whether to apply the sampled transformations in a random order.
""" """
def __init__( def __init__(
@@ -51,12 +48,6 @@ class RandomSubsetApply(Transform):
n_subset: int | None = None, n_subset: int | None = None,
random_order: bool = False, random_order: bool = False,
) -> None: ) -> None:
"""Validate and store the transform pool, sampling weights, and subset size.
Raises:
TypeError: If `transforms` is not a sequence, or `n_subset` is not an int or `None`.
ValueError: If `p`'s length doesn't match `transforms`, or `n_subset` is out of range.
"""
super().__init__() super().__init__()
if not isinstance(transforms, Sequence): if not isinstance(transforms, Sequence):
raise TypeError("Argument transforms should be a sequence of callables") raise TypeError("Argument transforms should be a sequence of callables")
@@ -83,7 +74,6 @@ class RandomSubsetApply(Transform):
self.selected_transforms: list[Callable[..., Any]] = [] self.selected_transforms: list[Callable[..., Any]] = []
def forward(self, *inputs: Any) -> Any: def forward(self, *inputs: Any) -> Any:
"""Sample a subset of `self.transforms` and apply them in sequence to `inputs`."""
needs_unpacking = len(inputs) > 1 needs_unpacking = len(inputs) > 1
selected_indices = torch.multinomial(torch.tensor(self.p), self.n_subset) selected_indices = torch.multinomial(torch.tensor(self.p), self.n_subset)
@@ -99,7 +89,6 @@ class RandomSubsetApply(Transform):
return outputs return outputs
def extra_repr(self) -> str: def extra_repr(self) -> str:
"""Return the constructor arguments shown in `repr(self)`."""
return ( return (
f"transforms={self.transforms}, " f"transforms={self.transforms}, "
f"p={self.p}, " f"p={self.p}, "
@@ -119,18 +108,16 @@ class SharpnessJitter(Transform):
A sharpness_factor of 0 gives a blurred image, 1 gives the original image while 2 increases the sharpness A sharpness_factor of 0 gives a blurred image, 1 gives the original image while 2 increases the sharpness
by a factor of 2. by a factor of 2.
If the input is a `torch.Tensor`, it is expected to have `[..., 1 or 3, H, W]` shape, where `...` means If the input is a :class:`torch.Tensor`,
an arbitrary number of leading dimensions. it is expected to have [..., 1 or 3, H, W] shape, where ... means an arbitrary number of leading dimensions.
Args: Args:
sharpness (`float | collections.abc.Sequence[float]`): sharpness: How much to jitter sharpness. sharpness_factor is chosen uniformly from
How much to jitter sharpness. `sharpness_factor` is chosen uniformly from [max(0, 1 - sharpness), 1 + sharpness] or the given
`[max(0, 1 - sharpness), 1 + sharpness]`, or the given `[min, max]`. Values must be [min, max]. Should be non negative numbers.
non-negative.
""" """
def __init__(self, sharpness: float | Sequence[float]) -> None: def __init__(self, sharpness: float | Sequence[float]) -> None:
"""Normalize `sharpness` into a `(min, max)` range to sample from on each call."""
super().__init__() super().__init__()
self.sharpness = self._check_input(sharpness) self.sharpness = self._check_input(sharpness)
@@ -151,12 +138,10 @@ class SharpnessJitter(Transform):
return float(sharpness[0]), float(sharpness[1]) return float(sharpness[0]), float(sharpness[1])
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a `sharpness_factor` uniformly from `self.sharpness`."""
sharpness_factor = torch.empty(1).uniform_(self.sharpness[0], self.sharpness[1]).item() sharpness_factor = torch.empty(1).uniform_(self.sharpness[0], self.sharpness[1]).item()
return {"sharpness_factor": sharpness_factor} return {"sharpness_factor": sharpness_factor}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Adjust `inpt`'s sharpness by `params["sharpness_factor"]`."""
sharpness_factor = params["sharpness_factor"] sharpness_factor = params["sharpness_factor"]
return self._call_kernel(F.adjust_sharpness, inpt, sharpness_factor=sharpness_factor) return self._call_kernel(F.adjust_sharpness, inpt, sharpness_factor=sharpness_factor)
@@ -168,17 +153,10 @@ class GaussianNoise(Transform):
Common in real-robot setups where wrist cameras operate in suboptimal lighting. Common in real-robot setups where wrist cameras operate in suboptimal lighting.
Args: Args:
std (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(5.0, 25.0)`): std: Range (min, max) for noise standard deviation in pixel-value scale (0-255).
Range `(min, max)` for the noise standard deviation, in pixel-value scale (0-255).
""" """
def __init__(self, std: float | Sequence[float] = (5.0, 25.0)) -> None: def __init__(self, std: float | Sequence[float] = (5.0, 25.0)) -> None:
"""Normalize `std` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `std` is not a number or a length-2 sequence.
ValueError: If the resulting range does not satisfy `0 <= min <= max`.
"""
super().__init__() super().__init__()
if isinstance(std, (int, float)): if isinstance(std, (int, float)):
self.std = (0.0, float(std)) self.std = (0.0, float(std))
@@ -190,14 +168,12 @@ class GaussianNoise(Transform):
raise ValueError(f"std must satisfy 0 <= min <= max, but got {self.std}.") raise ValueError(f"std must satisfy 0 <= min <= max, but got {self.std}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a noise `std` uniformly from `self.std`, plus a seed for reproducible noise."""
return { return {
"std": torch.empty(1).uniform_(self.std[0], self.std[1]).item(), "std": torch.empty(1).uniform_(self.std[0], self.std[1]).item(),
"seed": torch.randint(0, torch.iinfo(torch.int64).max, ()).item(), "seed": torch.randint(0, torch.iinfo(torch.int64).max, ()).item(),
} }
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Add Gaussian noise with `params["std"]` (in pixel-value scale) to `inpt`, if it's a float tensor."""
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point(): if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
generator = torch.Generator(device=inpt.device).manual_seed(params["seed"]) generator = torch.Generator(device=inpt.device).manual_seed(params["seed"])
noise = torch.randn(inpt.shape, device=inpt.device, dtype=inpt.dtype, generator=generator) noise = torch.randn(inpt.shape, device=inpt.device, dtype=inpt.dtype, generator=generator)
@@ -211,17 +187,10 @@ class MotionBlur(Transform):
Generates a 1D averaging kernel along a random direction, applied via depthwise convolution. Generates a 1D averaging kernel along a random direction, applied via depthwise convolution.
Args: Args:
kernel_size (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3, 11)`): kernel_size: An odd kernel size or a range containing at least one odd kernel size.
An odd kernel size, or a `(min, max)` range containing at least one odd kernel size.
""" """
def __init__(self, kernel_size: int | Sequence[int] = (3, 11)) -> None: def __init__(self, kernel_size: int | Sequence[int] = (3, 11)) -> None:
"""Normalize `kernel_size` into a `(min, max)` range containing at least one odd value.
Raises:
TypeError: If `kernel_size` is not an int or a length-2 sequence.
ValueError: If the resulting range does not satisfy `1 <= min <= max`, or contains no odd value.
"""
super().__init__() super().__init__()
if isinstance(kernel_size, int): if isinstance(kernel_size, int):
self.kernel_size = (kernel_size, kernel_size) self.kernel_size = (kernel_size, kernel_size)
@@ -236,7 +205,6 @@ class MotionBlur(Transform):
raise ValueError(f"kernel_size range must contain an odd value, but got {self.kernel_size}.") raise ValueError(f"kernel_size range must contain an odd value, but got {self.kernel_size}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample an odd kernel size from `self.kernel_size` and a random blur direction in degrees."""
num_odd_sizes = (self.kernel_size[1] - self._first_odd_kernel_size) // 2 + 1 num_odd_sizes = (self.kernel_size[1] - self._first_odd_kernel_size) // 2 + 1
size_index = int(torch.randint(0, num_odd_sizes, ()).item()) size_index = int(torch.randint(0, num_odd_sizes, ()).item())
ks = self._first_odd_kernel_size + 2 * size_index ks = self._first_odd_kernel_size + 2 * size_index
@@ -244,11 +212,6 @@ class MotionBlur(Transform):
return {"kernel_size": ks, "angle": angle} return {"kernel_size": ks, "angle": angle}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Convolve `inpt` with a directional averaging kernel per `params`.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point(): if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt return inpt
if inpt.ndim < 3: if inpt.ndim < 3:
@@ -278,17 +241,10 @@ class JPEGCompression(Transform):
Models quality degradation from video compression in network-streamed camera feeds. Models quality degradation from video compression in network-streamed camera feeds.
Args: Args:
quality (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(15, 75)`): quality: Range (min, max) for JPEG quality factor (lower = more artifacts).
Range `(min, max)` for the JPEG quality factor. Lower values produce more artifacts.
""" """
def __init__(self, quality: int | Sequence[int] = (15, 75)) -> None: def __init__(self, quality: int | Sequence[int] = (15, 75)) -> None:
"""Normalize `quality` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `quality` is not an int or a length-2 sequence.
ValueError: If the resulting range does not satisfy `1 <= min <= max <= 100`.
"""
super().__init__() super().__init__()
if isinstance(quality, int): if isinstance(quality, int):
self.quality = (quality, quality) self.quality = (quality, quality)
@@ -300,16 +256,9 @@ class JPEGCompression(Transform):
raise ValueError(f"quality must satisfy 1 <= min <= max <= 100, but got {self.quality}.") raise ValueError(f"quality must satisfy 1 <= min <= max <= 100, but got {self.quality}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a JPEG `quality` factor uniformly (as an int) from `self.quality`."""
return {"quality": int(torch.randint(self.quality[0], self.quality[1] + 1, (1,)).item())} return {"quality": int(torch.randint(self.quality[0], self.quality[1] + 1, (1,)).item())}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Re-encode and decode `inpt` as JPEG at `params["quality"]`, introducing compression artifacts.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions, or with a channel count
other than 1 or 3.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point(): if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt return inpt
if inpt.ndim < 3: if inpt.ndim < 3:
@@ -335,12 +284,9 @@ class GaussianPatchBrightness(Transform):
encountered in real robot workspaces with multiple light sources. encountered in real robot workspaces with multiple light sources.
Args: Args:
num_patches (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(1, 4)`): num_patches: Range (min, max) for number of brightness patches.
Range `(min, max)` for the number of brightness patches. sigma_range: Range for Gaussian sigma as fraction of image size.
sigma_range (`Sequence`, *optional*, defaults to `(0.05, 0.25)`): factor_range: Range for brightness factor (< 1 darkens, > 1 brightens).
Range `(min, max)` for each patch's Gaussian sigma, as a fraction of image size.
factor_range (`Sequence`, *optional*, defaults to `(0.4, 1.6)`):
Range `(min, max)` for the brightness factor; below 1 darkens, above 1 brightens.
""" """
def __init__( def __init__(
@@ -349,12 +295,6 @@ class GaussianPatchBrightness(Transform):
sigma_range: Sequence[float] = (0.05, 0.25), sigma_range: Sequence[float] = (0.05, 0.25),
factor_range: Sequence[float] = (0.4, 1.6), factor_range: Sequence[float] = (0.4, 1.6),
) -> None: ) -> None:
"""Validate and store the patch count, size, and brightness ranges.
Raises:
TypeError: If any range argument is not the expected type or length.
ValueError: If any range does not satisfy `min <= max` within its valid bounds.
"""
super().__init__() super().__init__()
if isinstance(num_patches, int): if isinstance(num_patches, int):
self.num_patches = (num_patches, num_patches) self.num_patches = (num_patches, num_patches)
@@ -376,7 +316,6 @@ class GaussianPatchBrightness(Transform):
raise ValueError(f"factor_range must satisfy 0 <= min <= max, but got {self.factor_range}.") raise ValueError(f"factor_range must satisfy 0 <= min <= max, but got {self.factor_range}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a random number of patches, each with a random center, sigma, and brightness factor."""
n = int(torch.randint(self.num_patches[0], self.num_patches[1] + 1, (1,)).item()) n = int(torch.randint(self.num_patches[0], self.num_patches[1] + 1, (1,)).item())
return { return {
"centers": torch.rand(n, 2).tolist(), "centers": torch.rand(n, 2).tolist(),
@@ -385,7 +324,6 @@ class GaussianPatchBrightness(Transform):
} }
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Multiply `inpt` by a mask of overlapping Gaussian brightness patches per `params`."""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point(): if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt return inpt
h, w = inpt.shape[-2:] h, w = inpt.shape[-2:]
@@ -409,17 +347,10 @@ class RandomShadow(Transform):
Symmetric: randomly brightens or darkens to prevent BatchNorm stats shift. Symmetric: randomly brightens or darkens to prevent BatchNorm stats shift.
Args: Args:
opacity (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.3, 0.6)`): opacity: Range (min, max) for shadow/highlight opacity.
Range `(min, max)` for the shadow/highlight opacity.
""" """
def __init__(self, opacity: float | Sequence[float] = (0.3, 0.6)) -> None: def __init__(self, opacity: float | Sequence[float] = (0.3, 0.6)) -> None:
"""Normalize `opacity` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `opacity` is not a number or a length-2 sequence.
ValueError: If the resulting range does not satisfy `0 <= min <= max <= 1`.
"""
super().__init__() super().__init__()
if isinstance(opacity, (int, float)): if isinstance(opacity, (int, float)):
self.opacity = (float(opacity), float(opacity)) self.opacity = (float(opacity), float(opacity))
@@ -431,7 +362,6 @@ class RandomShadow(Transform):
raise ValueError(f"opacity must satisfy 0 <= min <= max <= 1, but got {self.opacity}.") raise ValueError(f"opacity must satisfy 0 <= min <= max <= 1, but got {self.opacity}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample the shadow band's opacity, horizontal position/width, and darken-vs-brighten direction."""
return { return {
"opacity": torch.empty(1).uniform_(self.opacity[0], self.opacity[1]).item(), "opacity": torch.empty(1).uniform_(self.opacity[0], self.opacity[1]).item(),
"start": torch.rand(1).item(), "start": torch.rand(1).item(),
@@ -440,11 +370,6 @@ class RandomShadow(Transform):
} }
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Multiply `inpt` by a soft-edged vertical band mask per `params`.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point(): if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt return inpt
if inpt.ndim < 3: if inpt.ndim < 3:
@@ -476,14 +401,10 @@ class CoarseDropout(Transform):
during robot manipulation. during robot manipulation.
Args: Args:
max_holes (`int`, *optional*, defaults to 8): max_holes: Maximum number of rectangular patches to drop.
Maximum number of rectangular patches to drop. max_height_frac: Maximum patch height as fraction of image height.
max_height_frac (`float`, *optional*, defaults to 0.07): max_width_frac: Maximum patch width as fraction of image width.
Maximum patch height, as a fraction of image height. fill_value: Value to fill dropped regions with.
max_width_frac (`float`, *optional*, defaults to 0.07):
Maximum patch width, as a fraction of image width.
fill_value (`float`, *optional*, defaults to 0.0):
Value to fill dropped regions with.
""" """
def __init__( def __init__(
@@ -493,12 +414,6 @@ class CoarseDropout(Transform):
max_width_frac: float = 0.07, max_width_frac: float = 0.07,
fill_value: float = 0.0, fill_value: float = 0.0,
) -> None: ) -> None:
"""Validate and store the dropout patch count, size limits, and fill value.
Raises:
TypeError: If `max_holes` is not an int.
ValueError: If any argument is out of its valid range.
"""
super().__init__() super().__init__()
if not isinstance(max_holes, int): if not isinstance(max_holes, int):
raise TypeError("max_holes must be an int.") raise TypeError("max_holes must be an int.")
@@ -516,7 +431,6 @@ class CoarseDropout(Transform):
self.fill_value = fill_value self.fill_value = fill_value
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a random number of dropout patches, each with a random size and position."""
n = int(torch.randint(1, self.max_holes + 1, (1,)).item()) n = int(torch.randint(1, self.max_holes + 1, (1,)).item())
sizes = torch.rand(n, 2) sizes = torch.rand(n, 2)
sizes[:, 0] *= self.max_height_frac sizes[:, 0] *= self.max_height_frac
@@ -524,11 +438,6 @@ class CoarseDropout(Transform):
return {"sizes": sizes.tolist(), "positions": torch.rand(n, 2).tolist()} return {"sizes": sizes.tolist(), "positions": torch.rand(n, 2).tolist()}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Fill the rectangular patches described by `params` in `inpt` with `self.fill_value`.
Raises:
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point(): if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt return inpt
if inpt.ndim < 3: if inpt.ndim < 3:
@@ -555,18 +464,10 @@ class GammaCorrection(Transform):
preventing BatchNorm statistics shift. preventing BatchNorm statistics shift.
Args: Args:
gamma (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.5, 2.0)`): gamma: Range (min, max) for gamma value. Values < 1 brighten, > 1 darken.
Range `(min, max)` for the gamma value. Values below 1 brighten, above 1 darken.
""" """
def __init__(self, gamma: float | Sequence[float] = (0.5, 2.0)) -> None: def __init__(self, gamma: float | Sequence[float] = (0.5, 2.0)) -> None:
"""Normalize `gamma` into a log-symmetric `(min, max)` range to sample from on each call.
Raises:
TypeError: If `gamma` is not a number or a length-2 sequence.
ValueError: If a single `gamma` is not positive, or the resulting range does not satisfy
`0 < min <= max`.
"""
super().__init__() super().__init__()
if isinstance(gamma, (int, float)): if isinstance(gamma, (int, float)):
gamma = float(gamma) gamma = float(gamma)
@@ -581,14 +482,12 @@ class GammaCorrection(Transform):
raise ValueError(f"gamma must satisfy 0 < min <= max, but got {self.gamma}.") raise ValueError(f"gamma must satisfy 0 < min <= max, but got {self.gamma}.")
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a `gamma` value log-uniformly from `self.gamma`."""
log_lo = math.log(self.gamma[0]) log_lo = math.log(self.gamma[0])
log_hi = math.log(self.gamma[1]) log_hi = math.log(self.gamma[1])
gamma = math.exp(torch.empty(1).uniform_(log_lo, log_hi).item()) gamma = math.exp(torch.empty(1).uniform_(log_lo, log_hi).item())
return {"gamma": gamma} return {"gamma": gamma}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Raise `inpt` to the power `params["gamma"]`, if it's a float tensor."""
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point(): if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
return inpt.pow(params["gamma"]).clamp(0.0, 1.0) return inpt.pow(params["gamma"]).clamp(0.0, 1.0)
return inpt return inpt
@@ -638,18 +537,11 @@ class PlanckianJitter(Transform):
Reference: Zini et al., "Planckian Jitter", CVPR 2022 Workshop. Reference: Zini et al., "Planckian Jitter", CVPR 2022 Workshop.
Args: Args:
temperature (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3000, 15000)`): temperature: A fixed color temperature or range in Kelvin. Supported values
A fixed color temperature, or a `(min, max)` range, in Kelvin. Supported values are between are between 3000 K and 15000 K.
3000 K and 15000 K.
""" """
def __init__(self, temperature: int | Sequence[int] = (3_000, 15_000)) -> None: def __init__(self, temperature: int | Sequence[int] = (3_000, 15_000)) -> None:
"""Normalize `temperature` into a `(min, max)` range to sample from on each call.
Raises:
TypeError: If `temperature` is not an int or a length-2 sequence.
ValueError: If the resulting range falls outside `[3000, 15000]` Kelvin.
"""
super().__init__() super().__init__()
if isinstance(temperature, int): if isinstance(temperature, int):
self.temperature = (temperature, temperature) self.temperature = (temperature, temperature)
@@ -670,16 +562,10 @@ class PlanckianJitter(Transform):
) )
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]: def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
"""Sample a color `temperature` in Kelvin uniformly from `self.temperature`."""
temperature = int(torch.randint(self.temperature[0], self.temperature[1] + 1, ()).item()) temperature = int(torch.randint(self.temperature[0], self.temperature[1] + 1, ()).item())
return {"temperature": temperature} return {"temperature": temperature}
def transform(self, inpt: Any, params: dict[str, Any]) -> Any: def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
"""Scale `inpt`'s red/blue channels per the black-body coefficients at `params["temperature"]`.
Raises:
ValueError: If `inpt` is a float tensor that isn't 3-channel with at least 3 dimensions.
"""
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point(): if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
return inpt return inpt
if inpt.ndim < 3 or inpt.shape[-3] != 3: if inpt.ndim < 3 or inpt.shape[-3] != 3:
@@ -727,18 +613,15 @@ _CUSTOM_TRANSFORMS: dict[str, type[Transform]] = {
@dataclass @dataclass
class ImageTransformConfig: class ImageTransformConfig:
"""Configuration for one entry in an [`~transforms.ImageTransformsConfig`]'s `tfs` mapping. """
For each transform, the following parameters are available:
Args: weight: This represents the multinomial probability (with no replacement)
weight (`float`, *optional*, defaults to 1.0): used for sampling the transform. If the sum of the weights is not 1,
Multinomial probability (with no replacement) of sampling this transform. Normalized against they will be normalized.
the other transforms' weights if they don't already sum to 1. type: The name of the class used. This is either a class available under torchvision.transforms.v2 or a
type (`str`, *optional*, defaults to `"Identity"`): custom transform defined here.
Name of the transform class to build either a class under `torchvision.transforms.v2` or one kwargs: Lower & upper bound respectively used for sampling the transform's parameter
of the custom transforms in this module. Passed to (following uniform distribution) when it's applied.
[`~transforms.make_transform_from_config`].
kwargs (`dict[str, Any]`, *optional*):
Keyword arguments passed to the transform's constructor.
""" """
weight: float = 1.0 weight: float = 1.0
@@ -748,21 +631,11 @@ class ImageTransformConfig:
@dataclass @dataclass
class ImageTransformsConfig: class ImageTransformsConfig:
"""Configuration for [`~transforms.ImageTransforms`], a random subset of image augmentations. """
These transforms are all using standard torchvision.transforms.v2
Transforms are standard [`torchvision.transforms.v2`](https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html) You can find out how these transformations affect images here:
or custom transforms from this module, sampled via [`~transforms.RandomSubsetApply`]. https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html
We use a custom RandomSubsetApply container to sample them.
Args:
enable (`bool`, *optional*, defaults to `False`):
Whether to apply transforms at all. `False` disables augmentation entirely.
max_num_transforms (`int`, *optional*, defaults to 3):
Maximum number of transforms (sampled from `tfs`) applied to each frame. Must be in
`[1, len(tfs)]`.
random_order (`bool`, *optional*, defaults to `False`):
Whether to apply the sampled transforms in a random order, instead of the order in `tfs`.
tfs (`dict[str, ImageTransformConfig]`, *optional*):
The available transforms, keyed by name, with their sampling weight and constructor arguments.
""" """
# Set this flag to `true` to enable transforms during training # Set this flag to `true` to enable transforms during training
@@ -810,19 +683,6 @@ class ImageTransformsConfig:
def make_transform_from_config(cfg: ImageTransformConfig) -> Transform: def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
"""Instantiate the transform named by `cfg.type`, from `torchvision.transforms.v2` or this module.
Args:
cfg (`ImageTransformConfig`):
Configuration naming the transform class and its constructor arguments.
Returns:
`Transform`: The instantiated transform.
Raises:
ValueError: If `cfg.type` is not a `torchvision.transforms.v2` transform or one of this module's
custom transforms.
"""
if cfg.type in _CUSTOM_TRANSFORMS: if cfg.type in _CUSTOM_TRANSFORMS:
return _CUSTOM_TRANSFORMS[cfg.type](**cfg.kwargs) return _CUSTOM_TRANSFORMS[cfg.type](**cfg.kwargs)
@@ -838,20 +698,9 @@ def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
class ImageTransforms(Transform): class ImageTransforms(Transform):
"""Composes a random subset of image augmentations from an [`~transforms.ImageTransformsConfig`]. """A class to compose image transforms based on configuration."""
Builds each enabled transform (weight > 0) named in `cfg.tfs`, then wraps them in a
[`~transforms.RandomSubsetApply`] so a random subset is applied on each call. If `cfg.enable` is
`False` or no transforms are enabled, this is equivalent to the identity transform.
"""
def __init__(self, cfg: ImageTransformsConfig) -> None: def __init__(self, cfg: ImageTransformsConfig) -> None:
"""Build the enabled transforms from `cfg` and wrap them in a random-subset sampler.
Args:
cfg (`ImageTransformsConfig`):
Configuration listing the available transforms and how many to sample per call.
"""
super().__init__() super().__init__()
self._cfg = cfg self._cfg = cfg
@@ -876,5 +725,4 @@ class ImageTransforms(Transform):
) )
def forward(self, *inputs: Any) -> Any: def forward(self, *inputs: Any) -> Any:
"""Apply the sampled subset of transforms (or the identity, if none are enabled) to `inputs`."""
return self.tf(*inputs) return self.tf(*inputs)
+1 -1
View File
@@ -60,7 +60,7 @@ PATH_TO_LEROBOT = PATH_TO_REPO / "src" / "lerobot"
# Modules whose public objects are checked. Add a module here once its docstrings follow the standard. # Modules whose public objects are checked. Add a module here once its docstrings follow the standard.
MODULES_TO_CHECK = [ MODULES_TO_CHECK = [
"lerobot.robots", "lerobot.robots",
"lerobot.transforms", "lerobot.cameras",
] ]
# Objects that do not yet follow the standard, so the check can be green from day one. Removing an entry # Objects that do not yet follow the standard, so the check can be green from day one. Removing an entry
+1
View File
@@ -10,6 +10,7 @@
# `utils/check_doctest_list.py` does not care which way round it is. # `utils/check_doctest_list.py` does not care which way round it is.
# #
# Keep alphabetically sorted: `make check-doctest-list` enforces it, `make fix-docstrings` sorts it. # Keep alphabetically sorted: `make check-doctest-list` enforces it, `make fix-docstrings` sorts it.
src/lerobot/cameras/utils.py
src/lerobot/motors/motors_bus.py src/lerobot/motors/motors_bus.py
src/lerobot/robots/robot.py src/lerobot/robots/robot.py
src/lerobot/robots/so_follower/config_so_follower.py src/lerobot/robots/so_follower/config_so_follower.py