mirror of
https://github.com/huggingface/lerobot.git
synced 2026-08-08 17:39:44 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f1e61b9ac | |||
| 2178e11eaa |
@@ -22,3 +22,52 @@ See the [Cameras guide](../cameras) for choosing and configuring a camera, and
|
||||
## make_cameras_from_configs
|
||||
|
||||
[[autodoc]] lerobot.cameras.make_cameras_from_configs
|
||||
|
||||
## ColorMode
|
||||
|
||||
[[autodoc]] lerobot.cameras.ColorMode
|
||||
|
||||
## Cv2Rotation
|
||||
|
||||
[[autodoc]] lerobot.cameras.Cv2Rotation
|
||||
|
||||
## Cv2Backends
|
||||
|
||||
[[autodoc]] lerobot.cameras.Cv2Backends
|
||||
|
||||
## OpenCVCamera
|
||||
|
||||
The default backend for USB webcams and video files, built on OpenCV.
|
||||
|
||||
[[autodoc]] lerobot.cameras.opencv.OpenCVCamera
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.cameras.opencv.OpenCVCameraConfig
|
||||
|
||||
## RealSenseCamera
|
||||
|
||||
Intel RealSense cameras, with optional depth sensing.
|
||||
|
||||
[[autodoc]] lerobot.cameras.realsense.RealSenseCamera
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.cameras.realsense.RealSenseCameraConfig
|
||||
|
||||
## Reachy2Camera
|
||||
|
||||
Cameras exposed by a Reachy 2 robot's own camera manager.
|
||||
|
||||
[[autodoc]] lerobot.cameras.reachy2_camera.Reachy2Camera
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.cameras.reachy2_camera.Reachy2CameraConfig
|
||||
|
||||
## ZMQCamera
|
||||
|
||||
Reads frames published over a ZeroMQ socket by `ImageServer`, for cameras attached to a different machine
|
||||
than the one running the policy.
|
||||
|
||||
[[autodoc]] lerobot.cameras.zmq.ZMQCamera
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.cameras.zmq.ZMQCameraConfig
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
# Image Transforms
|
||||
|
||||
Data-augmentation transforms applied to camera observations during training. [`~transforms.ImageTransforms`]
|
||||
composes a random subset of them, configured via [`~transforms.ImageTransformsConfig`].
|
||||
|
||||
## ImageTransforms
|
||||
|
||||
[[autodoc]] lerobot.transforms.ImageTransforms
|
||||
- all
|
||||
|
||||
## ImageTransformsConfig
|
||||
|
||||
[[autodoc]] lerobot.transforms.ImageTransformsConfig
|
||||
|
||||
## ImageTransformConfig
|
||||
|
||||
[[autodoc]] lerobot.transforms.ImageTransformConfig
|
||||
|
||||
## make_transform_from_config
|
||||
|
||||
[[autodoc]] lerobot.transforms.make_transform_from_config
|
||||
|
||||
## RandomSubsetApply
|
||||
|
||||
[[autodoc]] lerobot.transforms.RandomSubsetApply
|
||||
- all
|
||||
|
||||
## SharpnessJitter
|
||||
|
||||
[[autodoc]] lerobot.transforms.SharpnessJitter
|
||||
- all
|
||||
|
||||
## GaussianNoise
|
||||
|
||||
[[autodoc]] lerobot.transforms.GaussianNoise
|
||||
- all
|
||||
|
||||
## MotionBlur
|
||||
|
||||
[[autodoc]] lerobot.transforms.MotionBlur
|
||||
- all
|
||||
|
||||
## JPEGCompression
|
||||
|
||||
[[autodoc]] lerobot.transforms.JPEGCompression
|
||||
- all
|
||||
|
||||
## GaussianPatchBrightness
|
||||
|
||||
[[autodoc]] lerobot.transforms.GaussianPatchBrightness
|
||||
- all
|
||||
|
||||
## RandomShadow
|
||||
|
||||
[[autodoc]] lerobot.transforms.RandomShadow
|
||||
- all
|
||||
|
||||
## CoarseDropout
|
||||
|
||||
[[autodoc]] lerobot.transforms.CoarseDropout
|
||||
- all
|
||||
|
||||
## GammaCorrection
|
||||
|
||||
[[autodoc]] lerobot.transforms.GammaCorrection
|
||||
- all
|
||||
|
||||
## PlanckianJitter
|
||||
|
||||
[[autodoc]] lerobot.transforms.PlanckianJitter
|
||||
- all
|
||||
+1
-1
@@ -437,7 +437,6 @@ ignore = [
|
||||
# Awaiting conversion, one PR per module.
|
||||
"src/lerobot/annotations/**" = ["D"]
|
||||
"src/lerobot/async_inference/**" = ["D"]
|
||||
"src/lerobot/cameras/**" = ["D"]
|
||||
"src/lerobot/common/**" = ["D"]
|
||||
"src/lerobot/configs/**" = ["D"]
|
||||
"src/lerobot/data_processing/**" = ["D"]
|
||||
@@ -455,6 +454,7 @@ ignore = [
|
||||
"src/lerobot/rollout/**" = ["D"]
|
||||
"src/lerobot/scripts/**" = ["D"]
|
||||
"src/lerobot/teleoperators/**" = ["D"]
|
||||
"src/lerobot/transforms/**" = ["D"]
|
||||
"src/lerobot/transport/**" = ["D"]
|
||||
"src/lerobot/utils/**" = ["D"]
|
||||
"src/lerobot/lerobot_types.py" = ["D"]
|
||||
|
||||
@@ -50,23 +50,23 @@ class Camera(abc.ABC):
|
||||
self.height: int | None = config.height
|
||||
|
||||
def __enter__(self):
|
||||
"""
|
||||
Context manager entry.
|
||||
"""Context manager entry.
|
||||
|
||||
Automatically connects to the camera.
|
||||
"""
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
||||
"""
|
||||
Context manager exit.
|
||||
"""Context manager exit.
|
||||
|
||||
Automatically disconnects, ensuring resources are released even on error.
|
||||
"""
|
||||
self.disconnect()
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""
|
||||
Destructor safety net.
|
||||
"""Destructor safety net.
|
||||
|
||||
Attempts to disconnect if the object is garbage collected without cleanup.
|
||||
"""
|
||||
try:
|
||||
@@ -90,6 +90,7 @@ class Camera(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
def find_cameras() -> list[dict[str, Any]]:
|
||||
"""Detects available cameras connected to the system.
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: A list of dictionaries,
|
||||
where each dictionary contains information about a detected camera.
|
||||
|
||||
@@ -22,15 +22,36 @@ import draccus # type: ignore # TODO: add type stubs for draccus
|
||||
|
||||
|
||||
class ColorMode(str, Enum):
|
||||
"""Color channel order for frames returned by a camera.
|
||||
|
||||
**Attributes**:
|
||||
- **RGB** -- Red-green-blue channel order.
|
||||
- **BGR** -- Blue-green-red channel order, OpenCV's native order.
|
||||
"""
|
||||
|
||||
RGB = "rgb"
|
||||
BGR = "bgr"
|
||||
|
||||
@classmethod
|
||||
def _missing_(cls, value: object) -> None:
|
||||
"""Reject a value that is not a valid `ColorMode`.
|
||||
|
||||
Raises:
|
||||
ValueError: Always, naming the invalid value and the valid choices.
|
||||
"""
|
||||
raise ValueError(f"`color_mode` is expected to be in {list(cls)}, but {value} is provided.")
|
||||
|
||||
|
||||
class Cv2Rotation(int, Enum):
|
||||
"""Clockwise rotation to apply to a frame after capture, in degrees.
|
||||
|
||||
**Attributes**:
|
||||
- **NO_ROTATION** -- No rotation.
|
||||
- **ROTATE_90** -- Rotate 90° clockwise.
|
||||
- **ROTATE_180** -- Rotate 180°.
|
||||
- **ROTATE_270** -- Rotate 270° clockwise (90° counter-clockwise).
|
||||
"""
|
||||
|
||||
NO_ROTATION = 0
|
||||
ROTATE_90 = 90
|
||||
ROTATE_180 = 180
|
||||
@@ -38,11 +59,31 @@ class Cv2Rotation(int, Enum):
|
||||
|
||||
@classmethod
|
||||
def _missing_(cls, value: object) -> None:
|
||||
"""Reject a value that is not a valid `Cv2Rotation`.
|
||||
|
||||
Raises:
|
||||
ValueError: Always, naming the invalid value and the valid choices.
|
||||
"""
|
||||
raise ValueError(f"`rotation` is expected to be in {list(cls)}, but {value} is provided.")
|
||||
|
||||
|
||||
# Subset from https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html
|
||||
class Cv2Backends(int, Enum):
|
||||
"""OpenCV capture backend to request when opening a device.
|
||||
|
||||
See the [OpenCV `VideoCaptureAPIs` reference](https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html)
|
||||
for the full list this is a subset of.
|
||||
|
||||
**Attributes**:
|
||||
- **ANY** -- Let OpenCV auto-detect the backend.
|
||||
- **V4L2** -- Video4Linux2, the usual choice on Linux.
|
||||
- **DSHOW** -- DirectShow, a Windows backend.
|
||||
- **PVAPI** -- PvAPI, for Prosilica GigE cameras.
|
||||
- **ANDROID** -- Android's native camera API.
|
||||
- **AVFOUNDATION** -- AVFoundation, the usual choice on macOS.
|
||||
- **MSMF** -- Microsoft Media Foundation, a Windows backend.
|
||||
"""
|
||||
|
||||
ANY = 0
|
||||
V4L2 = 200
|
||||
DSHOW = 700
|
||||
@@ -53,15 +94,40 @@ class Cv2Backends(int, Enum):
|
||||
|
||||
@classmethod
|
||||
def _missing_(cls, value: object) -> None:
|
||||
"""Reject a value that is not a valid `Cv2Backends`.
|
||||
|
||||
Raises:
|
||||
ValueError: Always, naming the invalid value and the valid choices.
|
||||
"""
|
||||
raise ValueError(f"`backend` is expected to be in {list(cls)}, but {value} is provided.")
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class CameraConfig(draccus.ChoiceRegistry, abc.ABC): # type: ignore # TODO: add type stubs for draccus
|
||||
"""Base configuration shared by every camera backend.
|
||||
|
||||
Concrete backends subclass this and register themselves with
|
||||
`@CameraConfig.register_subclass("name")`, which is what makes `--camera.type=name` work on the
|
||||
command line. Subclasses inherit the three fields below and must document them alongside their own.
|
||||
|
||||
Args:
|
||||
fps (`int`, *optional*):
|
||||
Requested frames per second for the color stream. `None` leaves it at the backend's default.
|
||||
width (`int`, *optional*):
|
||||
Requested frame width in pixels. `None` leaves it at the backend's default.
|
||||
height (`int`, *optional*):
|
||||
Requested frame height in pixels. `None` leaves it at the backend's default.
|
||||
"""
|
||||
|
||||
fps: int | None = None
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
"""Return the registered name this config was registered under.
|
||||
|
||||
Returns:
|
||||
`str`: The name passed to `@CameraConfig.register_subclass`, e.g. `"opencv"`.
|
||||
"""
|
||||
return str(self.get_choice_name(self.__class__))
|
||||
|
||||
@@ -12,9 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Provides the OpenCVCamera class for capturing frames from cameras using OpenCV.
|
||||
"""
|
||||
"""Provides the OpenCVCamera class for capturing frames from cameras using OpenCV."""
|
||||
|
||||
import logging
|
||||
import math
|
||||
@@ -50,8 +48,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenCVCamera(Camera):
|
||||
"""
|
||||
Manages camera interactions using OpenCV for efficient frame recording.
|
||||
"""Manages camera interactions using OpenCV for efficient frame recording.
|
||||
|
||||
This class provides a high-level interface to connect to, configure, and read
|
||||
frames from cameras compatible with OpenCV's VideoCapture. It supports both
|
||||
@@ -93,8 +90,7 @@ class OpenCVCamera(Camera):
|
||||
"""
|
||||
|
||||
def __init__(self, config: OpenCVCameraConfig):
|
||||
"""
|
||||
Initializes the OpenCVCamera instance.
|
||||
"""Initializes the OpenCVCamera instance.
|
||||
|
||||
Args:
|
||||
config: The configuration settings for the camera.
|
||||
@@ -125,6 +121,7 @@ class OpenCVCamera(Camera):
|
||||
self._reset_connection_settings()
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Return a short representation naming the class and its `index_or_path`."""
|
||||
return f"{self.__class__.__name__}({self.index_or_path})"
|
||||
|
||||
def _reset_connection_settings(self) -> None:
|
||||
@@ -143,8 +140,7 @@ class OpenCVCamera(Camera):
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, warmup: bool = True) -> None:
|
||||
"""
|
||||
Connects to the OpenCV camera specified in the configuration.
|
||||
"""Connects to the OpenCV camera specified in the configuration.
|
||||
|
||||
Initializes the OpenCV VideoCapture object, sets desired camera properties
|
||||
(FPS, width, height), starts the background reading thread and performs initial checks.
|
||||
@@ -158,7 +154,6 @@ class OpenCVCamera(Camera):
|
||||
ConnectionError: If the specified camera index/path is not found or fails to open.
|
||||
RuntimeError: If the camera opens but fails to apply requested settings.
|
||||
"""
|
||||
|
||||
# Use 1 thread for OpenCV operations to avoid potential conflicts or
|
||||
# blocking in multi-threaded applications, especially during data collection.
|
||||
cv2.setNumThreads(1)
|
||||
@@ -196,8 +191,7 @@ class OpenCVCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def _configure_capture_settings(self) -> None:
|
||||
"""
|
||||
Applies the specified FOURCC, FPS, width, and height settings to the connected camera.
|
||||
"""Applies the specified FOURCC, FPS, width, and height settings to the connected camera.
|
||||
|
||||
This method attempts to set the camera properties via OpenCV. It checks if
|
||||
the camera successfully applied the settings and raises an error if not.
|
||||
@@ -214,7 +208,6 @@ class OpenCVCamera(Camera):
|
||||
to the requested value.
|
||||
DeviceNotConnectedError: If the camera is not connected.
|
||||
"""
|
||||
|
||||
if self.videocapture is None:
|
||||
raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
|
||||
|
||||
@@ -246,7 +239,6 @@ class OpenCVCamera(Camera):
|
||||
|
||||
def _validate_fps(self) -> None:
|
||||
"""Validates and sets the camera's frames per second (FPS)."""
|
||||
|
||||
if self.videocapture is None:
|
||||
raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
|
||||
|
||||
@@ -261,7 +253,6 @@ class OpenCVCamera(Camera):
|
||||
|
||||
def _validate_fourcc(self) -> None:
|
||||
"""Validates and sets the camera's FOURCC code."""
|
||||
|
||||
fourcc_code = cv2.VideoWriter_fourcc(*self.config.fourcc)
|
||||
|
||||
if self.videocapture is None:
|
||||
@@ -282,7 +273,6 @@ class OpenCVCamera(Camera):
|
||||
|
||||
def _validate_width_and_height(self) -> None:
|
||||
"""Validates and sets the camera's frame capture width and height."""
|
||||
|
||||
if self.videocapture is None:
|
||||
raise DeviceNotConnectedError(f"{self} videocapture is not initialized")
|
||||
|
||||
@@ -306,8 +296,7 @@ class OpenCVCamera(Camera):
|
||||
|
||||
@staticmethod
|
||||
def find_cameras() -> list[dict[str, Any]]:
|
||||
"""
|
||||
Detects available OpenCV cameras connected to the system.
|
||||
"""Detects available OpenCV cameras connected to the system.
|
||||
|
||||
On Linux, it scans '/dev/video*' paths. On other systems (like macOS, Windows),
|
||||
it checks indices from 0 up to `MAX_OPENCV_INDEX`.
|
||||
@@ -375,8 +364,7 @@ class OpenCVCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
|
||||
"""
|
||||
Reads a single frame synchronously from the camera.
|
||||
"""Reads a single frame synchronously from the camera.
|
||||
|
||||
This is a blocking call. It waits for the next available frame from the
|
||||
camera hardware via OpenCV.
|
||||
@@ -392,7 +380,6 @@ class OpenCVCamera(Camera):
|
||||
received frame dimensions don't match expectations before rotation.
|
||||
ValueError: If an invalid `color_mode` is requested.
|
||||
"""
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
if color_mode is not None:
|
||||
@@ -412,8 +399,7 @@ class OpenCVCamera(Camera):
|
||||
return frame
|
||||
|
||||
def _postprocess_image(self, image: NDArray[Any]) -> NDArray[Any]:
|
||||
"""
|
||||
Applies color conversion, dimension validation, and rotation to a raw frame.
|
||||
"""Applies color conversion, dimension validation, and rotation to a raw frame.
|
||||
|
||||
Args:
|
||||
image (np.ndarray): The raw image frame (expected BGR format from OpenCV).
|
||||
@@ -426,7 +412,6 @@ class OpenCVCamera(Camera):
|
||||
RuntimeError: If the raw frame dimensions do not match the configured
|
||||
`width` and `height`.
|
||||
"""
|
||||
|
||||
if self.color_mode not in (ColorMode.RGB, ColorMode.BGR):
|
||||
raise ValueError(
|
||||
f"Invalid color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}."
|
||||
@@ -452,8 +437,7 @@ class OpenCVCamera(Camera):
|
||||
return processed_image
|
||||
|
||||
def _read_loop(self) -> None:
|
||||
"""
|
||||
Internal loop run by the background thread for asynchronous reading.
|
||||
"""Internal loop run by the background thread for asynchronous reading.
|
||||
|
||||
On each iteration:
|
||||
1. Reads a color frame (blocking call)
|
||||
@@ -538,8 +522,7 @@ class OpenCVCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
|
||||
"""
|
||||
Reads the latest available frame asynchronously.
|
||||
"""Reads the latest available frame asynchronously.
|
||||
|
||||
This method retrieves the most recent frame captured by the background
|
||||
read thread. It does not block waiting for the camera hardware directly,
|
||||
@@ -559,7 +542,6 @@ class OpenCVCamera(Camera):
|
||||
TimeoutError: If no frame becomes available within the specified timeout.
|
||||
RuntimeError: If an unexpected error occurs.
|
||||
"""
|
||||
|
||||
if self.thread is None or not self.thread.is_alive():
|
||||
raise RuntimeError(f"{self} read thread is not running.")
|
||||
|
||||
@@ -594,7 +576,6 @@ class OpenCVCamera(Camera):
|
||||
DeviceNotConnectedError: If the camera is not connected.
|
||||
RuntimeError: If the camera is connected but has not captured any frames yet.
|
||||
"""
|
||||
|
||||
if self.thread is None or not self.thread.is_alive():
|
||||
raise RuntimeError(f"{self} read thread is not running.")
|
||||
|
||||
@@ -614,8 +595,7 @@ class OpenCVCamera(Camera):
|
||||
return frame
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""
|
||||
Disconnects from the camera and cleans up resources.
|
||||
"""Disconnects from the camera and cleans up resources.
|
||||
|
||||
Stops the background read thread (if running) and releases the OpenCV
|
||||
VideoCapture object.
|
||||
|
||||
@@ -40,20 +40,28 @@ class OpenCVCameraConfig(CameraConfig):
|
||||
OpenCVCameraConfig(0, 30, 1280, 720, fourcc="YUYV") # With YUYV format
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **index_or_path** (`int | Path`) -- Either an integer representing the camera device index, or a
|
||||
Path object pointing to a video file.
|
||||
- **fps** -- Requested frames per second for the color stream.
|
||||
- **width** -- Requested frame width in pixels for the color stream.
|
||||
- **height** -- Requested frame height in pixels for the color stream.
|
||||
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no
|
||||
rotation.
|
||||
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds)
|
||||
- **fourcc** (`str | None`) -- FOURCC code for video format (e.g., "MJPG", "YUYV", "I420"). Defaults
|
||||
to None (auto-detect).
|
||||
- **backend** (`Cv2Backends`) -- OpenCV backend identifier
|
||||
(https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html). Defaults to ANY.
|
||||
Args:
|
||||
index_or_path (`int | Path`):
|
||||
Either an integer representing the camera device index, or a Path object pointing to a video
|
||||
file.
|
||||
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
|
||||
Color mode for image output.
|
||||
rotation (`Cv2Rotation`, *optional*, defaults to `Cv2Rotation.NO_ROTATION`):
|
||||
Image rotation setting (0°, 90°, 180°, or 270°).
|
||||
warmup_s (`int`, *optional*, defaults to 1):
|
||||
Time reading frames before returning from connect (in seconds).
|
||||
fourcc (`str`, *optional*):
|
||||
FOURCC code for video format (e.g., `"MJPG"`, `"YUYV"`, `"I420"`). `None` auto-detects.
|
||||
backend (`Cv2Backends`, *optional*, defaults to `Cv2Backends.ANY`):
|
||||
OpenCV backend identifier. See [`~cameras.Cv2Backends`] for the supported values.
|
||||
fps (`int`, *optional*):
|
||||
Requested frames per second for the color stream. `None` leaves it at the backend's default.
|
||||
width (`int`, *optional*):
|
||||
Requested frame width in pixels for the color stream. `None` leaves it at the backend's
|
||||
default.
|
||||
height (`int`, *optional*):
|
||||
Requested frame height in pixels for the color stream. `None` leaves it at the backend's
|
||||
default.
|
||||
|
||||
Note:
|
||||
- Only 3-channel color output (RGB/BGR) is currently supported.
|
||||
@@ -69,6 +77,11 @@ class OpenCVCameraConfig(CameraConfig):
|
||||
backend: Cv2Backends = Cv2Backends.ANY
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Normalize `color_mode`, `rotation`, and `backend`, and validate `fourcc`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `fourcc` is set and is not a 4-character string.
|
||||
"""
|
||||
self.color_mode = ColorMode(self.color_mode)
|
||||
self.rotation = Cv2Rotation(self.rotation)
|
||||
self.backend = Cv2Backends(self.backend)
|
||||
|
||||
@@ -43,16 +43,24 @@ class Reachy2CameraConfig(CameraConfig):
|
||||
) # Left teleop camera, 640x480 @ 30FPS
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **name** (`str`) -- Name of the camera device. Can be "teleop" or "depth".
|
||||
- **image_type** (`str`) -- Type of image stream. For "teleop" camera, can be "left" or "right". For
|
||||
"depth" camera, can be "rgb" or "depth". (depth is not supported yet)
|
||||
- **fps** -- Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
|
||||
- **width** -- Requested frame width in pixels for the color stream.
|
||||
- **height** -- Requested frame height in pixels for the color stream.
|
||||
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
- **ip_address** (`str | None`) -- IP address of the robot. Defaults to "localhost".
|
||||
- **port** (`int`) -- Port number for the camera server. Defaults to 50065.
|
||||
Args:
|
||||
name (`str`):
|
||||
Name of the camera device. Either `"teleop"` or `"depth"`.
|
||||
image_type (`str`):
|
||||
Type of image stream. For the `"teleop"` camera, either `"left"` or `"right"`. For the
|
||||
`"depth"` camera, either `"rgb"` or `"depth"` (depth is not supported yet).
|
||||
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
|
||||
Color mode for image output.
|
||||
ip_address (`str`, *optional*, defaults to `"localhost"`):
|
||||
IP address of the robot.
|
||||
port (`int`, *optional*, defaults to 50065):
|
||||
Port number for the camera server.
|
||||
fps (`int`, *optional*):
|
||||
Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
|
||||
width (`int`, *optional*):
|
||||
Requested frame width in pixels for the color stream.
|
||||
height (`int`, *optional*):
|
||||
Requested frame height in pixels for the color stream.
|
||||
|
||||
Note:
|
||||
- Only 3-channel color output (RGB/BGR) is currently supported.
|
||||
@@ -65,6 +73,11 @@ class Reachy2CameraConfig(CameraConfig):
|
||||
port: int = 50065
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Normalize `color_mode` and validate `name`/`image_type`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `name` is not `"teleop"`/`"depth"`, or `image_type` is not valid for `name`.
|
||||
"""
|
||||
if self.name not in ["teleop", "depth"]:
|
||||
raise ValueError(f"`name` is expected to be 'teleop' or 'depth', but {self.name} is provided.")
|
||||
if (self.name == "teleop" and self.image_type not in ["left", "right"]) or (
|
||||
|
||||
@@ -12,9 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Provides the Reachy2Camera class for capturing frames from Reachy 2 cameras using Reachy 2's CameraManager.
|
||||
"""
|
||||
"""Provides the Reachy2Camera class for capturing frames from Reachy 2 cameras using Reachy 2's CameraManager."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -42,6 +40,8 @@ else:
|
||||
CameraManager = None
|
||||
|
||||
class CameraView:
|
||||
"""Fallback stand-in for `reachy2_sdk`'s `CameraView` when the SDK is not installed."""
|
||||
|
||||
LEFT = 0
|
||||
RIGHT = 1
|
||||
|
||||
@@ -55,8 +55,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Reachy2Camera(Camera):
|
||||
"""
|
||||
Manages Reachy 2 camera using Reachy 2 CameraManager.
|
||||
"""Manages Reachy 2 camera using Reachy 2 CameraManager.
|
||||
|
||||
This class provides a high-level interface to connect to, configure, and read
|
||||
frames from Reachy 2 cameras. It supports both synchronous and asynchronous
|
||||
@@ -70,8 +69,7 @@ class Reachy2Camera(Camera):
|
||||
"""
|
||||
|
||||
def __init__(self, config: Reachy2CameraConfig):
|
||||
"""
|
||||
Initializes the Reachy2Camera instance.
|
||||
"""Initializes the Reachy2Camera instance.
|
||||
|
||||
Args:
|
||||
config: The configuration settings for the camera.
|
||||
@@ -88,6 +86,7 @@ class Reachy2Camera(Camera):
|
||||
self.cam_manager: CameraManager | None = None
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Return a short representation naming the class, camera name, and image type."""
|
||||
return f"{self.__class__.__name__}({self.config.name}, {self.config.image_type})"
|
||||
|
||||
@property
|
||||
@@ -105,8 +104,7 @@ class Reachy2Camera(Camera):
|
||||
raise ValueError(f"Invalid camera name '{self.config.name}'. Expected 'teleop' or 'depth'.")
|
||||
|
||||
def connect(self, warmup: bool = True) -> None:
|
||||
"""
|
||||
Connects to the Reachy2 CameraManager as specified in the configuration.
|
||||
"""Connects to the Reachy2 CameraManager as specified in the configuration.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the camera is not connected.
|
||||
@@ -120,15 +118,12 @@ class Reachy2Camera(Camera):
|
||||
|
||||
@staticmethod
|
||||
def find_cameras() -> list[dict[str, Any]]:
|
||||
"""
|
||||
Detection not implemented for Reachy2 cameras.
|
||||
"""
|
||||
"""Detection not implemented for Reachy2 cameras."""
|
||||
raise NotImplementedError("Camera detection is not implemented for Reachy2 cameras.")
|
||||
|
||||
@check_if_not_connected
|
||||
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
|
||||
"""
|
||||
Reads a single frame synchronously from the camera.
|
||||
"""Reads a single frame synchronously from the camera.
|
||||
|
||||
This method retrieves the most recent frame available in Reachy 2's low-level software.
|
||||
|
||||
@@ -187,8 +182,7 @@ class Reachy2Camera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
|
||||
"""
|
||||
Same as read()
|
||||
"""Same as [`~cameras.reachy2_camera.Reachy2Camera.read`]; Reachy 2 has no separate async path.
|
||||
|
||||
Returns:
|
||||
np.ndarray: The latest captured frame as a NumPy array in the format
|
||||
@@ -199,7 +193,6 @@ class Reachy2Camera(Camera):
|
||||
TimeoutError: If no frame becomes available within the specified timeout.
|
||||
RuntimeError: If an unexpected error occurs.
|
||||
"""
|
||||
|
||||
return self.read()
|
||||
|
||||
@check_if_not_connected
|
||||
@@ -220,7 +213,6 @@ class Reachy2Camera(Camera):
|
||||
DeviceNotConnectedError: If the camera is not connected.
|
||||
RuntimeError: If the camera is connected but has not captured any frames yet.
|
||||
"""
|
||||
|
||||
if self.latest_frame is None or self.latest_timestamp is None:
|
||||
raise RuntimeError(f"{self} has not captured any frames yet.")
|
||||
|
||||
@@ -234,13 +226,11 @@ class Reachy2Camera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def disconnect(self) -> None:
|
||||
"""
|
||||
Stops the background read thread (if running).
|
||||
"""Stops the background read thread (if running).
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the camera is already disconnected.
|
||||
"""
|
||||
|
||||
if self.cam_manager is not None:
|
||||
self.cam_manager.disconnect()
|
||||
|
||||
|
||||
@@ -12,9 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Provides the RealSenseCamera class for capturing frames from Intel RealSense cameras.
|
||||
"""
|
||||
"""Provides the RealSenseCamera class for capturing frames from Intel RealSense cameras."""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
@@ -46,8 +44,7 @@ pkg_name = "pyrealsense2-macosx" if sys.platform == "darwin" else "pyrealsense2"
|
||||
|
||||
|
||||
class RealSenseCamera(Camera):
|
||||
"""
|
||||
Manages interactions with Intel RealSense cameras for frame and depth recording.
|
||||
"""Manages interactions with Intel RealSense cameras for frame and depth recording.
|
||||
|
||||
This class provides an interface similar to `OpenCVCamera` but tailored for
|
||||
RealSense devices, leveraging the `pyrealsense2` library. It uses the camera's
|
||||
@@ -115,8 +112,7 @@ class RealSenseCamera(Camera):
|
||||
_MAX_CONNECT_ATTEMPTS = 3
|
||||
|
||||
def __init__(self, config: RealSenseCameraConfig):
|
||||
"""
|
||||
Initializes the RealSenseCamera instance.
|
||||
"""Initializes the RealSenseCamera instance.
|
||||
|
||||
Args:
|
||||
config: The configuration settings for the camera.
|
||||
@@ -161,6 +157,7 @@ class RealSenseCamera(Camera):
|
||||
self._reset_connection_settings()
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Return a short representation naming the class and its `serial_number`."""
|
||||
return f"{self.__class__.__name__}({self.serial_number})"
|
||||
|
||||
def _reset_connection_settings(self) -> None:
|
||||
@@ -250,8 +247,7 @@ class RealSenseCamera(Camera):
|
||||
|
||||
@check_if_already_connected
|
||||
def connect(self, warmup: bool = True) -> None:
|
||||
"""
|
||||
Connects to the RealSense camera specified in the configuration.
|
||||
"""Connects to the RealSense camera specified in the configuration.
|
||||
|
||||
Initializes the RealSense pipeline, configures the required streams (color
|
||||
and optionally depth), starts the pipeline, and validates the actual stream settings.
|
||||
@@ -270,7 +266,6 @@ class RealSenseCamera(Camera):
|
||||
ConnectionError: If the camera is found but fails to start the pipeline or no RealSense devices are detected at all.
|
||||
RuntimeError: If the pipeline starts but fails to apply requested settings.
|
||||
"""
|
||||
|
||||
if not warmup:
|
||||
self._open_pipeline()
|
||||
logger.info(f"{self} connected.")
|
||||
@@ -306,8 +301,7 @@ class RealSenseCamera(Camera):
|
||||
|
||||
@staticmethod
|
||||
def find_cameras() -> list[dict[str, Any]]:
|
||||
"""
|
||||
Detects available Intel RealSense cameras connected to the system.
|
||||
"""Detects available Intel RealSense cameras connected to the system.
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: A list of dictionaries,
|
||||
@@ -406,7 +400,6 @@ class RealSenseCamera(Camera):
|
||||
Raises:
|
||||
DeviceNotConnectedError: If device is not connected.
|
||||
"""
|
||||
|
||||
if self.rs_profile is None:
|
||||
raise RuntimeError(f"{self}: rs_profile must be initialized before use.")
|
||||
|
||||
@@ -544,8 +537,7 @@ class RealSenseCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def read_depth(self, timeout_ms: int = 200) -> NDArray[Any]:
|
||||
"""
|
||||
Reads a single frame (depth) synchronously from the camera.
|
||||
"""Reads a single frame (depth) synchronously from the camera.
|
||||
|
||||
This is a blocking call. It waits for a coherent set of frames (depth)
|
||||
from the camera hardware via the RealSense pipeline.
|
||||
@@ -583,8 +575,7 @@ class RealSenseCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def read(self, color_mode: ColorMode | None = None, timeout_ms: int = 0) -> NDArray[Any]:
|
||||
"""
|
||||
Reads a single frame (color) synchronously from the camera.
|
||||
"""Reads a single frame (color) synchronously from the camera.
|
||||
|
||||
This is a blocking call. It waits for a coherent set of frames (color)
|
||||
from the camera hardware via the RealSense pipeline.
|
||||
@@ -598,7 +589,6 @@ class RealSenseCamera(Camera):
|
||||
RuntimeError: If reading frames from the pipeline fails or frames are invalid.
|
||||
ValueError: If an invalid `color_mode` is requested.
|
||||
"""
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
if color_mode is not None:
|
||||
@@ -622,11 +612,12 @@ class RealSenseCamera(Camera):
|
||||
return frame
|
||||
|
||||
def _postprocess_image(self, image: NDArray[Any], depth_frame: bool = False) -> NDArray[Any]:
|
||||
"""
|
||||
Applies color conversion, dimension validation, and rotation to a raw color frame.
|
||||
"""Applies color conversion, dimension validation, and rotation to a raw color frame.
|
||||
|
||||
Args:
|
||||
image (np.ndarray): The raw image frame (expected RGB format from RealSense).
|
||||
depth_frame (bool): Whether `image` is a single-channel depth frame rather than a 3-channel
|
||||
color frame.
|
||||
|
||||
Returns:
|
||||
np.ndarray: The processed image frame according to `self.color_mode` and `self.rotation`.
|
||||
@@ -636,7 +627,6 @@ class RealSenseCamera(Camera):
|
||||
RuntimeError: If the raw frame dimensions do not match the configured
|
||||
`width` and `height`.
|
||||
"""
|
||||
|
||||
if self.color_mode and self.color_mode not in (ColorMode.RGB, ColorMode.BGR):
|
||||
raise ValueError(
|
||||
f"Invalid requested color mode '{self.color_mode}'. Expected {ColorMode.RGB} or {ColorMode.BGR}."
|
||||
@@ -665,8 +655,7 @@ class RealSenseCamera(Camera):
|
||||
return processed_image
|
||||
|
||||
def _read_loop(self) -> None:
|
||||
"""
|
||||
Internal loop run by the background thread for asynchronous reading.
|
||||
"""Internal loop run by the background thread for asynchronous reading.
|
||||
|
||||
On each iteration:
|
||||
1. Reads a color/depth frame (blocking call with 10s timeout)
|
||||
@@ -790,8 +779,7 @@ class RealSenseCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
|
||||
"""
|
||||
Reads the latest available frame data (color) asynchronously.
|
||||
"""Reads the latest available frame data (color) asynchronously.
|
||||
|
||||
This method retrieves the most recent color frame captured by the background
|
||||
read thread. It does not block waiting for the camera hardware directly,
|
||||
@@ -811,7 +799,6 @@ class RealSenseCamera(Camera):
|
||||
TimeoutError: If no frame data becomes available within the specified timeout.
|
||||
RuntimeError: If the background thread died unexpectedly or another error occurs.
|
||||
"""
|
||||
|
||||
if not self.use_rgb:
|
||||
raise RuntimeError(f"{self}: cannot read color — camera was configured with use_rgb=False.")
|
||||
|
||||
@@ -897,15 +884,13 @@ class RealSenseCamera(Camera):
|
||||
return self._read_latest(max_age_ms=max_age_ms, read_depth=True)
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""
|
||||
Disconnects from the camera, stops the pipeline, and cleans up resources.
|
||||
"""Disconnects from the camera, stops the pipeline, and cleans up resources.
|
||||
|
||||
Stops the background read thread (if running) and stops the RealSense pipeline.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the camera is already disconnected (pipeline not running).
|
||||
"""
|
||||
|
||||
if not self.is_connected and self.thread is None:
|
||||
raise DeviceNotConnectedError(
|
||||
f"Attempted to disconnect {self}, but it appears already disconnected."
|
||||
|
||||
@@ -36,28 +36,39 @@ class RealSenseCameraConfig(CameraConfig):
|
||||
RealSenseCameraConfig("0123456789", 30, 640, 480, rotation=Cv2Rotation.ROTATE_90) # With 90° rotation
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **fps** -- Requested frames per second for the color stream.
|
||||
- **width** -- Requested frame width in pixels for the color stream.
|
||||
- **height** -- Requested frame height in pixels for the color stream.
|
||||
- **serial_number_or_name** (`str`) -- Unique serial number or human-readable name to identify the
|
||||
camera.
|
||||
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
- **use_rgb** (`bool`) -- Whether to enable the color stream. Defaults to True.
|
||||
- **use_depth** (`bool`) -- Whether to enable depth stream. Defaults to False.
|
||||
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no
|
||||
rotation.
|
||||
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds)
|
||||
- **exposure** (`int | None`) -- Manual exposure value for the color sensor. When set, auto-exposure
|
||||
is disabled and this fixed value is used. Valid ranges are camera-model specific and reported if the
|
||||
value is rejected. Defaults to None (leave unchanged).
|
||||
- **gain** (`int | None`) -- Manual gain value for the color sensor. When set, auto-exposure is
|
||||
disabled and this fixed gain is used, which also freezes exposure at its current value when no
|
||||
exposure is configured. Valid ranges are camera-model specific and reported if the value is
|
||||
rejected. Defaults to None (leave unchanged).
|
||||
- **white_balance** (`int | None`) -- Manual white balance value for the color sensor. When set, auto
|
||||
white balance is disabled and this fixed value is used. Valid ranges are camera-model specific and
|
||||
reported if the value is rejected. Defaults to None (leave unchanged).
|
||||
Args:
|
||||
serial_number_or_name (`str`):
|
||||
Unique serial number or human-readable name to identify the camera.
|
||||
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
|
||||
Color mode for image output.
|
||||
use_rgb (`bool`, *optional*, defaults to `True`):
|
||||
Whether to enable the color stream.
|
||||
use_depth (`bool`, *optional*, defaults to `False`):
|
||||
Whether to enable the depth stream.
|
||||
rotation (`Cv2Rotation`, *optional*, defaults to `Cv2Rotation.NO_ROTATION`):
|
||||
Image rotation setting (0°, 90°, 180°, or 270°).
|
||||
warmup_s (`int`, *optional*, defaults to 1):
|
||||
Time reading frames before returning from connect (in seconds).
|
||||
exposure (`int`, *optional*):
|
||||
Manual exposure value for the color sensor. When set, auto-exposure is disabled and this fixed
|
||||
value is used. Valid ranges are camera-model specific and reported if the value is rejected.
|
||||
`None` leaves auto-exposure unchanged.
|
||||
gain (`int`, *optional*):
|
||||
Manual gain value for the color sensor. When set, auto-exposure is disabled and this fixed
|
||||
gain is used, which also freezes exposure at its current value when no exposure is configured.
|
||||
Valid ranges are camera-model specific and reported if the value is rejected. `None` leaves it
|
||||
unchanged.
|
||||
white_balance (`int`, *optional*):
|
||||
Manual white balance value for the color sensor. When set, auto white balance is disabled and
|
||||
this fixed value is used. Valid ranges are camera-model specific and reported if the value is
|
||||
rejected. `None` leaves it unchanged.
|
||||
fps (`int`, *optional*):
|
||||
Requested frames per second for the color stream. The depth stream, when enabled, uses the
|
||||
same FPS. Either all of `fps`, `width`, and `height` must be set, or none of them.
|
||||
width (`int`, *optional*):
|
||||
Requested frame width in pixels for the color stream.
|
||||
height (`int`, *optional*):
|
||||
Requested frame height in pixels for the color stream.
|
||||
|
||||
Note:
|
||||
- Either name or serial_number must be specified.
|
||||
@@ -78,6 +89,12 @@ class RealSenseCameraConfig(CameraConfig):
|
||||
white_balance: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Normalize `color_mode`/`rotation` and validate the stream and sensor-option settings.
|
||||
|
||||
Raises:
|
||||
ValueError: If neither `use_rgb` nor `use_depth` is enabled, if a manual color sensor option
|
||||
is set without `use_rgb=True`, or if only some of `fps`/`width`/`height` are set.
|
||||
"""
|
||||
self.color_mode = ColorMode(self.color_mode)
|
||||
self.rotation = Cv2Rotation(self.rotation)
|
||||
|
||||
|
||||
@@ -23,6 +23,33 @@ from .configs import CameraConfig, Cv2Rotation
|
||||
|
||||
|
||||
def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[str, Camera]:
|
||||
"""Instantiate one [`~cameras.Camera`] per entry in a mapping of configs.
|
||||
|
||||
Dispatches on each config's registered `type` to build the matching backend class. This only
|
||||
constructs the camera objects; call [`~cameras.Camera.connect`] on each before use.
|
||||
|
||||
Args:
|
||||
camera_configs (`dict[str, CameraConfig]`):
|
||||
Camera configs keyed by the name each camera should be identified by, e.g. in a robot's
|
||||
observation features.
|
||||
|
||||
Returns:
|
||||
`dict[str, Camera]`: A camera instance per key, in the same order as `camera_configs`.
|
||||
|
||||
Raises:
|
||||
ValueError: If a config's type is not a known backend and building it via the generic device
|
||||
factory also fails.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.cameras.opencv import OpenCVCameraConfig
|
||||
>>> from lerobot.cameras.utils import make_cameras_from_configs
|
||||
>>> configs = {"top": OpenCVCameraConfig(index_or_path=0, fps=30, width=640, height=480)}
|
||||
>>> cameras = make_cameras_from_configs(configs)
|
||||
>>> list(cameras.keys())
|
||||
['top']
|
||||
```
|
||||
"""
|
||||
cameras: dict[str, Camera] = {}
|
||||
|
||||
for key, cfg in camera_configs.items():
|
||||
@@ -57,6 +84,15 @@ def make_cameras_from_configs(camera_configs: dict[str, CameraConfig]) -> dict[s
|
||||
|
||||
|
||||
def get_cv2_rotation(rotation: Cv2Rotation) -> int | None:
|
||||
"""Map a [`~cameras.Cv2Rotation`] to the OpenCV rotation flag `cv2.rotate` expects.
|
||||
|
||||
Args:
|
||||
rotation (`Cv2Rotation`):
|
||||
The configured rotation.
|
||||
|
||||
Returns:
|
||||
`int | None`: The matching `cv2.ROTATE_*` constant, or `None` for [`~cameras.Cv2Rotation.NO_ROTATION`].
|
||||
"""
|
||||
import cv2 # type: ignore # TODO: add type stubs for OpenCV
|
||||
|
||||
if rotation == Cv2Rotation.ROTATE_90:
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
ZMQCamera - Captures frames from remote cameras via ZeroMQ using JSON protocol in the
|
||||
following format:
|
||||
"""Captures frames from remote cameras via ZeroMQ.
|
||||
|
||||
Uses a JSON protocol of the following form:
|
||||
{
|
||||
"timestamps": {"camera_name": float},
|
||||
"images": {"camera_name": "<base64-jpeg>"}
|
||||
@@ -52,8 +52,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZMQCamera(Camera):
|
||||
"""
|
||||
Manages camera interactions via ZeroMQ for receiving frames from a remote server.
|
||||
"""Manages camera interactions via ZeroMQ for receiving frames from a remote server.
|
||||
|
||||
This class connects to a ZMQ Publisher, subscribes to frame topics, and decodes
|
||||
incoming JSON messages containing Base64 encoded images. It supports both
|
||||
@@ -81,6 +80,11 @@ class ZMQCamera(Camera):
|
||||
"""
|
||||
|
||||
def __init__(self, config: ZMQCameraConfig):
|
||||
"""Initialize the camera with the given configuration.
|
||||
|
||||
Args:
|
||||
config: Camera configuration, including the image server's address and port.
|
||||
"""
|
||||
require_package("pyzmq", extra="pyzmq-dep", import_name="zmq")
|
||||
super().__init__(config)
|
||||
|
||||
@@ -105,6 +109,7 @@ class ZMQCamera(Camera):
|
||||
self.new_frame_event: Event = Event()
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Return a short representation naming the camera and its server address."""
|
||||
return f"ZMQCamera({self.camera_name}@{self.server_address}:{self.port})"
|
||||
|
||||
@property
|
||||
@@ -120,7 +125,6 @@ class ZMQCamera(Camera):
|
||||
warmup (bool): If True, waits for the camera to provide at least one
|
||||
valid frame before returning. Defaults to True.
|
||||
"""
|
||||
|
||||
logger.info(f"Connecting to {self}...")
|
||||
|
||||
try:
|
||||
@@ -171,15 +175,11 @@ class ZMQCamera(Camera):
|
||||
|
||||
@staticmethod
|
||||
def find_cameras() -> list[dict[str, Any]]:
|
||||
"""
|
||||
Detection not implemented for ZMQ cameras. These cameras require manual configuration (server address/port).
|
||||
"""
|
||||
"""Detection not implemented for ZMQ cameras; they require manual server address/port configuration."""
|
||||
raise NotImplementedError("Camera detection is not implemented for ZMQ cameras.")
|
||||
|
||||
def _read_from_hardware(self) -> NDArray[Any]:
|
||||
"""
|
||||
Reads a single frame directly from the ZMQ socket.
|
||||
"""
|
||||
"""Reads a single frame directly from the ZMQ socket."""
|
||||
if not self.is_connected or self.socket is None:
|
||||
raise DeviceNotConnectedError(f"{self} is not connected.")
|
||||
|
||||
@@ -215,8 +215,7 @@ class ZMQCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def read(self, color_mode: ColorMode | None = None) -> NDArray[Any]:
|
||||
"""
|
||||
Reads a single frame synchronously from the camera.
|
||||
"""Reads a single frame synchronously from the camera.
|
||||
|
||||
This is a blocking call. It waits for the next available frame from the
|
||||
camera background thread.
|
||||
@@ -243,9 +242,7 @@ class ZMQCamera(Camera):
|
||||
return frame
|
||||
|
||||
def _read_loop(self) -> None:
|
||||
"""
|
||||
Internal loop run by the background thread for asynchronous reading.
|
||||
"""
|
||||
"""Internal loop run by the background thread for asynchronous reading."""
|
||||
stop_event = self.stop_event
|
||||
if stop_event is None:
|
||||
raise RuntimeError(f"{self}: stop_event is not initialized.")
|
||||
@@ -306,8 +303,7 @@ class ZMQCamera(Camera):
|
||||
|
||||
@check_if_not_connected
|
||||
def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
|
||||
"""
|
||||
Reads the latest available frame asynchronously.
|
||||
"""Reads the latest available frame asynchronously.
|
||||
|
||||
Args:
|
||||
timeout_ms (float): Maximum time in milliseconds to wait for a frame
|
||||
@@ -321,7 +317,6 @@ class ZMQCamera(Camera):
|
||||
TimeoutError: If no frame data becomes available within the specified timeout.
|
||||
RuntimeError: If the background thread is not running.
|
||||
"""
|
||||
|
||||
if self.thread is None or not self.thread.is_alive():
|
||||
raise RuntimeError(f"{self} read thread is not running.")
|
||||
|
||||
@@ -353,7 +348,6 @@ class ZMQCamera(Camera):
|
||||
DeviceNotConnectedError: If the camera is not connected.
|
||||
RuntimeError: If the camera is connected but has not captured any frames yet.
|
||||
"""
|
||||
|
||||
if self.thread is None or not self.thread.is_alive():
|
||||
raise RuntimeError(f"{self} read thread is not running.")
|
||||
|
||||
|
||||
@@ -24,6 +24,32 @@ __all__ = ["ZMQCameraConfig", "ColorMode"]
|
||||
@CameraConfig.register_subclass("zmq")
|
||||
@dataclass
|
||||
class ZMQCameraConfig(CameraConfig):
|
||||
"""Configuration for a camera served over a ZeroMQ socket by `ImageServer`.
|
||||
|
||||
Use this to read frames from a camera attached to a different machine (e.g. a Raspberry Pi on a
|
||||
robot), which streams JPEG-encoded frames to this config's `server_address`/`port` over ZMQ.
|
||||
|
||||
Args:
|
||||
server_address (`str`):
|
||||
Address of the machine running the image server, e.g. `"192.168.1.50"`.
|
||||
port (`int`, *optional*, defaults to 5555):
|
||||
TCP port the image server is publishing on. Must be between 1 and 65535.
|
||||
camera_name (`str`, *optional*, defaults to `"zmq_camera"`):
|
||||
Name used to identify this camera in logs and observation keys.
|
||||
color_mode (`ColorMode`, *optional*, defaults to `ColorMode.RGB`):
|
||||
Color mode for the decoded frames.
|
||||
timeout_ms (`int`, *optional*, defaults to 5000):
|
||||
How long to wait for a frame before raising a timeout, in milliseconds. Must be positive.
|
||||
warmup_s (`int`, *optional*, defaults to 1):
|
||||
Time spent reading frames before returning from connect, in seconds.
|
||||
fps (`int`, *optional*):
|
||||
Requested frames per second. `None` leaves it at the server's own rate.
|
||||
width (`int`, *optional*):
|
||||
Requested frame width in pixels. `None` leaves it at the server's own resolution.
|
||||
height (`int`, *optional*):
|
||||
Requested frame height in pixels. `None` leaves it at the server's own resolution.
|
||||
"""
|
||||
|
||||
server_address: str
|
||||
port: int = 5555
|
||||
camera_name: str = "zmq_camera"
|
||||
@@ -32,6 +58,12 @@ class ZMQCameraConfig(CameraConfig):
|
||||
warmup_s: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Normalize `color_mode` and validate the socket settings.
|
||||
|
||||
Raises:
|
||||
ValueError: If `timeout_ms` is not positive, `server_address` is empty, or `port` is outside
|
||||
the 1-65535 range.
|
||||
"""
|
||||
self.color_mode = ColorMode(self.color_mode)
|
||||
|
||||
if self.timeout_ms <= 0:
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Streams camera images over ZMQ.
|
||||
"""Streams camera images over ZMQ.
|
||||
|
||||
Uses lerobot's OpenCVCamera for capture, encodes images to base64 and sends them over ZMQ.
|
||||
"""
|
||||
|
||||
@@ -47,6 +47,12 @@ class CameraCaptureThread:
|
||||
"""Background thread that continuously captures and encodes frames from a camera."""
|
||||
|
||||
def __init__(self, camera: OpenCVCamera, name: str):
|
||||
"""Initialize the capture thread for one camera.
|
||||
|
||||
Args:
|
||||
camera: The already-connected camera to read frames from.
|
||||
name: Name used to identify this camera in log messages.
|
||||
"""
|
||||
self.camera = camera
|
||||
self.name = name
|
||||
self.latest_encoded: str | None = None # Pre-encoded JPEG as base64
|
||||
@@ -89,7 +95,21 @@ class CameraCaptureThread:
|
||||
|
||||
|
||||
class ImageServer:
|
||||
"""Reads from one or more local OpenCV cameras and publishes their frames over a ZMQ `PUB` socket.
|
||||
|
||||
Runs on the machine physically connected to the cameras (e.g. a Raspberry Pi on a robot); pair with
|
||||
[`~cameras.zmq.ZMQCamera`] on the machine that consumes the stream.
|
||||
"""
|
||||
|
||||
def __init__(self, config: dict, port: int = 5555):
|
||||
"""Connect every camera listed in `config` and open the publishing socket.
|
||||
|
||||
Args:
|
||||
config: A mapping with an optional `"fps"` (the publish loop rate, not the camera capture
|
||||
rate) and a `"cameras"` mapping of camera name to a dict with `"device_id"`, `"shape"`
|
||||
(`[height, width]`), and `"fourcc"` keys.
|
||||
port: TCP port to publish on.
|
||||
"""
|
||||
# fps controls the publish loop rate (how often frames are sent over ZMQ), not the camera capture rate
|
||||
self.fps = config.get("fps", 30)
|
||||
self.cameras: dict[str, OpenCVCamera] = {}
|
||||
@@ -124,6 +144,11 @@ class ImageServer:
|
||||
logger.info(f"ImageServer running on port {port}")
|
||||
|
||||
def run(self):
|
||||
"""Start each camera's capture thread and publish frames until interrupted.
|
||||
|
||||
Blocks until `KeyboardInterrupt`, then stops the capture threads, disconnects the cameras, and
|
||||
closes the socket.
|
||||
"""
|
||||
frame_count = 0
|
||||
frame_times = deque(maxlen=60)
|
||||
last_published_ts: dict[str, float] = {}
|
||||
|
||||
@@ -32,16 +32,13 @@ class RandomSubsetApply(Transform):
|
||||
"""Apply a random subset of N transformations from a list of transformations.
|
||||
|
||||
Args:
|
||||
transforms (`Sequence`):
|
||||
List of transformations.
|
||||
p (`list[float] | None`, *optional*):
|
||||
Multinomial probabilities (with no replacement) used for sampling the transform. Normalized if
|
||||
they don't already sum to 1. `None` gives all transforms the same probability.
|
||||
n_subset (`int | None`, *optional*):
|
||||
Number of transformations to apply. Must be in `[1, len(transforms)]`. `None` applies all of
|
||||
them.
|
||||
random_order (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply the sampled transformations in a random order.
|
||||
transforms: list of transformations.
|
||||
p: represents the multinomial probabilities (with no replacement) used for sampling the transform.
|
||||
If the sum of the weights is not 1, they will be normalized. If ``None`` (default), all transforms
|
||||
have the same probability.
|
||||
n_subset: number of transformations to apply. If ``None``, all transforms are applied.
|
||||
Must be in [1, len(transforms)].
|
||||
random_order: apply transformations in a random order.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -51,12 +48,6 @@ class RandomSubsetApply(Transform):
|
||||
n_subset: int | None = None,
|
||||
random_order: bool = False,
|
||||
) -> None:
|
||||
"""Validate and store the transform pool, sampling weights, and subset size.
|
||||
|
||||
Raises:
|
||||
TypeError: If `transforms` is not a sequence, or `n_subset` is not an int or `None`.
|
||||
ValueError: If `p`'s length doesn't match `transforms`, or `n_subset` is out of range.
|
||||
"""
|
||||
super().__init__()
|
||||
if not isinstance(transforms, Sequence):
|
||||
raise TypeError("Argument transforms should be a sequence of callables")
|
||||
@@ -83,7 +74,6 @@ class RandomSubsetApply(Transform):
|
||||
self.selected_transforms: list[Callable[..., Any]] = []
|
||||
|
||||
def forward(self, *inputs: Any) -> Any:
|
||||
"""Sample a subset of `self.transforms` and apply them in sequence to `inputs`."""
|
||||
needs_unpacking = len(inputs) > 1
|
||||
|
||||
selected_indices = torch.multinomial(torch.tensor(self.p), self.n_subset)
|
||||
@@ -99,7 +89,6 @@ class RandomSubsetApply(Transform):
|
||||
return outputs
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
"""Return the constructor arguments shown in `repr(self)`."""
|
||||
return (
|
||||
f"transforms={self.transforms}, "
|
||||
f"p={self.p}, "
|
||||
@@ -119,18 +108,16 @@ class SharpnessJitter(Transform):
|
||||
A sharpness_factor of 0 gives a blurred image, 1 gives the original image while 2 increases the sharpness
|
||||
by a factor of 2.
|
||||
|
||||
If the input is a `torch.Tensor`, it is expected to have `[..., 1 or 3, H, W]` shape, where `...` means
|
||||
an arbitrary number of leading dimensions.
|
||||
If the input is a :class:`torch.Tensor`,
|
||||
it is expected to have [..., 1 or 3, H, W] shape, where ... means an arbitrary number of leading dimensions.
|
||||
|
||||
Args:
|
||||
sharpness (`float | collections.abc.Sequence[float]`):
|
||||
How much to jitter sharpness. `sharpness_factor` is chosen uniformly from
|
||||
`[max(0, 1 - sharpness), 1 + sharpness]`, or the given `[min, max]`. Values must be
|
||||
non-negative.
|
||||
sharpness: How much to jitter sharpness. sharpness_factor is chosen uniformly from
|
||||
[max(0, 1 - sharpness), 1 + sharpness] or the given
|
||||
[min, max]. Should be non negative numbers.
|
||||
"""
|
||||
|
||||
def __init__(self, sharpness: float | Sequence[float]) -> None:
|
||||
"""Normalize `sharpness` into a `(min, max)` range to sample from on each call."""
|
||||
super().__init__()
|
||||
self.sharpness = self._check_input(sharpness)
|
||||
|
||||
@@ -151,12 +138,10 @@ class SharpnessJitter(Transform):
|
||||
return float(sharpness[0]), float(sharpness[1])
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a `sharpness_factor` uniformly from `self.sharpness`."""
|
||||
sharpness_factor = torch.empty(1).uniform_(self.sharpness[0], self.sharpness[1]).item()
|
||||
return {"sharpness_factor": sharpness_factor}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Adjust `inpt`'s sharpness by `params["sharpness_factor"]`."""
|
||||
sharpness_factor = params["sharpness_factor"]
|
||||
return self._call_kernel(F.adjust_sharpness, inpt, sharpness_factor=sharpness_factor)
|
||||
|
||||
@@ -168,17 +153,10 @@ class GaussianNoise(Transform):
|
||||
Common in real-robot setups where wrist cameras operate in suboptimal lighting.
|
||||
|
||||
Args:
|
||||
std (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(5.0, 25.0)`):
|
||||
Range `(min, max)` for the noise standard deviation, in pixel-value scale (0-255).
|
||||
std: Range (min, max) for noise standard deviation in pixel-value scale (0-255).
|
||||
"""
|
||||
|
||||
def __init__(self, std: float | Sequence[float] = (5.0, 25.0)) -> None:
|
||||
"""Normalize `std` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `std` is not a number or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `0 <= min <= max`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(std, (int, float)):
|
||||
self.std = (0.0, float(std))
|
||||
@@ -190,14 +168,12 @@ class GaussianNoise(Transform):
|
||||
raise ValueError(f"std must satisfy 0 <= min <= max, but got {self.std}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a noise `std` uniformly from `self.std`, plus a seed for reproducible noise."""
|
||||
return {
|
||||
"std": torch.empty(1).uniform_(self.std[0], self.std[1]).item(),
|
||||
"seed": torch.randint(0, torch.iinfo(torch.int64).max, ()).item(),
|
||||
}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Add Gaussian noise with `params["std"]` (in pixel-value scale) to `inpt`, if it's a float tensor."""
|
||||
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
|
||||
generator = torch.Generator(device=inpt.device).manual_seed(params["seed"])
|
||||
noise = torch.randn(inpt.shape, device=inpt.device, dtype=inpt.dtype, generator=generator)
|
||||
@@ -211,17 +187,10 @@ class MotionBlur(Transform):
|
||||
Generates a 1D averaging kernel along a random direction, applied via depthwise convolution.
|
||||
|
||||
Args:
|
||||
kernel_size (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3, 11)`):
|
||||
An odd kernel size, or a `(min, max)` range containing at least one odd kernel size.
|
||||
kernel_size: An odd kernel size or a range containing at least one odd kernel size.
|
||||
"""
|
||||
|
||||
def __init__(self, kernel_size: int | Sequence[int] = (3, 11)) -> None:
|
||||
"""Normalize `kernel_size` into a `(min, max)` range containing at least one odd value.
|
||||
|
||||
Raises:
|
||||
TypeError: If `kernel_size` is not an int or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `1 <= min <= max`, or contains no odd value.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(kernel_size, int):
|
||||
self.kernel_size = (kernel_size, kernel_size)
|
||||
@@ -236,7 +205,6 @@ class MotionBlur(Transform):
|
||||
raise ValueError(f"kernel_size range must contain an odd value, but got {self.kernel_size}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample an odd kernel size from `self.kernel_size` and a random blur direction in degrees."""
|
||||
num_odd_sizes = (self.kernel_size[1] - self._first_odd_kernel_size) // 2 + 1
|
||||
size_index = int(torch.randint(0, num_odd_sizes, ()).item())
|
||||
ks = self._first_odd_kernel_size + 2 * size_index
|
||||
@@ -244,11 +212,6 @@ class MotionBlur(Transform):
|
||||
return {"kernel_size": ks, "angle": angle}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Convolve `inpt` with a directional averaging kernel per `params`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -278,17 +241,10 @@ class JPEGCompression(Transform):
|
||||
Models quality degradation from video compression in network-streamed camera feeds.
|
||||
|
||||
Args:
|
||||
quality (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(15, 75)`):
|
||||
Range `(min, max)` for the JPEG quality factor. Lower values produce more artifacts.
|
||||
quality: Range (min, max) for JPEG quality factor (lower = more artifacts).
|
||||
"""
|
||||
|
||||
def __init__(self, quality: int | Sequence[int] = (15, 75)) -> None:
|
||||
"""Normalize `quality` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `quality` is not an int or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `1 <= min <= max <= 100`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(quality, int):
|
||||
self.quality = (quality, quality)
|
||||
@@ -300,16 +256,9 @@ class JPEGCompression(Transform):
|
||||
raise ValueError(f"quality must satisfy 1 <= min <= max <= 100, but got {self.quality}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a JPEG `quality` factor uniformly (as an int) from `self.quality`."""
|
||||
return {"quality": int(torch.randint(self.quality[0], self.quality[1] + 1, (1,)).item())}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Re-encode and decode `inpt` as JPEG at `params["quality"]`, introducing compression artifacts.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions, or with a channel count
|
||||
other than 1 or 3.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -335,12 +284,9 @@ class GaussianPatchBrightness(Transform):
|
||||
encountered in real robot workspaces with multiple light sources.
|
||||
|
||||
Args:
|
||||
num_patches (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(1, 4)`):
|
||||
Range `(min, max)` for the number of brightness patches.
|
||||
sigma_range (`Sequence`, *optional*, defaults to `(0.05, 0.25)`):
|
||||
Range `(min, max)` for each patch's Gaussian sigma, as a fraction of image size.
|
||||
factor_range (`Sequence`, *optional*, defaults to `(0.4, 1.6)`):
|
||||
Range `(min, max)` for the brightness factor; below 1 darkens, above 1 brightens.
|
||||
num_patches: Range (min, max) for number of brightness patches.
|
||||
sigma_range: Range for Gaussian sigma as fraction of image size.
|
||||
factor_range: Range for brightness factor (< 1 darkens, > 1 brightens).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -349,12 +295,6 @@ class GaussianPatchBrightness(Transform):
|
||||
sigma_range: Sequence[float] = (0.05, 0.25),
|
||||
factor_range: Sequence[float] = (0.4, 1.6),
|
||||
) -> None:
|
||||
"""Validate and store the patch count, size, and brightness ranges.
|
||||
|
||||
Raises:
|
||||
TypeError: If any range argument is not the expected type or length.
|
||||
ValueError: If any range does not satisfy `min <= max` within its valid bounds.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(num_patches, int):
|
||||
self.num_patches = (num_patches, num_patches)
|
||||
@@ -376,7 +316,6 @@ class GaussianPatchBrightness(Transform):
|
||||
raise ValueError(f"factor_range must satisfy 0 <= min <= max, but got {self.factor_range}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a random number of patches, each with a random center, sigma, and brightness factor."""
|
||||
n = int(torch.randint(self.num_patches[0], self.num_patches[1] + 1, (1,)).item())
|
||||
return {
|
||||
"centers": torch.rand(n, 2).tolist(),
|
||||
@@ -385,7 +324,6 @@ class GaussianPatchBrightness(Transform):
|
||||
}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Multiply `inpt` by a mask of overlapping Gaussian brightness patches per `params`."""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
h, w = inpt.shape[-2:]
|
||||
@@ -409,17 +347,10 @@ class RandomShadow(Transform):
|
||||
Symmetric: randomly brightens or darkens to prevent BatchNorm stats shift.
|
||||
|
||||
Args:
|
||||
opacity (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.3, 0.6)`):
|
||||
Range `(min, max)` for the shadow/highlight opacity.
|
||||
opacity: Range (min, max) for shadow/highlight opacity.
|
||||
"""
|
||||
|
||||
def __init__(self, opacity: float | Sequence[float] = (0.3, 0.6)) -> None:
|
||||
"""Normalize `opacity` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `opacity` is not a number or a length-2 sequence.
|
||||
ValueError: If the resulting range does not satisfy `0 <= min <= max <= 1`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(opacity, (int, float)):
|
||||
self.opacity = (float(opacity), float(opacity))
|
||||
@@ -431,7 +362,6 @@ class RandomShadow(Transform):
|
||||
raise ValueError(f"opacity must satisfy 0 <= min <= max <= 1, but got {self.opacity}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample the shadow band's opacity, horizontal position/width, and darken-vs-brighten direction."""
|
||||
return {
|
||||
"opacity": torch.empty(1).uniform_(self.opacity[0], self.opacity[1]).item(),
|
||||
"start": torch.rand(1).item(),
|
||||
@@ -440,11 +370,6 @@ class RandomShadow(Transform):
|
||||
}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Multiply `inpt` by a soft-edged vertical band mask per `params`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -476,14 +401,10 @@ class CoarseDropout(Transform):
|
||||
during robot manipulation.
|
||||
|
||||
Args:
|
||||
max_holes (`int`, *optional*, defaults to 8):
|
||||
Maximum number of rectangular patches to drop.
|
||||
max_height_frac (`float`, *optional*, defaults to 0.07):
|
||||
Maximum patch height, as a fraction of image height.
|
||||
max_width_frac (`float`, *optional*, defaults to 0.07):
|
||||
Maximum patch width, as a fraction of image width.
|
||||
fill_value (`float`, *optional*, defaults to 0.0):
|
||||
Value to fill dropped regions with.
|
||||
max_holes: Maximum number of rectangular patches to drop.
|
||||
max_height_frac: Maximum patch height as fraction of image height.
|
||||
max_width_frac: Maximum patch width as fraction of image width.
|
||||
fill_value: Value to fill dropped regions with.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -493,12 +414,6 @@ class CoarseDropout(Transform):
|
||||
max_width_frac: float = 0.07,
|
||||
fill_value: float = 0.0,
|
||||
) -> None:
|
||||
"""Validate and store the dropout patch count, size limits, and fill value.
|
||||
|
||||
Raises:
|
||||
TypeError: If `max_holes` is not an int.
|
||||
ValueError: If any argument is out of its valid range.
|
||||
"""
|
||||
super().__init__()
|
||||
if not isinstance(max_holes, int):
|
||||
raise TypeError("max_holes must be an int.")
|
||||
@@ -516,7 +431,6 @@ class CoarseDropout(Transform):
|
||||
self.fill_value = fill_value
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a random number of dropout patches, each with a random size and position."""
|
||||
n = int(torch.randint(1, self.max_holes + 1, (1,)).item())
|
||||
sizes = torch.rand(n, 2)
|
||||
sizes[:, 0] *= self.max_height_frac
|
||||
@@ -524,11 +438,6 @@ class CoarseDropout(Transform):
|
||||
return {"sizes": sizes.tolist(), "positions": torch.rand(n, 2).tolist()}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Fill the rectangular patches described by `params` in `inpt` with `self.fill_value`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor with fewer than 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3:
|
||||
@@ -555,18 +464,10 @@ class GammaCorrection(Transform):
|
||||
preventing BatchNorm statistics shift.
|
||||
|
||||
Args:
|
||||
gamma (`float | collections.abc.Sequence[float]`, *optional*, defaults to `(0.5, 2.0)`):
|
||||
Range `(min, max)` for the gamma value. Values below 1 brighten, above 1 darken.
|
||||
gamma: Range (min, max) for gamma value. Values < 1 brighten, > 1 darken.
|
||||
"""
|
||||
|
||||
def __init__(self, gamma: float | Sequence[float] = (0.5, 2.0)) -> None:
|
||||
"""Normalize `gamma` into a log-symmetric `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `gamma` is not a number or a length-2 sequence.
|
||||
ValueError: If a single `gamma` is not positive, or the resulting range does not satisfy
|
||||
`0 < min <= max`.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(gamma, (int, float)):
|
||||
gamma = float(gamma)
|
||||
@@ -581,14 +482,12 @@ class GammaCorrection(Transform):
|
||||
raise ValueError(f"gamma must satisfy 0 < min <= max, but got {self.gamma}.")
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a `gamma` value log-uniformly from `self.gamma`."""
|
||||
log_lo = math.log(self.gamma[0])
|
||||
log_hi = math.log(self.gamma[1])
|
||||
gamma = math.exp(torch.empty(1).uniform_(log_lo, log_hi).item())
|
||||
return {"gamma": gamma}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Raise `inpt` to the power `params["gamma"]`, if it's a float tensor."""
|
||||
if isinstance(inpt, torch.Tensor) and inpt.is_floating_point():
|
||||
return inpt.pow(params["gamma"]).clamp(0.0, 1.0)
|
||||
return inpt
|
||||
@@ -638,18 +537,11 @@ class PlanckianJitter(Transform):
|
||||
Reference: Zini et al., "Planckian Jitter", CVPR 2022 Workshop.
|
||||
|
||||
Args:
|
||||
temperature (`int | collections.abc.Sequence[int]`, *optional*, defaults to `(3000, 15000)`):
|
||||
A fixed color temperature, or a `(min, max)` range, in Kelvin. Supported values are between
|
||||
3000 K and 15000 K.
|
||||
temperature: A fixed color temperature or range in Kelvin. Supported values
|
||||
are between 3000 K and 15000 K.
|
||||
"""
|
||||
|
||||
def __init__(self, temperature: int | Sequence[int] = (3_000, 15_000)) -> None:
|
||||
"""Normalize `temperature` into a `(min, max)` range to sample from on each call.
|
||||
|
||||
Raises:
|
||||
TypeError: If `temperature` is not an int or a length-2 sequence.
|
||||
ValueError: If the resulting range falls outside `[3000, 15000]` Kelvin.
|
||||
"""
|
||||
super().__init__()
|
||||
if isinstance(temperature, int):
|
||||
self.temperature = (temperature, temperature)
|
||||
@@ -670,16 +562,10 @@ class PlanckianJitter(Transform):
|
||||
)
|
||||
|
||||
def make_params(self, flat_inputs: list[Any]) -> dict[str, Any]:
|
||||
"""Sample a color `temperature` in Kelvin uniformly from `self.temperature`."""
|
||||
temperature = int(torch.randint(self.temperature[0], self.temperature[1] + 1, ()).item())
|
||||
return {"temperature": temperature}
|
||||
|
||||
def transform(self, inpt: Any, params: dict[str, Any]) -> Any:
|
||||
"""Scale `inpt`'s red/blue channels per the black-body coefficients at `params["temperature"]`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `inpt` is a float tensor that isn't 3-channel with at least 3 dimensions.
|
||||
"""
|
||||
if not isinstance(inpt, torch.Tensor) or not inpt.is_floating_point():
|
||||
return inpt
|
||||
if inpt.ndim < 3 or inpt.shape[-3] != 3:
|
||||
@@ -727,18 +613,15 @@ _CUSTOM_TRANSFORMS: dict[str, type[Transform]] = {
|
||||
|
||||
@dataclass
|
||||
class ImageTransformConfig:
|
||||
"""Configuration for one entry in an [`~transforms.ImageTransformsConfig`]'s `tfs` mapping.
|
||||
|
||||
Args:
|
||||
weight (`float`, *optional*, defaults to 1.0):
|
||||
Multinomial probability (with no replacement) of sampling this transform. Normalized against
|
||||
the other transforms' weights if they don't already sum to 1.
|
||||
type (`str`, *optional*, defaults to `"Identity"`):
|
||||
Name of the transform class to build — either a class under `torchvision.transforms.v2` or one
|
||||
of the custom transforms in this module. Passed to
|
||||
[`~transforms.make_transform_from_config`].
|
||||
kwargs (`dict[str, Any]`, *optional*):
|
||||
Keyword arguments passed to the transform's constructor.
|
||||
"""
|
||||
For each transform, the following parameters are available:
|
||||
weight: This represents the multinomial probability (with no replacement)
|
||||
used for sampling the transform. If the sum of the weights is not 1,
|
||||
they will be normalized.
|
||||
type: The name of the class used. This is either a class available under torchvision.transforms.v2 or a
|
||||
custom transform defined here.
|
||||
kwargs: Lower & upper bound respectively used for sampling the transform's parameter
|
||||
(following uniform distribution) when it's applied.
|
||||
"""
|
||||
|
||||
weight: float = 1.0
|
||||
@@ -748,21 +631,11 @@ class ImageTransformConfig:
|
||||
|
||||
@dataclass
|
||||
class ImageTransformsConfig:
|
||||
"""Configuration for [`~transforms.ImageTransforms`], a random subset of image augmentations.
|
||||
|
||||
Transforms are standard [`torchvision.transforms.v2`](https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html)
|
||||
or custom transforms from this module, sampled via [`~transforms.RandomSubsetApply`].
|
||||
|
||||
Args:
|
||||
enable (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply transforms at all. `False` disables augmentation entirely.
|
||||
max_num_transforms (`int`, *optional*, defaults to 3):
|
||||
Maximum number of transforms (sampled from `tfs`) applied to each frame. Must be in
|
||||
`[1, len(tfs)]`.
|
||||
random_order (`bool`, *optional*, defaults to `False`):
|
||||
Whether to apply the sampled transforms in a random order, instead of the order in `tfs`.
|
||||
tfs (`dict[str, ImageTransformConfig]`, *optional*):
|
||||
The available transforms, keyed by name, with their sampling weight and constructor arguments.
|
||||
"""
|
||||
These transforms are all using standard torchvision.transforms.v2
|
||||
You can find out how these transformations affect images here:
|
||||
https://pytorch.org/vision/0.18/auto_examples/transforms/plot_transforms_illustrations.html
|
||||
We use a custom RandomSubsetApply container to sample them.
|
||||
"""
|
||||
|
||||
# Set this flag to `true` to enable transforms during training
|
||||
@@ -810,19 +683,6 @@ class ImageTransformsConfig:
|
||||
|
||||
|
||||
def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
|
||||
"""Instantiate the transform named by `cfg.type`, from `torchvision.transforms.v2` or this module.
|
||||
|
||||
Args:
|
||||
cfg (`ImageTransformConfig`):
|
||||
Configuration naming the transform class and its constructor arguments.
|
||||
|
||||
Returns:
|
||||
`Transform`: The instantiated transform.
|
||||
|
||||
Raises:
|
||||
ValueError: If `cfg.type` is not a `torchvision.transforms.v2` transform or one of this module's
|
||||
custom transforms.
|
||||
"""
|
||||
if cfg.type in _CUSTOM_TRANSFORMS:
|
||||
return _CUSTOM_TRANSFORMS[cfg.type](**cfg.kwargs)
|
||||
|
||||
@@ -838,20 +698,9 @@ def make_transform_from_config(cfg: ImageTransformConfig) -> Transform:
|
||||
|
||||
|
||||
class ImageTransforms(Transform):
|
||||
"""Composes a random subset of image augmentations from an [`~transforms.ImageTransformsConfig`].
|
||||
|
||||
Builds each enabled transform (weight > 0) named in `cfg.tfs`, then wraps them in a
|
||||
[`~transforms.RandomSubsetApply`] so a random subset is applied on each call. If `cfg.enable` is
|
||||
`False` or no transforms are enabled, this is equivalent to the identity transform.
|
||||
"""
|
||||
"""A class to compose image transforms based on configuration."""
|
||||
|
||||
def __init__(self, cfg: ImageTransformsConfig) -> None:
|
||||
"""Build the enabled transforms from `cfg` and wrap them in a random-subset sampler.
|
||||
|
||||
Args:
|
||||
cfg (`ImageTransformsConfig`):
|
||||
Configuration listing the available transforms and how many to sample per call.
|
||||
"""
|
||||
super().__init__()
|
||||
self._cfg = cfg
|
||||
|
||||
@@ -876,5 +725,4 @@ class ImageTransforms(Transform):
|
||||
)
|
||||
|
||||
def forward(self, *inputs: Any) -> Any:
|
||||
"""Apply the sampled subset of transforms (or the identity, if none are enabled) to `inputs`."""
|
||||
return self.tf(*inputs)
|
||||
|
||||
@@ -60,7 +60,7 @@ PATH_TO_LEROBOT = PATH_TO_REPO / "src" / "lerobot"
|
||||
# Modules whose public objects are checked. Add a module here once its docstrings follow the standard.
|
||||
MODULES_TO_CHECK = [
|
||||
"lerobot.robots",
|
||||
"lerobot.transforms",
|
||||
"lerobot.cameras",
|
||||
]
|
||||
|
||||
# Objects that do not yet follow the standard, so the check can be green from day one. Removing an entry
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
# `utils/check_doctest_list.py` does not care which way round it is.
|
||||
#
|
||||
# Keep alphabetically sorted: `make check-doctest-list` enforces it, `make fix-docstrings` sorts it.
|
||||
src/lerobot/cameras/utils.py
|
||||
src/lerobot/motors/motors_bus.py
|
||||
src/lerobot/robots/robot.py
|
||||
src/lerobot/robots/so_follower/config_so_follower.py
|
||||
|
||||
Reference in New Issue
Block a user