diff --git a/docs/source/earthrover_mini_plus.mdx b/docs/source/earthrover_mini_plus.mdx index 508c0e3a9..f3b324093 100644 --- a/docs/source/earthrover_mini_plus.mdx +++ b/docs/source/earthrover_mini_plus.mdx @@ -194,7 +194,7 @@ lerobot-record \ --dataset.single_task="Navigate around obstacles" \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --display_data=true ``` diff --git a/docs/source/groot.mdx b/docs/source/groot.mdx index a10b5e369..3ab202fb2 100644 --- a/docs/source/groot.mdx +++ b/docs/source/groot.mdx @@ -124,7 +124,7 @@ lerobot-rollout\ --dataset.single_task="Grab and handover the red cube to the other arm" \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --policy.path=/groot-bimanual \ # your trained model --duration=600 ``` diff --git a/docs/source/hope_jr.mdx b/docs/source/hope_jr.mdx index 1f3b08fd7..c29a9f216 100644 --- a/docs/source/hope_jr.mdx +++ b/docs/source/hope_jr.mdx @@ -232,7 +232,7 @@ lerobot-record \ --dataset.private=true \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --display_data=true ``` @@ -278,6 +278,6 @@ lerobot-record \ --dataset.num_episodes=10 \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --policy.path=outputs/train/hopejr_hand/checkpoints/last/pretrained_model ``` diff --git a/docs/source/il_robots.mdx b/docs/source/il_robots.mdx index 53ae5af82..6cf445297 100644 --- a/docs/source/il_robots.mdx +++ b/docs/source/il_robots.mdx @@ -207,7 +207,7 @@ lerobot-record \ --dataset.num_episodes=5 \ --dataset.single_task="Grab the black cube" \ --dataset.streaming_encoding=true \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --dataset.encoder_threads=2 ``` diff --git a/docs/source/lerobot-dataset-v3.mdx b/docs/source/lerobot-dataset-v3.mdx index 21cb232d3..0647af0b0 100644 --- a/docs/source/lerobot-dataset-v3.mdx +++ b/docs/source/lerobot-dataset-v3.mdx @@ -44,7 +44,7 @@ lerobot-record \ --dataset.num_episodes=5 \ --dataset.single_task="Grab the black cube" \ --dataset.streaming_encoding=true \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --dataset.encoder_threads=2 ``` diff --git a/docs/source/reachy2.mdx b/docs/source/reachy2.mdx index 4b08569db..7f975af43 100644 --- a/docs/source/reachy2.mdx +++ b/docs/source/reachy2.mdx @@ -161,7 +161,7 @@ lerobot-record \ --dataset.private=true \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --display_data=true ``` @@ -203,7 +203,7 @@ lerobot-record \ --dataset.private=true \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - # --dataset.camera_encoder.vcodec=auto \ + # --dataset.rgb_encoder.vcodec=auto \ --display_data=true ``` diff --git a/docs/source/streaming_video_encoding.mdx b/docs/source/streaming_video_encoding.mdx index 96e049eb3..0be32b717 100644 --- a/docs/source/streaming_video_encoding.mdx +++ b/docs/source/streaming_video_encoding.mdx @@ -17,7 +17,7 @@ This makes `save_episode()` near-instant (the video is already encoded by the ti | Parameter | CLI Flag | Type | Default | Description | | ----------------------- | --------------------------------- | ------------- | ------------- | ----------------------------------------------------------------- | | `streaming_encoding` | `--dataset.streaming_encoding` | `bool` | `True` | Enable real-time encoding during capture | -| `vcodec` | `--dataset.camera_encoder.vcodec` | `str` | `"libsvtav1"` | Video codec. `"auto"` detects best HW encoder | +| `vcodec` | `--dataset.rgb_encoder.vcodec` | `str` | `"libsvtav1"` | Video codec. `"auto"` detects best HW encoder | | `encoder_threads` | `--dataset.encoder_threads` | `int \| None` | `None` (auto) | Threads per encoder instance. `None` will leave the vcoded decide | | `encoder_queue_maxsize` | `--dataset.encoder_queue_maxsize` | `int` | `30` | Max buffered frames per camera (~1s at 30fps). Consumes RAM | @@ -82,15 +82,15 @@ Use HW encoding when: ### Available HW Encoders -| Encoder | Platform | Hardware | CLI Value | -| ------------------- | ------------- | ------------------------------------------------------------------------------------------------ | --------------------------------------------------- | -| `h264_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.camera_encoder.vcodec=h264_videotoolbox` | -| `hevc_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.camera_encoder.vcodec=hevc_videotoolbox` | -| `h264_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.camera_encoder.vcodec=h264_nvenc` | -| `hevc_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.camera_encoder.vcodec=hevc_nvenc` | -| `h264_vaapi` | Linux | Intel/AMD GPU | `--dataset.camera_encoder.vcodec=h264_vaapi` | -| `h264_qsv` | Linux/Windows | Intel Quick Sync | `--dataset.camera_encoder.vcodec=h264_qsv` | -| `auto` | Any | Probes the system for available HW encoders. Falls back to `libsvtav1` if no HW encoder is found | `--dataset.camera_encoder.vcodec=auto` | +| Encoder | Platform | Hardware | CLI Value | +| ------------------- | ------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------ | +| `h264_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.rgb_encoder.vcodec=h264_videotoolbox` | +| `hevc_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.rgb_encoder.vcodec=hevc_videotoolbox` | +| `h264_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.rgb_encoder.vcodec=h264_nvenc` | +| `hevc_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.rgb_encoder.vcodec=hevc_nvenc` | +| `h264_vaapi` | Linux | Intel/AMD GPU | `--dataset.rgb_encoder.vcodec=h264_vaapi` | +| `h264_qsv` | Linux/Windows | Intel Quick Sync | `--dataset.rgb_encoder.vcodec=h264_qsv` | +| `auto` | Any | Probes the system for available HW encoders. Falls back to `libsvtav1` if no HW encoder is found | `--dataset.rgb_encoder.vcodec=auto` | > [!NOTE] > In order to use the HW accelerated encoders you might need to upgrade your GPU drivers. @@ -100,15 +100,15 @@ Use HW encoding when: ## 5. Troubleshooting -| Symptom | Likely Cause | Fix | -| ------------------------------------------------------------------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| System freezes or choppy robot movement or Rerun visualization lag | CPU starved (100% load usage) | Close other apps, reduce encoding throughput, lower `encoder_threads`, use `h264`, use `display_data=False`. If the CPU continues to be at 100% then it might be insufficient for your setup, consider `--dataset.streaming_encoding=false` or HW encoding (`--dataset.camera_encoder.vcodec=auto`) | -| "Encoder queue full" warnings or dropped frames in dataset | Encoder can't keep up (Queue overflow) | If CPU is not at 100%: Increase `encoder_threads`, increase `encoder_queue_maxsize` or use HW encoding (`--dataset.camera_encoder.vcodec=auto`). | -| High RAM usage | Queue filling faster than encoding | `encoder_threads` too low or CPU insufficient. Reduce `encoder_queue_maxsize` or use HW encoding | -| Large video files | Using HW encoder or H.264 | Expected trade-off. Switch to `libsvtav1` if CPU allows | -| `save_episode()` still slow | `streaming_encoding` is `False` | Set `--dataset.streaming_encoding=true` | -| Encoder thread crash | Codec not available or invalid settings | Check `vcodec` is installed, try `--dataset.camera_encoder.vcodec=auto` | -| Recorded dataset is missing frames | CPU/GPU starvation or occasional load spikes | If ~5% of frames are missing, your system is likely overloaded — follow the recommendations above. If fewer frames are missing (~2%), they are probably due to occasional transient load spikes (often at startup) and can be considered expected. | +| Symptom | Likely Cause | Fix | +| ------------------------------------------------------------------ | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| System freezes or choppy robot movement or Rerun visualization lag | CPU starved (100% load usage) | Close other apps, reduce encoding throughput, lower `encoder_threads`, use `h264`, use `display_data=False`. If the CPU continues to be at 100% then it might be insufficient for your setup, consider `--dataset.streaming_encoding=false` or HW encoding (`--dataset.rgb_encoder.vcodec=auto`) | +| "Encoder queue full" warnings or dropped frames in dataset | Encoder can't keep up (Queue overflow) | If CPU is not at 100%: Increase `encoder_threads`, increase `encoder_queue_maxsize` or use HW encoding (`--dataset.rgb_encoder.vcodec=auto`). | +| High RAM usage | Queue filling faster than encoding | `encoder_threads` too low or CPU insufficient. Reduce `encoder_queue_maxsize` or use HW encoding | +| Large video files | Using HW encoder or H.264 | Expected trade-off. Switch to `libsvtav1` if CPU allows | +| `save_episode()` still slow | `streaming_encoding` is `False` | Set `--dataset.streaming_encoding=true` | +| Encoder thread crash | Codec not available or invalid settings | Check `vcodec` is installed, try `--dataset.rgb_encoder.vcodec=auto` | +| Recorded dataset is missing frames | CPU/GPU starvation or occasional load spikes | If ~5% of frames are missing, your system is likely overloaded — follow the recommendations above. If fewer frames are missing (~2%), they are probably due to occasional transient load spikes (often at startup) and can be considered expected. | ## 6. Recommended Configurations @@ -146,7 +146,7 @@ On very constrained systems, streaming encoding may compete too heavily with the # 2camsx 640x480x3 @30fps: Requires some tuning. # Use H.264, disable streaming, consider batching encoding -lerobot-record --dataset.camera_encoder.vcodec=h264 --dataset.streaming_encoding=false ... +lerobot-record --dataset.rgb_encoder.vcodec=h264 --dataset.streaming_encoding=false ... ``` ## 7. Closing note diff --git a/docs/source/using_dataset_tools.mdx b/docs/source/using_dataset_tools.mdx index 7c8df6e38..56334250f 100644 --- a/docs/source/using_dataset_tools.mdx +++ b/docs/source/using_dataset_tools.mdx @@ -118,10 +118,10 @@ lerobot-edit-dataset \ --repo_id lerobot/pusht_image \ --operation.type convert_image_to_video \ --operation.output_dir outputs/pusht_video \ - --operation.camera_encoder.vcodec libsvtav1 \ - --operation.camera_encoder.pix_fmt yuv420p \ - --operation.camera_encoder.g 2 \ - --operation.camera_encoder.crf 30 + --operation.rgb_encoder.vcodec libsvtav1 \ + --operation.rgb_encoder.pix_fmt yuv420p \ + --operation.rgb_encoder.g 2 \ + --operation.rgb_encoder.crf 30 # Convert a dataset that includes depth maps, customizing the depth encoder lerobot-edit-dataset \ @@ -157,37 +157,37 @@ lerobot-edit-dataset \ **Parameters:** - `output_dir`: Custom output directory (optional - by default uses `new_repo_id` or `{repo_id}_video`) -- `camera_encoder`: Video encoder settings applied to RGB cameras — all sub-fields accessible via `--operation.camera_encoder.`. See [Video Encoding Parameters](./video_encoding_parameters) for more details. +- `rgb_encoder`: Video encoder settings applied to RGB cameras — all sub-fields accessible via `--operation.rgb_encoder.`. See [Video Encoding Parameters](./video_encoding_parameters) for more details. - `depth_encoder`: Video encoder settings applied to depth-map cameras (e.g. from an Intel RealSense). In addition to the standard encoder fields it exposes the depth quantization knobs (`depth_min`, `depth_max`, `shift`, `use_log`), accessible via `--operation.depth_encoder.`. These quantization settings are persisted to the dataset metadata so depth can be dequantized back to physical units on load. See the [Depth streams](./video_encoding_parameters#depth-streams) section for details. - `episode_indices`: List of specific episodes to convert (default: all episodes) - `num_workers`: Number of parallel workers for processing (default: 4) -**Note:** The resulting dataset will be a proper LeRobotDataset with all cameras encoded as videos in the `videos/` directory, with parquet files containing only metadata (no raw image data). Depth-map cameras are detected automatically and routed to the `depth_encoder`, while RGB cameras use the `camera_encoder`. All episodes, stats, and tasks are preserved. +**Note:** The resulting dataset will be a proper LeRobotDataset with all cameras encoded as videos in the `videos/` directory, with parquet files containing only metadata (no raw image data). Depth-map cameras are detected automatically and routed to the `depth_encoder`, while RGB cameras use the `rgb_encoder`. All episodes, stats, and tasks are preserved. #### Re-encode Videos -Re-encode the videos of an existing video dataset with different encoder settings, without going back to raw frames. RGB videos use the `camera_encoder` and depth videos use the `depth_encoder`. Provide only the encoder(s) you want to re-encode; the other stream type is left untouched. +Re-encode the videos of an existing video dataset with different encoder settings, without going back to raw frames. RGB videos use the `rgb_encoder` and depth videos use the `depth_encoder`. Provide only the encoder(s) you want to re-encode; the other stream type is left untouched. ```bash # Re-encode all RGB videos with new settings (saves to lerobot/pusht_reencoded by default) lerobot-edit-dataset \ --repo_id lerobot/pusht \ --operation.type reencode_videos \ - --operation.camera_encoder.vcodec h264 \ - --operation.camera_encoder.pix_fmt yuv420p \ - --operation.camera_encoder.crf 23 + --operation.rgb_encoder.vcodec h264 \ + --operation.rgb_encoder.pix_fmt yuv420p \ + --operation.rgb_encoder.crf 23 # Re-encode both RGB and depth videos in a dataset with depth maps lerobot-edit-dataset \ --repo_id lerobot/pusht_depth \ --operation.type reencode_videos \ - --operation.camera_encoder.vcodec libx264 \ + --operation.rgb_encoder.vcodec libx264 \ --operation.depth_encoder.vcodec ffv1 ``` **Parameters:** -- `camera_encoder`: Encoder settings applied to every RGB video. Omit to skip re-encoding RGB videos. +- `rgb_encoder`: Encoder settings applied to every RGB video. Omit to skip re-encoding RGB videos. - `depth_encoder`: Encoder settings applied to every depth video. Omit to skip re-encoding depth videos. - `num_workers`: Number of parallel workers for processing. diff --git a/docs/source/video_encoding_parameters.mdx b/docs/source/video_encoding_parameters.mdx index 09cacf388..991ca8d26 100644 --- a/docs/source/video_encoding_parameters.mdx +++ b/docs/source/video_encoding_parameters.mdx @@ -2,15 +2,15 @@ When video storage is enabled, LeRobot stores each camera stream as an **MP4** file instead of saving one image file per timestep. Video encoding compresses across time, which usually cuts dataset size and I/O compared to a pile of PNG, while keeping MP4 — a format every player and loader understands. -Encoding frames into an MP4 is a full FFmpeg pipeline: choice of encoder, pixel format, GOP/keyframes, quality vs. speed, and optional extra encoder flags. Most of these knobs are user-tunable through `camera_encoder`, a nested `VideoEncoderConfig` (`lerobot.configs.video.VideoEncoderConfig`) passed through PyAV. +Encoding frames into an MP4 is a full FFmpeg pipeline: choice of encoder, pixel format, GOP/keyframes, quality vs. speed, and optional extra encoder flags. Most of these knobs are user-tunable through `rgb_encoder`, a nested `RGBEncoderConfig` (`lerobot.configs.video.RGBEncoderConfig`) passed through PyAV. -You can set these parameters from the CLI with `--dataset.camera_encoder.` (e.g. with `lerobot-record` or `lerobot-rollout`). The same block applies to every camera video stream in that run. +You can set these parameters from the CLI with `--dataset.rgb_encoder.` (e.g. with `lerobot-record` or `lerobot-rollout`). The same block applies to every camera video stream in that run. - Video storage must be on for `camera_encoder` to have any effect — + Video storage must be on for `rgb_encoder` to have any effect — `use_videos=True` in Python APIs, or `--dataset.video=true` on the CLI (the - recording default). With video off, inputs stay as images and `camera_encoder` - is ignored. + recording default). With video off, inputs stay as images and `rgb_encoder` is + ignored. For details on **when** frames are written vs. encoded (streaming vs. post-episode), queues, and other top-level `--dataset.*` switches, see [Streaming Video Encoding](./streaming_video_encoding). For an encoding-parameter comparison and experiments, see the [video-benchmark Space](https://huggingface.co/spaces/lerobot/video-benchmark). @@ -33,9 +33,9 @@ lerobot-record \ --dataset.single_task="Grab the cube" \ --dataset.streaming_encoding=true \ --dataset.encoder_threads=2 \ - --dataset.camera_encoder.vcodec=h264 \ - --dataset.camera_encoder.preset=fast \ - --dataset.camera_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \ + --dataset.rgb_encoder.vcodec=h264 \ + --dataset.rgb_encoder.preset=fast \ + --dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \ --display_data=true ``` @@ -50,7 +50,7 @@ Only override these parameters if you have a specific reason to, and measure the -All flags below are prefixed with `--dataset.camera_encoder.` on the CLI. +All flags below are prefixed with `--dataset.rgb_encoder.` on the CLI. | Parameter | Type | Default | Description | | --------------- | ---------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -172,7 +172,7 @@ Two sources contribute to the `info` block: This block is populated **once**, from the **first** episode. It assumes every - episode in the dataset was encoded with the same `camera_encoder`. Changing + episode in the dataset was encoded with the same `rgb_encoder`. Changing encoder settings partway through a recording is not supported — the `info.json` will only reflect the parameters used for the first episode. diff --git a/src/lerobot/configs/__init__.py b/src/lerobot/configs/__init__.py index 9f31dd63e..fa5942129 100644 --- a/src/lerobot/configs/__init__.py +++ b/src/lerobot/configs/__init__.py @@ -37,10 +37,11 @@ from .video import ( VALID_VIDEO_CODECS, VIDEO_ENCODER_INFO_KEYS, DepthEncoderConfig, + RGBEncoderConfig, VideoEncoderConfig, - camera_encoder_defaults, depth_encoder_defaults, encoder_config_from_video_info, + rgb_encoder_defaults, ) __all__ = [ @@ -61,9 +62,10 @@ __all__ = [ "WandBConfig", "load_recipe", "VideoEncoderConfig", + "RGBEncoderConfig", "DepthEncoderConfig", # Defaults - "camera_encoder_defaults", + "rgb_encoder_defaults", "depth_encoder_defaults", # Factories "encoder_config_from_video_info", diff --git a/src/lerobot/configs/dataset.py b/src/lerobot/configs/dataset.py index db4d713fd..7d30ca038 100644 --- a/src/lerobot/configs/dataset.py +++ b/src/lerobot/configs/dataset.py @@ -18,7 +18,7 @@ from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from .video import DepthEncoderConfig, VideoEncoderConfig, camera_encoder_defaults, depth_encoder_defaults +from .video import DepthEncoderConfig, RGBEncoderConfig, depth_encoder_defaults, rgb_encoder_defaults @dataclass @@ -58,8 +58,8 @@ class DatasetRecordConfig: # Set to 1 for immediate encoding (default behavior), or higher for batched encoding video_encoding_batch_size: int = 1 # Video encoder settings for camera MP4s (codec, quality, GOP, etc.). Tuned via CLI nested keys, - # e.g. ``--dataset.camera_encoder.vcodec=h264`` (see ``VideoEncoderConfig``). - camera_encoder: VideoEncoderConfig = field(default_factory=camera_encoder_defaults) + # e.g. ``--dataset.rgb_encoder.vcodec=h264`` (see ``RGBEncoderConfig``). + rgb_encoder: RGBEncoderConfig = field(default_factory=rgb_encoder_defaults) # Video encoder settings for depth-map MP4s (codec, quality, GOP, etc.). Tuned via CLI nested keys. depth_encoder: DepthEncoderConfig = field(default_factory=depth_encoder_defaults) # Enable streaming video encoding: encode frames in real-time during capture instead diff --git a/src/lerobot/configs/video.py b/src/lerobot/configs/video.py index 640c0409e..7542420f4 100644 --- a/src/lerobot/configs/video.py +++ b/src/lerobot/configs/video.py @@ -84,9 +84,10 @@ class VideoEncoderConfig: # Extra codec options merged last, e.g. {"tune": "film"}. extra_options: dict[str, Any] = field(default_factory=dict) - # Source-data channel count this encoder is expected to handle (3 for RGB, - # 1 for depth, etc.) - _DEFAULT_CHANNELS: ClassVar[int] = 3 + # Source-data channel count this encoder is expected to handle. ``None`` + # disables the pix_fmt channel-count check; concrete subclasses set it + # (3 for RGB, 1 for depth, etc.). + _DEFAULT_CHANNELS: ClassVar[int | None] = None def __post_init__(self) -> None: self.resolve_vcodec() @@ -246,9 +247,20 @@ class VideoEncoderConfig: return opts -def camera_encoder_defaults() -> VideoEncoderConfig: - """Return a :class:`VideoEncoderConfig` with RGB-camera defaults.""" - return VideoEncoderConfig() +@dataclass +class RGBEncoderConfig(VideoEncoderConfig): + """Encoder configuration for RGB camera streams. + + Identical to :class:`VideoEncoderConfig` but declares the 3-channel + source-data layout so ``pix_fmt`` is validated against RGB inputs. + """ + + _DEFAULT_CHANNELS: ClassVar[int] = 3 + + +def rgb_encoder_defaults() -> RGBEncoderConfig: + """Return a :class:`RGBEncoderConfig` with RGB-camera defaults.""" + return RGBEncoderConfig() @dataclass @@ -295,7 +307,7 @@ def encoder_config_from_video_info(video_info: dict | None) -> VideoEncoderConfi """Build the appropriate encoder config from a feature's ``info`` block. Dispatches to :class:`DepthEncoderConfig` when the dict marks the feature - as a depth map and to :class:`VideoEncoderConfig` + as a depth map and to :class:`RGBEncoderConfig` otherwise. Args: @@ -304,9 +316,9 @@ def encoder_config_from_video_info(video_info: dict | None) -> VideoEncoderConfi Returns: A :class:`DepthEncoderConfig` for depth features, otherwise a - :class:`VideoEncoderConfig`. + :class:`RGBEncoderConfig`. """ video_info = video_info or {} is_depth = bool(video_info.get("is_depth_map") or video_info.get("video.is_depth_map")) - cls: type[VideoEncoderConfig] = DepthEncoderConfig if is_depth else VideoEncoderConfig + cls: type[VideoEncoderConfig] = DepthEncoderConfig if is_depth else RGBEncoderConfig return cls.from_video_info(video_info) diff --git a/src/lerobot/datasets/dataset_tools.py b/src/lerobot/datasets/dataset_tools.py index dc8beeca0..c6f2b7f0c 100644 --- a/src/lerobot/datasets/dataset_tools.py +++ b/src/lerobot/datasets/dataset_tools.py @@ -40,9 +40,9 @@ from tqdm import tqdm from lerobot.configs import ( DepthEncoderConfig, VideoEncoderConfig, - camera_encoder_defaults, depth_encoder_defaults, encoder_config_from_video_info, + rgb_encoder_defaults, ) from lerobot.configs.video import DEPTH_ENCODER_INFO_FIELD_NAMES from lerobot.utils.constants import ACTION, HF_LEROBOT_HOME, OBS_IMAGE, OBS_STATE @@ -1670,7 +1670,7 @@ def convert_image_to_video_dataset( dataset: LeRobotDataset, output_dir: Path | None = None, repo_id: str | None = None, - camera_encoder: VideoEncoderConfig | None = None, + rgb_encoder: VideoEncoderConfig | None = None, depth_encoder: DepthEncoderConfig | None = None, episode_indices: list[int] | None = None, num_workers: int = 4, @@ -1689,8 +1689,8 @@ def convert_image_to_video_dataset( ``new_root`` in ``EditDatasetConfig``. repo_id: Converted dataset identifier. Equivalent to ``new_repo_id`` in ``EditDatasetConfig``. - camera_encoder: Video encoder settings applied to RGB cameras. When ``None``, - :func:`~lerobot.configs.video.camera_encoder_defaults` is used. + rgb_encoder: Video encoder settings applied to RGB cameras. When ``None``, + :func:`~lerobot.configs.video.rgb_encoder_defaults` is used. depth_encoder: Video encoder settings applied to depth-map cameras, including the quantization parameters persisted to the dataset metadata. When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used. @@ -1705,8 +1705,8 @@ def convert_image_to_video_dataset( Returns: A new :class:`LeRobotDataset` with images encoded as videos. """ - if camera_encoder is None: - camera_encoder = camera_encoder_defaults() + if rgb_encoder is None: + rgb_encoder = rgb_encoder_defaults() if depth_encoder is None: depth_encoder = depth_encoder_defaults() @@ -1733,7 +1733,7 @@ def convert_image_to_video_dataset( logging.info( f"Converting {len(episode_indices)} episodes with {len(img_keys)} cameras from {dataset.repo_id}" ) - logging.info(f"RGB video encoder: {camera_encoder}, depth video encoder: {depth_encoder}") + logging.info(f"RGB video encoder: {rgb_encoder}, depth video encoder: {depth_encoder}") # Create new features dict, converting image features to video features new_features = {} @@ -1795,7 +1795,7 @@ def convert_image_to_video_dataset( episode_lengths = {ep_idx: dataset.meta.episodes["length"][ep_idx] for ep_idx in episode_indices} for img_key in tqdm(img_keys, desc="Processing cameras"): - target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else camera_encoder + target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else rgb_encoder # Estimate size per frame by encoding a small calibration sample # This provides accurate compression ratio for the specific codec parameters @@ -1889,7 +1889,7 @@ def convert_image_to_video_dataset( # Update video info for all image keys (now videos). They are registered as # video features above, so update_video_info populates their (still-empty) info. for img_key in img_keys: - target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else camera_encoder + target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else rgb_encoder new_meta.update_video_info(video_key=img_key, video_encoder=target_encoder) write_info(new_meta.info, new_meta.root) @@ -1930,7 +1930,7 @@ def _reencode_video_worker(args: tuple) -> Path: def reencode_dataset( dataset: LeRobotDataset, - camera_encoder: VideoEncoderConfig | None = None, + rgb_encoder: VideoEncoderConfig | None = None, depth_encoder: DepthEncoderConfig | None = None, encoder_threads: int | None = None, num_workers: int | None = None, @@ -1942,7 +1942,7 @@ def reencode_dataset( Args: dataset: An existing :class:`LeRobotDataset` whose videos will be re-encoded. - camera_encoder: Target encoder configuration applied to every RGB video + rgb_encoder: Target encoder configuration applied to every RGB video file. If ``None``, re-encoding is skipped for RGB videos. depth_encoder: Target encoder configuration applied to every depth video file. If ``None``, re-encoding is skipped for depth videos. @@ -1961,14 +1961,14 @@ def reencode_dataset( video_keys_encoders_dict = {} video_keys_paths_dict = {} - if camera_encoder is None and depth_encoder is None: - raise ValueError("Either camera_encoder or depth_encoder must be provided") + if rgb_encoder is None and depth_encoder is None: + raise ValueError("Either rgb_encoder or depth_encoder must be provided") # Only re-encode if the videos are not already encoded with the given video encoding parameters for video_key in meta.video_keys: current_info = meta.info.features[video_key].get("info", {}) current_encoder = encoder_config_from_video_info(current_info) - target_encoder = depth_encoder if video_key in meta.depth_keys else camera_encoder + target_encoder = depth_encoder if video_key in meta.depth_keys else rgb_encoder if target_encoder is None: logging.info(f"No encoder provided for {video_key} video. Skipping re-encoding.") elif current_encoder != target_encoder: diff --git a/src/lerobot/datasets/dataset_writer.py b/src/lerobot/datasets/dataset_writer.py index 8467c31f1..5e51be00f 100644 --- a/src/lerobot/datasets/dataset_writer.py +++ b/src/lerobot/datasets/dataset_writer.py @@ -34,8 +34,8 @@ import torch from lerobot.configs import ( DepthEncoderConfig, VideoEncoderConfig, - camera_encoder_defaults, depth_encoder_defaults, + rgb_encoder_defaults, ) from .compute_stats import compute_episode_stats @@ -107,7 +107,7 @@ class DatasetWriter: self, meta: LeRobotDatasetMetadata, root: Path, - camera_encoder: VideoEncoderConfig | None, + rgb_encoder: VideoEncoderConfig | None, depth_encoder: DepthEncoderConfig | None, encoder_threads: int | None, batch_encoding_size: int, @@ -120,8 +120,8 @@ class DatasetWriter: meta: Dataset metadata instance (used for feature schema, chunk settings, and episode persistence). root: Local dataset root directory. - camera_encoder: Video encoder settings applied to RGB cameras. When - ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` is used. + rgb_encoder: Video encoder settings applied to RGB cameras. When + ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used. depth_encoder: Video encoder settings applied to depth cameras, including the quantization parameters. When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used. @@ -135,7 +135,7 @@ class DatasetWriter: """ self._meta = meta self._root = root - self._camera_encoder = camera_encoder or camera_encoder_defaults() + self._rgb_encoder = rgb_encoder or rgb_encoder_defaults() self._depth_encoder = depth_encoder or depth_encoder_defaults() self._encoder_threads = encoder_threads self._batch_encoding_size = batch_encoding_size @@ -321,9 +321,7 @@ class DatasetWriter: episode_index, self._root, self._meta.fps, - self._depth_encoder - if video_key in self._meta.depth_keys - else self._camera_encoder, + self._depth_encoder if video_key in self._meta.depth_keys else self._rgb_encoder, self._encoder_threads, ): video_key for video_key in self._meta.video_keys @@ -538,7 +536,7 @@ class DatasetWriter: video_key, video_encoder=self._depth_encoder if video_key in self._meta.depth_keys - else self._camera_encoder, + else self._rgb_encoder, ) write_info(self._meta.info, self._meta.root) @@ -613,7 +611,7 @@ class DatasetWriter: episode_index, self._root, self._meta.fps, - self._depth_encoder if is_depth else self._camera_encoder, + self._depth_encoder if is_depth else self._rgb_encoder, self._encoder_threads, ) diff --git a/src/lerobot/datasets/lerobot_dataset.py b/src/lerobot/datasets/lerobot_dataset.py index a694ff776..07fc6f83c 100644 --- a/src/lerobot/datasets/lerobot_dataset.py +++ b/src/lerobot/datasets/lerobot_dataset.py @@ -60,7 +60,7 @@ class LeRobotDataset(torch.utils.data.Dataset): return_uint8: bool = False, depth_output_unit: str = DEFAULT_DEPTH_UNIT, batch_encoding_size: int = 1, - camera_encoder: VideoEncoderConfig | None = None, + rgb_encoder: VideoEncoderConfig | None = None, depth_encoder: DepthEncoderConfig | None = None, encoder_threads: int | None = None, streaming_encoding: bool = False, @@ -185,8 +185,8 @@ class LeRobotDataset(torch.utils.data.Dataset): You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision. batch_encoding_size (int, optional): Number of episodes to accumulate before batch encoding videos. Set to 1 for immediate encoding (default), or higher for batched encoding. Defaults to 1. - camera_encoder (VideoEncoderConfig | None, optional): Video encoder settings for cameras - (codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` + rgb_encoder (VideoEncoderConfig | None, optional): Video encoder settings for cameras + (codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used by the writer. depth_encoder (DepthEncoderConfig | None, optional): Video encoder settings for depth cameras (codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` @@ -278,7 +278,7 @@ class LeRobotDataset(torch.utils.data.Dataset): if streaming_encoding and len(self.meta.video_keys) > 0: streaming_enc = self._build_streaming_encoder( self.meta.fps, - camera_encoder, + rgb_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads, @@ -286,7 +286,7 @@ class LeRobotDataset(torch.utils.data.Dataset): self.writer = DatasetWriter( meta=self.meta, root=self.root, - camera_encoder=camera_encoder, + rgb_encoder=rgb_encoder, depth_encoder=depth_encoder, encoder_threads=encoder_threads, batch_encoding_size=batch_encoding_size, @@ -330,14 +330,14 @@ class LeRobotDataset(torch.utils.data.Dataset): @staticmethod def _build_streaming_encoder( fps: int, - camera_encoder: VideoEncoderConfig | None, + rgb_encoder: VideoEncoderConfig | None, depth_encoder: DepthEncoderConfig | None, encoder_queue_maxsize: int, encoder_threads: int | None, ) -> StreamingVideoEncoder: return StreamingVideoEncoder( fps=fps, - camera_encoder=camera_encoder, + rgb_encoder=rgb_encoder, depth_encoder=depth_encoder, queue_maxsize=encoder_queue_maxsize, encoder_threads=encoder_threads, @@ -655,7 +655,7 @@ class LeRobotDataset(torch.utils.data.Dataset): image_writer_threads: int = 0, video_backend: str | None = None, batch_encoding_size: int = 1, - camera_encoder: VideoEncoderConfig | None = None, + rgb_encoder: VideoEncoderConfig | None = None, depth_encoder: DepthEncoderConfig | None = None, metadata_buffer_size: int = 10, streaming_encoding: bool = False, @@ -687,8 +687,8 @@ class LeRobotDataset(torch.utils.data.Dataset): video_backend: Video decoding backend (used when reading back). batch_encoding_size: Number of episodes to accumulate before batch-encoding videos. ``1`` means encode immediately. - camera_encoder: Video encoder settings for cameras (codec, quality, etc.). - When ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` is used. + rgb_encoder: Video encoder settings for cameras (codec, quality, etc.). + When ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used. depth_encoder: Video encoder settings for depth cameras (codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used. encoder_threads: Number of encoder threads (global). ``None`` @@ -735,12 +735,12 @@ class LeRobotDataset(torch.utils.data.Dataset): streaming_enc = None if streaming_encoding and len(obj.meta.video_keys) > 0: streaming_enc = cls._build_streaming_encoder( - fps, camera_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads + fps, rgb_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads ) obj.writer = DatasetWriter( meta=obj.meta, root=obj.root, - camera_encoder=camera_encoder, + rgb_encoder=rgb_encoder, depth_encoder=depth_encoder, encoder_threads=encoder_threads, batch_encoding_size=batch_encoding_size, @@ -764,7 +764,7 @@ class LeRobotDataset(torch.utils.data.Dataset): force_cache_sync: bool = False, video_backend: str | None = None, batch_encoding_size: int = 1, - camera_encoder: VideoEncoderConfig | None = None, + rgb_encoder: VideoEncoderConfig | None = None, depth_encoder: DepthEncoderConfig | None = None, encoder_threads: int | None = None, image_writer_processes: int = 0, @@ -793,8 +793,8 @@ class LeRobotDataset(torch.utils.data.Dataset): video_backend: Video decoding backend for reading back data. batch_encoding_size: Number of episodes to accumulate before batch-encoding videos. - camera_encoder: Video encoder settings for cameras (codec, quality, etc.). - When ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` is used. + rgb_encoder: Video encoder settings for cameras (codec, quality, etc.). + When ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used. depth_encoder: Video encoder settings for depth cameras (codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used. encoder_threads: Number of encoder threads (global). ``None`` @@ -844,12 +844,12 @@ class LeRobotDataset(torch.utils.data.Dataset): streaming_enc = None if streaming_encoding and len(obj.meta.video_keys) > 0: streaming_enc = cls._build_streaming_encoder( - obj.meta.fps, camera_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads + obj.meta.fps, rgb_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads ) obj.writer = DatasetWriter( meta=obj.meta, root=obj.root, - camera_encoder=camera_encoder, + rgb_encoder=rgb_encoder, depth_encoder=depth_encoder, encoder_threads=encoder_threads, batch_encoding_size=batch_encoding_size, diff --git a/src/lerobot/datasets/video_utils.py b/src/lerobot/datasets/video_utils.py index 03e1c707d..e9792b891 100644 --- a/src/lerobot/datasets/video_utils.py +++ b/src/lerobot/datasets/video_utils.py @@ -41,8 +41,8 @@ from PIL import Image from lerobot.configs import ( DepthEncoderConfig, VideoEncoderConfig, - camera_encoder_defaults, depth_encoder_defaults, + rgb_encoder_defaults, ) from lerobot.utils.import_utils import get_safe_default_video_backend @@ -449,7 +449,7 @@ def encode_video_frames( video_path: Output path for the encoded ``.mp4`` file. fps: Frame rate of the output video. video_encoder: Encoder settings (codec, pixel format, quality, ...). When - ``None``, :func:`camera_encoder_defaults` is used. Pass a + ``None``, :func:`rgb_encoder_defaults` is used. Pass a :class:`~lerobot.configs.video.DepthEncoderConfig` to encode depth frames. encoder_threads: Per-encoder thread count forwarded to the codec. ``None`` lets the codec decide. @@ -459,7 +459,7 @@ def encode_video_frames( log a warning. When ``True``, re-encode and replace the existing file. """ if video_encoder is None: - video_encoder = camera_encoder_defaults() + video_encoder = rgb_encoder_defaults() vcodec = video_encoder.vcodec pix_fmt = video_encoder.pix_fmt @@ -547,7 +547,7 @@ def reencode_video( Args: input_video_path: Existing video file to read. output_video_path: Path for the re-encoded file. - video_encoder: Encoder configuration. Defaults to :func:`camera_encoder_defaults`. + video_encoder: Encoder configuration. Defaults to :func:`rgb_encoder_defaults`. encoder_threads: Optional thread count forwarded to :meth:`VideoEncoderConfig.get_codec_options`. log_level: libav log level while encoding, or ``None`` to leave logging unchanged. Defaults to WARNING. overwrite: When ``False`` and ``output_video_path`` already exists, skip and log a warning. @@ -555,7 +555,7 @@ def reencode_video( end_time_s: When set, trim the output to end at this timestamp (seconds, exclusive). """ - video_encoder = video_encoder or camera_encoder_defaults() + video_encoder = video_encoder or rgb_encoder_defaults() if (start_time_s is not None and start_time_s < 0) or (end_time_s is not None and end_time_s < 0): raise ValueError(f"Trim times must be non-negative, got start={start_time_s}, end={end_time_s}.") @@ -886,7 +886,7 @@ class StreamingVideoEncoder: def __init__( self, fps: int, - camera_encoder: VideoEncoderConfig | None = None, + rgb_encoder: VideoEncoderConfig | None = None, depth_encoder: DepthEncoderConfig | None = None, queue_maxsize: int = 30, encoder_threads: int | None = None, @@ -894,8 +894,8 @@ class StreamingVideoEncoder: """ Args: fps: Frames per second for the output videos. - camera_encoder: Video encoder settings applied to all RGB cameras. - When ``None``, :func:`camera_encoder_defaults` is used. + rgb_encoder: Video encoder settings applied to all RGB cameras. + When ``None``, :func:`rgb_encoder_defaults` is used. depth_encoder: Video encoder settings applied to all depth cameras, including the depth quantization parameters. When ``None``, :func:`depth_encoder_defaults` is used. @@ -905,7 +905,7 @@ class StreamingVideoEncoder: ``None`` lets the codec decide. """ self.fps = fps - self._camera_encoder = camera_encoder or camera_encoder_defaults() + self._rgb_encoder = rgb_encoder or rgb_encoder_defaults() self._depth_encoder = depth_encoder or depth_encoder_defaults() self._encoder_threads = encoder_threads self.queue_maxsize = queue_maxsize @@ -946,7 +946,7 @@ class StreamingVideoEncoder: temp_video_dir = Path(tempfile.mkdtemp(dir=temp_dir)) video_path = temp_video_dir / f"{video_key.replace('/', '_')}_streaming.mp4" - encoder = self._depth_encoder if video_key in depth_video_keys else self._camera_encoder + encoder = self._depth_encoder if video_key in depth_video_keys else self._rgb_encoder encoder_thread = _CameraEncoderThread( video_path=video_path, fps=self.fps, diff --git a/src/lerobot/rollout/context.py b/src/lerobot/rollout/context.py index 527ba525a..62d844932 100644 --- a/src/lerobot/rollout/context.py +++ b/src/lerobot/rollout/context.py @@ -332,7 +332,7 @@ def build_rollout_context( cfg.dataset.repo_id, root=cfg.dataset.root, batch_encoding_size=cfg.dataset.video_encoding_batch_size, - camera_encoder=cfg.dataset.camera_encoder, + rgb_encoder=cfg.dataset.rgb_encoder, depth_encoder=cfg.dataset.depth_encoder, streaming_encoding=cfg.dataset.streaming_encoding, encoder_queue_maxsize=cfg.dataset.encoder_queue_maxsize, @@ -368,7 +368,7 @@ def build_rollout_context( image_writer_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras if hasattr(robot, "cameras") else []), batch_encoding_size=cfg.dataset.video_encoding_batch_size, - camera_encoder=cfg.dataset.camera_encoder, + rgb_encoder=cfg.dataset.rgb_encoder, depth_encoder=cfg.dataset.depth_encoder, streaming_encoding=cfg.dataset.streaming_encoding, encoder_queue_maxsize=cfg.dataset.encoder_queue_maxsize, diff --git a/src/lerobot/scripts/lerobot_edit_dataset.py b/src/lerobot/scripts/lerobot_edit_dataset.py index ab0be4a41..f7304f5e9 100644 --- a/src/lerobot/scripts/lerobot_edit_dataset.py +++ b/src/lerobot/scripts/lerobot_edit_dataset.py @@ -199,17 +199,17 @@ Re-encode all videos in a dataset (saves to lerobot/pusht_reencoded by default): lerobot-edit-dataset \ --repo_id lerobot/pusht \ --operation.type reencode_videos \ - --operation.camera_encoder.vcodec h264 \ - --operation.camera_encoder.pix_fmt yuv420p \ - --operation.camera_encoder.crf 23 + --operation.rgb_encoder.vcodec h264 \ + --operation.rgb_encoder.pix_fmt yuv420p \ + --operation.rgb_encoder.crf 23 Re-encode videos into a new dataset using 4 parallel processes: lerobot-edit-dataset \ --repo_id lerobot/pusht \ --new_repo_id lerobot/pusht_h264 \ --operation.type reencode_videos \ - --operation.camera_encoder.vcodec h264 \ - --operation.camera_encoder.crf 23 \ + --operation.rgb_encoder.vcodec h264 \ + --operation.rgb_encoder.crf 23 \ --operation.num_workers 4 Re-encode videos in-place (overwrites original dataset): @@ -217,14 +217,14 @@ Re-encode videos in-place (overwrites original dataset): --repo_id lerobot/pusht \ --new_repo_id lerobot/pusht \ --operation.type reencode_videos \ - --operation.camera_encoder.vcodec h264 \ + --operation.rgb_encoder.vcodec h264 \ --operation.overwrite true Re-encode both RGB and depth videos in a dataset (depth quantization params are preserved): lerobot-edit-dataset \ --repo_id lerobot/pusht_depth \ --operation.type reencode_videos \ - --operation.camera_encoder.vcodec libx264 \ + --operation.rgb_encoder.vcodec libx264 \ --operation.depth_encoder.vcodec ffv1 Using JSON config file: @@ -243,10 +243,10 @@ import draccus from lerobot.configs import ( DepthEncoderConfig, - VideoEncoderConfig, - camera_encoder_defaults, + RGBEncoderConfig, depth_encoder_defaults, parser, + rgb_encoder_defaults, ) from lerobot.datasets import ( LeRobotDataset, @@ -309,7 +309,7 @@ class ModifyTasksConfig(OperationConfig): @dataclass class ConvertImageToVideoConfig(OperationConfig): output_dir: str | None = None - camera_encoder: VideoEncoderConfig = field(default_factory=camera_encoder_defaults) + rgb_encoder: RGBEncoderConfig = field(default_factory=rgb_encoder_defaults) depth_encoder: DepthEncoderConfig = field(default_factory=depth_encoder_defaults) episode_indices: list[int] | None = None num_workers: int = 4 @@ -331,7 +331,7 @@ class RecomputeStatsConfig(OperationConfig): @OperationConfig.register_subclass("reencode_videos") @dataclass class ReencodeVideosConfig(OperationConfig): - camera_encoder: VideoEncoderConfig = field(default_factory=camera_encoder_defaults) + rgb_encoder: RGBEncoderConfig = field(default_factory=rgb_encoder_defaults) depth_encoder: DepthEncoderConfig = field(default_factory=depth_encoder_defaults) num_workers: int = 0 encoder_threads: int | None = None @@ -625,7 +625,7 @@ def handle_convert_image_to_video(cfg: EditDatasetConfig) -> None: dataset=dataset, output_dir=output_dir, repo_id=output_repo_id, - camera_encoder=getattr(cfg.operation, "camera_encoder", None) or camera_encoder_defaults(), + rgb_encoder=getattr(cfg.operation, "rgb_encoder", None) or rgb_encoder_defaults(), depth_encoder=getattr(cfg.operation, "depth_encoder", None) or depth_encoder_defaults(), episode_indices=getattr(cfg.operation, "episode_indices", None), num_workers=getattr(cfg.operation, "num_workers", 4), @@ -745,12 +745,12 @@ def handle_reencode_videos(cfg: EditDatasetConfig) -> None: dataset = LeRobotDataset(output_repo_id, root=output_root) logging.info( - f"Re-encoding videos in {output_repo_id} with RGB encoder {cfg.operation.camera_encoder} " + f"Re-encoding videos in {output_repo_id} with RGB encoder {cfg.operation.rgb_encoder} " f"and depth encoder {cfg.operation.depth_encoder}" ) reencode_dataset( dataset, - camera_encoder=cfg.operation.camera_encoder, + rgb_encoder=cfg.operation.rgb_encoder, depth_encoder=cfg.operation.depth_encoder, encoder_threads=cfg.operation.encoder_threads, num_workers=cfg.operation.num_workers, diff --git a/src/lerobot/scripts/lerobot_record.py b/src/lerobot/scripts/lerobot_record.py index 6fbb4a223..d06a45b3b 100644 --- a/src/lerobot/scripts/lerobot_record.py +++ b/src/lerobot/scripts/lerobot_record.py @@ -79,9 +79,9 @@ lerobot-record \\ --dataset.single_task="Grab the cube" \\ --dataset.streaming_encoding=true \\ --dataset.encoder_threads=2 \\ - --dataset.camera_encoder.vcodec=h264 \\ - --dataset.camera_encoder.preset=fast \\ - --dataset.camera_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \\ + --dataset.rgb_encoder.vcodec=h264 \\ + --dataset.rgb_encoder.preset=fast \\ + --dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \\ --display_data=true ``` """ @@ -403,7 +403,7 @@ def record( cfg.dataset.repo_id, root=cfg.dataset.root, batch_encoding_size=cfg.dataset.video_encoding_batch_size, - camera_encoder=cfg.dataset.camera_encoder, + rgb_encoder=cfg.dataset.rgb_encoder, depth_encoder=cfg.dataset.depth_encoder, encoder_threads=cfg.dataset.encoder_threads, streaming_encoding=cfg.dataset.streaming_encoding, @@ -433,7 +433,7 @@ def record( image_writer_processes=cfg.dataset.num_image_writer_processes, image_writer_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras), batch_encoding_size=cfg.dataset.video_encoding_batch_size, - camera_encoder=cfg.dataset.camera_encoder, + rgb_encoder=cfg.dataset.rgb_encoder, depth_encoder=cfg.dataset.depth_encoder, encoder_threads=cfg.dataset.encoder_threads, streaming_encoding=cfg.dataset.streaming_encoding, @@ -448,7 +448,7 @@ def record( if not cfg.dataset.streaming_encoding: logging.info( - "Streaming encoding is disabled. If you have capable hardware, consider enabling it for way faster episode saving. --dataset.streaming_encoding=true --dataset.encoder_threads=2 # --dataset.camera_encoder.vcodec=auto. More info in the documentation: https://huggingface.co/docs/lerobot/streaming_video_encoding" + "Streaming encoding is disabled. If you have capable hardware, consider enabling it for way faster episode saving. --dataset.streaming_encoding=true --dataset.encoder_threads=2 # --dataset.rgb_encoder.vcodec=auto. More info in the documentation: https://huggingface.co/docs/lerobot/streaming_video_encoding" ) with VideoEncodingManager(dataset): diff --git a/src/lerobot/scripts/lerobot_rollout.py b/src/lerobot/scripts/lerobot_rollout.py index 8515c4cc9..daee87bbe 100644 --- a/src/lerobot/scripts/lerobot_rollout.py +++ b/src/lerobot/scripts/lerobot_rollout.py @@ -142,9 +142,9 @@ Usage examples --robot.port=/dev/ttyACM0 \\ --task="pick up cube" --duration=60 \\ --display_data=true \\ - --dataset.camera_encoder.vcodec=h264 \\ - --dataset.camera_encoder.preset=fast \\ - --dataset.camera_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} + --dataset.rgb_encoder.vcodec=h264 \\ + --dataset.rgb_encoder.preset=fast \\ + --dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} """ import logging diff --git a/tests/datasets/test_dataset_tools.py b/tests/datasets/test_dataset_tools.py index 440ea3bb8..c19e7f41f 100644 --- a/tests/datasets/test_dataset_tools.py +++ b/tests/datasets/test_dataset_tools.py @@ -24,7 +24,7 @@ import torch pytest.importorskip("datasets", reason="datasets is required (install lerobot[dataset])") -from lerobot.configs import DepthEncoderConfig, VideoEncoderConfig +from lerobot.configs import DepthEncoderConfig, RGBEncoderConfig from lerobot.datasets.dataset_tools import ( add_features, convert_image_to_video_dataset, @@ -1253,7 +1253,7 @@ def test_convert_image_to_video_dataset(tmp_path): dataset=source_dataset, output_dir=output_dir, repo_id="lerobot/pusht_video", - camera_encoder=VideoEncoderConfig( + rgb_encoder=RGBEncoderConfig( vcodec="libsvtav1", pix_fmt="yuv420p", g=2, @@ -1396,7 +1396,7 @@ def test_convert_image_to_video_dataset_depth(tmp_path, empty_lerobot_dataset_fa dataset=source_dataset, output_dir=output_dir, repo_id="dummy/depth_video", - camera_encoder=VideoEncoderConfig(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), + rgb_encoder=RGBEncoderConfig(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), depth_encoder=depth_encoder, num_workers=1, ) @@ -1466,12 +1466,12 @@ def test_reencode_dataset_multi_key_multiprocessing( ): """Re-encode a two-camera dataset with num_workers=2 and verify metadata refresh.""" features = features_factory(use_videos=True) - initial_cfg = VideoEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12) + initial_cfg = RGBEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12) dataset = empty_lerobot_dataset_factory( root=tmp_path / "ds", features=features, use_videos=True, - camera_encoder=initial_cfg, + rgb_encoder=initial_cfg, ) add_frames(dataset, num_frames=4) @@ -1482,13 +1482,13 @@ def test_reencode_dataset_multi_key_multiprocessing( assert len(dataset.meta.video_keys) == 2 - target_cfg = VideoEncoderConfig(vcodec="h264", g=6, crf=23, pix_fmt="yuv420p") + target_cfg = RGBEncoderConfig(vcodec="h264", g=6, crf=23, pix_fmt="yuv420p") - result = reencode_dataset(dataset, camera_encoder=target_cfg, num_workers=2) + result = reencode_dataset(dataset, rgb_encoder=target_cfg, num_workers=2) assert result is dataset persisted_info = load_info(dataset.root) for vk in dataset.meta.video_keys: - persisted_encoder = VideoEncoderConfig.from_video_info(persisted_info.features[vk].get("info", {})) + persisted_encoder = RGBEncoderConfig.from_video_info(persisted_info.features[vk].get("info", {})) assert persisted_encoder == target_cfg diff --git a/tests/datasets/test_streaming_video_encoder.py b/tests/datasets/test_streaming_video_encoder.py index e2c51db20..cfab771bb 100644 --- a/tests/datasets/test_streaming_video_encoder.py +++ b/tests/datasets/test_streaming_video_encoder.py @@ -173,7 +173,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.laptop"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config( + rgb_encoder=self._make_encoder_config( vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30, preset=13 ), ) @@ -205,7 +205,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.laptop", f"{OBS_IMAGES}.phone"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), + rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), ) encoder.start_episode(video_keys, tmp_path) @@ -231,7 +231,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.cam"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), + rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), ) for ep in range(3): @@ -257,7 +257,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.cam"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), + rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), ) encoder.start_episode(video_keys, tmp_path) @@ -303,7 +303,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.cam"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config( + rgb_encoder=self._make_encoder_config( vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30, preset=13 ), ) @@ -340,7 +340,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.cam1", f"{OBS_IMAGES}.cam2"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), + rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30), ) encoder.start_episode(video_keys, tmp_path) @@ -377,7 +377,7 @@ class TestStreamingVideoEncoder: ) encoder = StreamingVideoEncoder( fps=30, - camera_encoder=cfg, + rgb_encoder=cfg, encoder_threads=2, ) assert encoder._encoder_threads == 2 @@ -417,7 +417,7 @@ class TestStreamingVideoEncoder: video_keys = [f"{OBS_IMAGES}.cam"] encoder = StreamingVideoEncoder( fps=30, - camera_encoder=self._make_encoder_config( + rgb_encoder=self._make_encoder_config( vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30, preset=13 ), queue_maxsize=1, diff --git a/tests/datasets/test_video_encoding.py b/tests/datasets/test_video_encoding.py index c78d81e6c..7681ea55e 100644 --- a/tests/datasets/test_video_encoding.py +++ b/tests/datasets/test_video_encoding.py @@ -595,7 +595,7 @@ class TestEncoderConfigPersistence: def test_first_episode_save_persists_encoder_config(self, tmp_path, empty_lerobot_dataset_factory): cfg = VideoEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12) dataset = empty_lerobot_dataset_factory( - root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, camera_encoder=cfg + root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, rgb_encoder=cfg ) add_frames(dataset, num_frames=4) @@ -618,7 +618,7 @@ class TestEncoderConfigPersistence: def test_second_episode_does_not_overwrite_encoder_fields(self, tmp_path, empty_lerobot_dataset_factory): cfg = VideoEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12) dataset = empty_lerobot_dataset_factory( - root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, camera_encoder=cfg + root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, rgb_encoder=cfg ) add_frames(dataset, num_frames=4)