admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-07 01:51:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(RGB <-> camera): renaming camera_encoder to rgb_encoder for clarity

Browse Source
This commit is contained in:
CarolinePascal
2026-06-23 11:28:13 +02:00
parent 8f939767ca
commit 2167731b2c
23 changed files with 170 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/source/earthrover_mini_plus.mdx
+1 -1
View File
@@ -194,7 +194,7 @@ lerobot-record \
--dataset.single_task="Navigate around obstacles" \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--display_data=true
```
docs/source/groot.mdx
+1 -1
View File
@@ -124,7 +124,7 @@ lerobot-rollout\
--dataset.single_task="Grab and handover the red cube to the other arm" \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--policy.path=<user>/groot-bimanual \ # your trained model
--duration=600
```
docs/source/hope_jr.mdx
+2 -2
View File
@@ -232,7 +232,7 @@ lerobot-record \
--dataset.private=true \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--display_data=true
```
@@ -278,6 +278,6 @@ lerobot-record \
--dataset.num_episodes=10 \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--policy.path=outputs/train/hopejr_hand/checkpoints/last/pretrained_model
```
docs/source/il_robots.mdx
+1 -1
View File
@@ -207,7 +207,7 @@ lerobot-record \
--dataset.num_episodes=5 \
--dataset.single_task="Grab the black cube" \
--dataset.streaming_encoding=true \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--dataset.encoder_threads=2
```
</hfoption>
docs/source/lerobot-dataset-v3.mdx
+1 -1
View File
@@ -44,7 +44,7 @@ lerobot-record \
--dataset.num_episodes=5 \
--dataset.single_task="Grab the black cube" \
--dataset.streaming_encoding=true \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--dataset.encoder_threads=2
```
docs/source/reachy2.mdx
+2 -2
View File
@@ -161,7 +161,7 @@ lerobot-record \
--dataset.private=true \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--display_data=true
```
@@ -203,7 +203,7 @@ lerobot-record \
--dataset.private=true \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
# --dataset.camera_encoder.vcodec=auto \
# --dataset.rgb_encoder.vcodec=auto \
--display_data=true
```
docs/source/streaming_video_encoding.mdx
+20 -20
View File
@@ -17,7 +17,7 @@ This makes `save_episode()` near-instant (the video is already encoded by the ti
| Parameter | CLI Flag | Type | Default | Description |
| ----------------------- | --------------------------------- | ------------- | ------------- | ----------------------------------------------------------------- |
| `streaming_encoding` | `--dataset.streaming_encoding` | `bool` | `True` | Enable real-time encoding during capture |
| `vcodec` | `--dataset.camera_encoder.vcodec` | `str` | `"libsvtav1"` | Video codec. `"auto"` detects best HW encoder |
| `vcodec` | `--dataset.rgb_encoder.vcodec` | `str` | `"libsvtav1"` | Video codec. `"auto"` detects best HW encoder |
| `encoder_threads` | `--dataset.encoder_threads` | `int \| None` | `None` (auto) | Threads per encoder instance. `None` will leave the vcoded decide |
| `encoder_queue_maxsize` | `--dataset.encoder_queue_maxsize` | `int` | `30` | Max buffered frames per camera (~1s at 30fps). Consumes RAM |
@@ -82,15 +82,15 @@ Use HW encoding when:
### Available HW Encoders
| Encoder | Platform | Hardware | CLI Value |
| ------------------- | ------------- | ------------------------------------------------------------------------------------------------ | --------------------------------------------------- |
| `h264_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.camera_encoder.vcodec=h264_videotoolbox` |
| `hevc_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.camera_encoder.vcodec=hevc_videotoolbox` |
| `h264_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.camera_encoder.vcodec=h264_nvenc` |
| `hevc_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.camera_encoder.vcodec=hevc_nvenc` |
| `h264_vaapi` | Linux | Intel/AMD GPU | `--dataset.camera_encoder.vcodec=h264_vaapi` |
| `h264_qsv` | Linux/Windows | Intel Quick Sync | `--dataset.camera_encoder.vcodec=h264_qsv` |
| `auto` | Any | Probes the system for available HW encoders. Falls back to `libsvtav1` if no HW encoder is found | `--dataset.camera_encoder.vcodec=auto` |
| Encoder | Platform | Hardware | CLI Value |
| ------------------- | ------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------ |
| `h264_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.rgb_encoder.vcodec=h264_videotoolbox` |
| `hevc_videotoolbox` | macOS | Apple Silicon / Intel | `--dataset.rgb_encoder.vcodec=hevc_videotoolbox` |
| `h264_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.rgb_encoder.vcodec=h264_nvenc` |
| `hevc_nvenc` | Linux/Windows | NVIDIA GPU | `--dataset.rgb_encoder.vcodec=hevc_nvenc` |
| `h264_vaapi` | Linux | Intel/AMD GPU | `--dataset.rgb_encoder.vcodec=h264_vaapi` |
| `h264_qsv` | Linux/Windows | Intel Quick Sync | `--dataset.rgb_encoder.vcodec=h264_qsv` |
| `auto` | Any | Probes the system for available HW encoders. Falls back to `libsvtav1` if no HW encoder is found | `--dataset.rgb_encoder.vcodec=auto` |
> [!NOTE]
> In order to use the HW accelerated encoders you might need to upgrade your GPU drivers.
@@ -100,15 +100,15 @@ Use HW encoding when:
## 5. Troubleshooting
| Symptom | Likely Cause | Fix |
| ------------------------------------------------------------------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| System freezes or choppy robot movement or Rerun visualization lag | CPU starved (100% load usage) | Close other apps, reduce encoding throughput, lower `encoder_threads`, use `h264`, use `display_data=False`. If the CPU continues to be at 100% then it might be insufficient for your setup, consider `--dataset.streaming_encoding=false` or HW encoding (`--dataset.camera_encoder.vcodec=auto`) |
| "Encoder queue full" warnings or dropped frames in dataset | Encoder can't keep up (Queue overflow) | If CPU is not at 100%: Increase `encoder_threads`, increase `encoder_queue_maxsize` or use HW encoding (`--dataset.camera_encoder.vcodec=auto`). |
| High RAM usage | Queue filling faster than encoding | `encoder_threads` too low or CPU insufficient. Reduce `encoder_queue_maxsize` or use HW encoding |
| Large video files | Using HW encoder or H.264 | Expected trade-off. Switch to `libsvtav1` if CPU allows |
| `save_episode()` still slow | `streaming_encoding` is `False` | Set `--dataset.streaming_encoding=true` |
| Encoder thread crash | Codec not available or invalid settings | Check `vcodec` is installed, try `--dataset.camera_encoder.vcodec=auto` |
| Recorded dataset is missing frames | CPU/GPU starvation or occasional load spikes | If ~5% of frames are missing, your system is likely overloaded — follow the recommendations above. If fewer frames are missing (~2%), they are probably due to occasional transient load spikes (often at startup) and can be considered expected. |
| Symptom | Likely Cause | Fix |
| ------------------------------------------------------------------ | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| System freezes or choppy robot movement or Rerun visualization lag | CPU starved (100% load usage) | Close other apps, reduce encoding throughput, lower `encoder_threads`, use `h264`, use `display_data=False`. If the CPU continues to be at 100% then it might be insufficient for your setup, consider `--dataset.streaming_encoding=false` or HW encoding (`--dataset.rgb_encoder.vcodec=auto`) |
| "Encoder queue full" warnings or dropped frames in dataset | Encoder can't keep up (Queue overflow) | If CPU is not at 100%: Increase `encoder_threads`, increase `encoder_queue_maxsize` or use HW encoding (`--dataset.rgb_encoder.vcodec=auto`). |
| High RAM usage | Queue filling faster than encoding | `encoder_threads` too low or CPU insufficient. Reduce `encoder_queue_maxsize` or use HW encoding |
| Large video files | Using HW encoder or H.264 | Expected trade-off. Switch to `libsvtav1` if CPU allows |
| `save_episode()` still slow | `streaming_encoding` is `False` | Set `--dataset.streaming_encoding=true` |
| Encoder thread crash | Codec not available or invalid settings | Check `vcodec` is installed, try `--dataset.rgb_encoder.vcodec=auto` |
| Recorded dataset is missing frames | CPU/GPU starvation or occasional load spikes | If ~5% of frames are missing, your system is likely overloaded — follow the recommendations above. If fewer frames are missing (~2%), they are probably due to occasional transient load spikes (often at startup) and can be considered expected. |
## 6. Recommended Configurations
@@ -146,7 +146,7 @@ On very constrained systems, streaming encoding may compete too heavily with the
# 2camsx 640x480x3 @30fps: Requires some tuning.
# Use H.264, disable streaming, consider batching encoding
lerobot-record --dataset.camera_encoder.vcodec=h264 --dataset.streaming_encoding=false ...
lerobot-record --dataset.rgb_encoder.vcodec=h264 --dataset.streaming_encoding=false ...
```
## 7. Closing note
docs/source/using_dataset_tools.mdx
+12 -12
View File
@@ -118,10 +118,10 @@ lerobot-edit-dataset \
--repo_id lerobot/pusht_image \
--operation.type convert_image_to_video \
--operation.output_dir outputs/pusht_video \
--operation.camera_encoder.vcodec libsvtav1 \
--operation.camera_encoder.pix_fmt yuv420p \
--operation.camera_encoder.g 2 \
--operation.camera_encoder.crf 30
--operation.rgb_encoder.vcodec libsvtav1 \
--operation.rgb_encoder.pix_fmt yuv420p \
--operation.rgb_encoder.g 2 \
--operation.rgb_encoder.crf 30
# Convert a dataset that includes depth maps, customizing the depth encoder
lerobot-edit-dataset \
@@ -157,37 +157,37 @@ lerobot-edit-dataset \
**Parameters:**
- `output_dir`: Custom output directory (optional - by default uses `new_repo_id` or `{repo_id}_video`)
- `camera_encoder`: Video encoder settings applied to RGB cameras — all sub-fields accessible via `--operation.camera_encoder.<field>`. See [Video Encoding Parameters](./video_encoding_parameters) for more details.
- `rgb_encoder`: Video encoder settings applied to RGB cameras — all sub-fields accessible via `--operation.rgb_encoder.<field>`. See [Video Encoding Parameters](./video_encoding_parameters) for more details.
- `depth_encoder`: Video encoder settings applied to depth-map cameras (e.g. from an Intel RealSense). In addition to the standard encoder fields it exposes the depth quantization knobs (`depth_min`, `depth_max`, `shift`, `use_log`), accessible via `--operation.depth_encoder.<field>`. These quantization settings are persisted to the dataset metadata so depth can be dequantized back to physical units on load. See the [Depth streams](./video_encoding_parameters#depth-streams) section for details.
- `episode_indices`: List of specific episodes to convert (default: all episodes)
- `num_workers`: Number of parallel workers for processing (default: 4)
**Note:** The resulting dataset will be a proper LeRobotDataset with all cameras encoded as videos in the `videos/` directory, with parquet files containing only metadata (no raw image data). Depth-map cameras are detected automatically and routed to the `depth_encoder`, while RGB cameras use the `camera_encoder`. All episodes, stats, and tasks are preserved.
**Note:** The resulting dataset will be a proper LeRobotDataset with all cameras encoded as videos in the `videos/` directory, with parquet files containing only metadata (no raw image data). Depth-map cameras are detected automatically and routed to the `depth_encoder`, while RGB cameras use the `rgb_encoder`. All episodes, stats, and tasks are preserved.
#### Re-encode Videos
Re-encode the videos of an existing video dataset with different encoder settings, without going back to raw frames. RGB videos use the `camera_encoder` and depth videos use the `depth_encoder`. Provide only the encoder(s) you want to re-encode; the other stream type is left untouched.
Re-encode the videos of an existing video dataset with different encoder settings, without going back to raw frames. RGB videos use the `rgb_encoder` and depth videos use the `depth_encoder`. Provide only the encoder(s) you want to re-encode; the other stream type is left untouched.
```bash
# Re-encode all RGB videos with new settings (saves to lerobot/pusht_reencoded by default)
lerobot-edit-dataset \
--repo_id lerobot/pusht \
--operation.type reencode_videos \
--operation.camera_encoder.vcodec h264 \
--operation.camera_encoder.pix_fmt yuv420p \
--operation.camera_encoder.crf 23
--operation.rgb_encoder.vcodec h264 \
--operation.rgb_encoder.pix_fmt yuv420p \
--operation.rgb_encoder.crf 23
# Re-encode both RGB and depth videos in a dataset with depth maps
lerobot-edit-dataset \
--repo_id lerobot/pusht_depth \
--operation.type reencode_videos \
--operation.camera_encoder.vcodec libx264 \
--operation.rgb_encoder.vcodec libx264 \
--operation.depth_encoder.vcodec ffv1
```
**Parameters:**
- `camera_encoder`: Encoder settings applied to every RGB video. Omit to skip re-encoding RGB videos.
- `rgb_encoder`: Encoder settings applied to every RGB video. Omit to skip re-encoding RGB videos.
- `depth_encoder`: Encoder settings applied to every depth video. Omit to skip re-encoding depth videos.
- `num_workers`: Number of parallel workers for processing.
docs/source/video_encoding_parameters.mdx
+10 -10
View File
@@ -2,15 +2,15 @@
When video storage is enabled, LeRobot stores each camera stream as an **MP4** file instead of saving one image file per timestep. Video encoding compresses across time, which usually cuts dataset size and I/O compared to a pile of PNG, while keeping MP4 — a format every player and loader understands.
Encoding frames into an MP4 is a full FFmpeg pipeline: choice of encoder, pixel format, GOP/keyframes, quality vs. speed, and optional extra encoder flags. Most of these knobs are user-tunable through `camera_encoder`, a nested `VideoEncoderConfig` (`lerobot.configs.video.VideoEncoderConfig`) passed through PyAV.
Encoding frames into an MP4 is a full FFmpeg pipeline: choice of encoder, pixel format, GOP/keyframes, quality vs. speed, and optional extra encoder flags. Most of these knobs are user-tunable through `rgb_encoder`, a nested `RGBEncoderConfig` (`lerobot.configs.video.RGBEncoderConfig`) passed through PyAV.
You can set these parameters from the CLI with `--dataset.camera_encoder.<field>` (e.g. with `lerobot-record` or `lerobot-rollout`). The same block applies to every camera video stream in that run.
You can set these parameters from the CLI with `--dataset.rgb_encoder.<field>` (e.g. with `lerobot-record` or `lerobot-rollout`). The same block applies to every camera video stream in that run.
<Tip>
Video storage must be on for `camera_encoder` to have any effect —
Video storage must be on for `rgb_encoder` to have any effect —
`use_videos=True` in Python APIs, or `--dataset.video=true` on the CLI (the
recording default). With video off, inputs stay as images and `camera_encoder`
is ignored.
recording default). With video off, inputs stay as images and `rgb_encoder` is
ignored.
</Tip>
For details on **when** frames are written vs. encoded (streaming vs. post-episode), queues, and other top-level `--dataset.*` switches, see [Streaming Video Encoding](./streaming_video_encoding). For an encoding-parameter comparison and experiments, see the [video-benchmark Space](https://huggingface.co/spaces/lerobot/video-benchmark).
@@ -33,9 +33,9 @@ lerobot-record \
--dataset.single_task="Grab the cube" \
--dataset.streaming_encoding=true \
--dataset.encoder_threads=2 \
--dataset.camera_encoder.vcodec=h264 \
--dataset.camera_encoder.preset=fast \
--dataset.camera_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \
--dataset.rgb_encoder.vcodec=h264 \
--dataset.rgb_encoder.preset=fast \
--dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \
--display_data=true
```
@@ -50,7 +50,7 @@ Only override these parameters if you have a specific reason to, and measure the
</Tip>
All flags below are prefixed with `--dataset.camera_encoder.` on the CLI.
All flags below are prefixed with `--dataset.rgb_encoder.` on the CLI.
| Parameter | Type | Default | Description |
| --------------- | ---------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -172,7 +172,7 @@ Two sources contribute to the `info` block:
<Tip>
This block is populated **once**, from the **first** episode. It assumes every
episode in the dataset was encoded with the same `camera_encoder`. Changing
episode in the dataset was encoded with the same `rgb_encoder`. Changing
encoder settings partway through a recording is not supported — the
`info.json` will only reflect the parameters used for the first episode.
</Tip>
src/lerobot/configs/__init__.py
+4 -2
View File
@@ -37,10 +37,11 @@ from .video import (
VALID_VIDEO_CODECS,
VIDEO_ENCODER_INFO_KEYS,
DepthEncoderConfig,
RGBEncoderConfig,
VideoEncoderConfig,
camera_encoder_defaults,
depth_encoder_defaults,
encoder_config_from_video_info,
rgb_encoder_defaults,
)
__all__ = [
@@ -61,9 +62,10 @@ __all__ = [
"WandBConfig",
"load_recipe",
"VideoEncoderConfig",
"RGBEncoderConfig",
"DepthEncoderConfig",
# Defaults
"camera_encoder_defaults",
"rgb_encoder_defaults",
"depth_encoder_defaults",
# Factories
"encoder_config_from_video_info",
src/lerobot/configs/dataset.py
+3 -3
View File
@@ -18,7 +18,7 @@ from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from .video import DepthEncoderConfig, VideoEncoderConfig, camera_encoder_defaults, depth_encoder_defaults
from .video import DepthEncoderConfig, RGBEncoderConfig, depth_encoder_defaults, rgb_encoder_defaults
@dataclass
@@ -58,8 +58,8 @@ class DatasetRecordConfig:
# Set to 1 for immediate encoding (default behavior), or higher for batched encoding
video_encoding_batch_size: int = 1
# Video encoder settings for camera MP4s (codec, quality, GOP, etc.). Tuned via CLI nested keys,
# e.g. ``--dataset.camera_encoder.vcodec=h264`` (see ``VideoEncoderConfig``).
camera_encoder: VideoEncoderConfig = field(default_factory=camera_encoder_defaults)
# e.g. ``--dataset.rgb_encoder.vcodec=h264`` (see ``RGBEncoderConfig``).
rgb_encoder: RGBEncoderConfig = field(default_factory=rgb_encoder_defaults)
# Video encoder settings for depth-map MP4s (codec, quality, GOP, etc.). Tuned via CLI nested keys.
depth_encoder: DepthEncoderConfig = field(default_factory=depth_encoder_defaults)
# Enable streaming video encoding: encode frames in real-time during capture instead
src/lerobot/configs/video.py
+21 -9
View File
@@ -84,9 +84,10 @@ class VideoEncoderConfig:
# Extra codec options merged last, e.g. {"tune": "film"}.
extra_options: dict[str, Any] = field(default_factory=dict)
# Source-data channel count this encoder is expected to handle (3 for RGB,
# 1 for depth, etc.)
_DEFAULT_CHANNELS: ClassVar[int] = 3
# Source-data channel count this encoder is expected to handle. ``None``
# disables the pix_fmt channel-count check; concrete subclasses set it
# (3 for RGB, 1 for depth, etc.).
_DEFAULT_CHANNELS: ClassVar[int | None] = None
def __post_init__(self) -> None:
self.resolve_vcodec()
@@ -246,9 +247,20 @@ class VideoEncoderConfig:
return opts
def camera_encoder_defaults() -> VideoEncoderConfig:
"""Return a :class:`VideoEncoderConfig` with RGB-camera defaults."""
return VideoEncoderConfig()
@dataclass
class RGBEncoderConfig(VideoEncoderConfig):
"""Encoder configuration for RGB camera streams.
Identical to :class:`VideoEncoderConfig` but declares the 3-channel
source-data layout so ``pix_fmt`` is validated against RGB inputs.
"""
_DEFAULT_CHANNELS: ClassVar[int] = 3
def rgb_encoder_defaults() -> RGBEncoderConfig:
"""Return a :class:`RGBEncoderConfig` with RGB-camera defaults."""
return RGBEncoderConfig()
@dataclass
@@ -295,7 +307,7 @@ def encoder_config_from_video_info(video_info: dict | None) -> VideoEncoderConfi
"""Build the appropriate encoder config from a feature's ``info`` block.
Dispatches to :class:`DepthEncoderConfig` when the dict marks the feature
as a depth map and to :class:`VideoEncoderConfig`
as a depth map and to :class:`RGBEncoderConfig`
otherwise.
Args:
@@ -304,9 +316,9 @@ def encoder_config_from_video_info(video_info: dict | None) -> VideoEncoderConfi
Returns:
A :class:`DepthEncoderConfig` for depth features, otherwise a
:class:`VideoEncoderConfig`.
:class:`RGBEncoderConfig`.
"""
video_info = video_info or {}
is_depth = bool(video_info.get("is_depth_map") or video_info.get("video.is_depth_map"))
cls: type[VideoEncoderConfig] = DepthEncoderConfig if is_depth else VideoEncoderConfig
cls: type[VideoEncoderConfig] = DepthEncoderConfig if is_depth else RGBEncoderConfig
return cls.from_video_info(video_info)
src/lerobot/datasets/dataset_tools.py
+14 -14
View File
@@ -40,9 +40,9 @@ from tqdm import tqdm
from lerobot.configs import (
DepthEncoderConfig,
VideoEncoderConfig,
camera_encoder_defaults,
depth_encoder_defaults,
encoder_config_from_video_info,
rgb_encoder_defaults,
)
from lerobot.configs.video import DEPTH_ENCODER_INFO_FIELD_NAMES
from lerobot.utils.constants import ACTION, HF_LEROBOT_HOME, OBS_IMAGE, OBS_STATE
@@ -1670,7 +1670,7 @@ def convert_image_to_video_dataset(
dataset: LeRobotDataset,
output_dir: Path | None = None,
repo_id: str | None = None,
camera_encoder: VideoEncoderConfig | None = None,
rgb_encoder: VideoEncoderConfig | None = None,
depth_encoder: DepthEncoderConfig | None = None,
episode_indices: list[int] | None = None,
num_workers: int = 4,
@@ -1689,8 +1689,8 @@ def convert_image_to_video_dataset(
``new_root`` in ``EditDatasetConfig``.
repo_id: Converted dataset identifier. Equivalent to ``new_repo_id`` in
``EditDatasetConfig``.
camera_encoder: Video encoder settings applied to RGB cameras. When ``None``,
:func:`~lerobot.configs.video.camera_encoder_defaults` is used.
rgb_encoder: Video encoder settings applied to RGB cameras. When ``None``,
:func:`~lerobot.configs.video.rgb_encoder_defaults` is used.
depth_encoder: Video encoder settings applied to depth-map cameras, including
the quantization parameters persisted to the dataset metadata. When
``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used.
@@ -1705,8 +1705,8 @@ def convert_image_to_video_dataset(
Returns:
A new :class:`LeRobotDataset` with images encoded as videos.
"""
if camera_encoder is None:
camera_encoder = camera_encoder_defaults()
if rgb_encoder is None:
rgb_encoder = rgb_encoder_defaults()
if depth_encoder is None:
depth_encoder = depth_encoder_defaults()
@@ -1733,7 +1733,7 @@ def convert_image_to_video_dataset(
logging.info(
f"Converting {len(episode_indices)} episodes with {len(img_keys)} cameras from {dataset.repo_id}"
)
logging.info(f"RGB video encoder: {camera_encoder}, depth video encoder: {depth_encoder}")
logging.info(f"RGB video encoder: {rgb_encoder}, depth video encoder: {depth_encoder}")
# Create new features dict, converting image features to video features
new_features = {}
@@ -1795,7 +1795,7 @@ def convert_image_to_video_dataset(
episode_lengths = {ep_idx: dataset.meta.episodes["length"][ep_idx] for ep_idx in episode_indices}
for img_key in tqdm(img_keys, desc="Processing cameras"):
target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else camera_encoder
target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else rgb_encoder
# Estimate size per frame by encoding a small calibration sample
# This provides accurate compression ratio for the specific codec parameters
@@ -1889,7 +1889,7 @@ def convert_image_to_video_dataset(
# Update video info for all image keys (now videos). They are registered as
# video features above, so update_video_info populates their (still-empty) info.
for img_key in img_keys:
target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else camera_encoder
target_encoder = depth_encoder if img_key in dataset.meta.depth_keys else rgb_encoder
new_meta.update_video_info(video_key=img_key, video_encoder=target_encoder)
write_info(new_meta.info, new_meta.root)
@@ -1930,7 +1930,7 @@ def _reencode_video_worker(args: tuple) -> Path:
def reencode_dataset(
dataset: LeRobotDataset,
camera_encoder: VideoEncoderConfig | None = None,
rgb_encoder: VideoEncoderConfig | None = None,
depth_encoder: DepthEncoderConfig | None = None,
encoder_threads: int | None = None,
num_workers: int | None = None,
@@ -1942,7 +1942,7 @@ def reencode_dataset(
Args:
dataset: An existing :class:`LeRobotDataset` whose videos will be
re-encoded.
camera_encoder: Target encoder configuration applied to every RGB video
rgb_encoder: Target encoder configuration applied to every RGB video
file. If ``None``, re-encoding is skipped for RGB videos.
depth_encoder: Target encoder configuration applied to every depth video
file. If ``None``, re-encoding is skipped for depth videos.
@@ -1961,14 +1961,14 @@ def reencode_dataset(
video_keys_encoders_dict = {}
video_keys_paths_dict = {}
if camera_encoder is None and depth_encoder is None:
raise ValueError("Either camera_encoder or depth_encoder must be provided")
if rgb_encoder is None and depth_encoder is None:
raise ValueError("Either rgb_encoder or depth_encoder must be provided")
# Only re-encode if the videos are not already encoded with the given video encoding parameters
for video_key in meta.video_keys:
current_info = meta.info.features[video_key].get("info", {})
current_encoder = encoder_config_from_video_info(current_info)
target_encoder = depth_encoder if video_key in meta.depth_keys else camera_encoder
target_encoder = depth_encoder if video_key in meta.depth_keys else rgb_encoder
if target_encoder is None:
logging.info(f"No encoder provided for {video_key} video. Skipping re-encoding.")
elif current_encoder != target_encoder:
src/lerobot/datasets/dataset_writer.py
+8 -10
View File
@@ -34,8 +34,8 @@ import torch
from lerobot.configs import (
DepthEncoderConfig,
VideoEncoderConfig,
camera_encoder_defaults,
depth_encoder_defaults,
rgb_encoder_defaults,
)
from .compute_stats import compute_episode_stats
@@ -107,7 +107,7 @@ class DatasetWriter:
self,
meta: LeRobotDatasetMetadata,
root: Path,
camera_encoder: VideoEncoderConfig | None,
rgb_encoder: VideoEncoderConfig | None,
depth_encoder: DepthEncoderConfig | None,
encoder_threads: int | None,
batch_encoding_size: int,
@@ -120,8 +120,8 @@ class DatasetWriter:
meta: Dataset metadata instance (used for feature schema, chunk
settings, and episode persistence).
root: Local dataset root directory.
camera_encoder: Video encoder settings applied to RGB cameras. When
``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` is used.
rgb_encoder: Video encoder settings applied to RGB cameras. When
``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used.
depth_encoder: Video encoder settings applied to depth cameras, including
the quantization parameters. When ``None``,
:func:`~lerobot.configs.video.depth_encoder_defaults` is used.
@@ -135,7 +135,7 @@ class DatasetWriter:
"""
self._meta = meta
self._root = root
self._camera_encoder = camera_encoder or camera_encoder_defaults()
self._rgb_encoder = rgb_encoder or rgb_encoder_defaults()
self._depth_encoder = depth_encoder or depth_encoder_defaults()
self._encoder_threads = encoder_threads
self._batch_encoding_size = batch_encoding_size
@@ -321,9 +321,7 @@ class DatasetWriter:
episode_index,
self._root,
self._meta.fps,
self._depth_encoder
if video_key in self._meta.depth_keys
else self._camera_encoder,
self._depth_encoder if video_key in self._meta.depth_keys else self._rgb_encoder,
self._encoder_threads,
): video_key
for video_key in self._meta.video_keys
@@ -538,7 +536,7 @@ class DatasetWriter:
video_key,
video_encoder=self._depth_encoder
if video_key in self._meta.depth_keys
else self._camera_encoder,
else self._rgb_encoder,
)
write_info(self._meta.info, self._meta.root)
@@ -613,7 +611,7 @@ class DatasetWriter:
episode_index,
self._root,
self._meta.fps,
self._depth_encoder if is_depth else self._camera_encoder,
self._depth_encoder if is_depth else self._rgb_encoder,
self._encoder_threads,
)
src/lerobot/datasets/lerobot_dataset.py
+17 -17
View File
@@ -60,7 +60,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
return_uint8: bool = False,
depth_output_unit: str = DEFAULT_DEPTH_UNIT,
batch_encoding_size: int = 1,
camera_encoder: VideoEncoderConfig | None = None,
rgb_encoder: VideoEncoderConfig | None = None,
depth_encoder: DepthEncoderConfig | None = None,
encoder_threads: int | None = None,
streaming_encoding: bool = False,
@@ -185,8 +185,8 @@ class LeRobotDataset(torch.utils.data.Dataset):
You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision.
batch_encoding_size (int, optional): Number of episodes to accumulate before batch encoding videos.
Set to 1 for immediate encoding (default), or higher for batched encoding. Defaults to 1.
camera_encoder (VideoEncoderConfig | None, optional): Video encoder settings for cameras
(codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults`
rgb_encoder (VideoEncoderConfig | None, optional): Video encoder settings for cameras
(codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults`
is used by the writer.
depth_encoder (DepthEncoderConfig | None, optional): Video encoder settings for depth cameras
(codec, quality, etc.). When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults`
@@ -278,7 +278,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
if streaming_encoding and len(self.meta.video_keys) > 0:
streaming_enc = self._build_streaming_encoder(
self.meta.fps,
camera_encoder,
rgb_encoder,
depth_encoder,
encoder_queue_maxsize,
encoder_threads,
@@ -286,7 +286,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
self.writer = DatasetWriter(
meta=self.meta,
root=self.root,
camera_encoder=camera_encoder,
rgb_encoder=rgb_encoder,
depth_encoder=depth_encoder,
encoder_threads=encoder_threads,
batch_encoding_size=batch_encoding_size,
@@ -330,14 +330,14 @@ class LeRobotDataset(torch.utils.data.Dataset):
@staticmethod
def _build_streaming_encoder(
fps: int,
camera_encoder: VideoEncoderConfig | None,
rgb_encoder: VideoEncoderConfig | None,
depth_encoder: DepthEncoderConfig | None,
encoder_queue_maxsize: int,
encoder_threads: int | None,
) -> StreamingVideoEncoder:
return StreamingVideoEncoder(
fps=fps,
camera_encoder=camera_encoder,
rgb_encoder=rgb_encoder,
depth_encoder=depth_encoder,
queue_maxsize=encoder_queue_maxsize,
encoder_threads=encoder_threads,
@@ -655,7 +655,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
image_writer_threads: int = 0,
video_backend: str | None = None,
batch_encoding_size: int = 1,
camera_encoder: VideoEncoderConfig | None = None,
rgb_encoder: VideoEncoderConfig | None = None,
depth_encoder: DepthEncoderConfig | None = None,
metadata_buffer_size: int = 10,
streaming_encoding: bool = False,
@@ -687,8 +687,8 @@ class LeRobotDataset(torch.utils.data.Dataset):
video_backend: Video decoding backend (used when reading back).
batch_encoding_size: Number of episodes to accumulate before
batch-encoding videos. ``1`` means encode immediately.
camera_encoder: Video encoder settings for cameras (codec, quality, etc.).
When ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` is used.
rgb_encoder: Video encoder settings for cameras (codec, quality, etc.).
When ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used.
depth_encoder: Video encoder settings for depth cameras (codec, quality, etc.).
When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used.
encoder_threads: Number of encoder threads (global). ``None``
@@ -735,12 +735,12 @@ class LeRobotDataset(torch.utils.data.Dataset):
streaming_enc = None
if streaming_encoding and len(obj.meta.video_keys) > 0:
streaming_enc = cls._build_streaming_encoder(
fps, camera_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads
fps, rgb_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads
)
obj.writer = DatasetWriter(
meta=obj.meta,
root=obj.root,
camera_encoder=camera_encoder,
rgb_encoder=rgb_encoder,
depth_encoder=depth_encoder,
encoder_threads=encoder_threads,
batch_encoding_size=batch_encoding_size,
@@ -764,7 +764,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
force_cache_sync: bool = False,
video_backend: str | None = None,
batch_encoding_size: int = 1,
camera_encoder: VideoEncoderConfig | None = None,
rgb_encoder: VideoEncoderConfig | None = None,
depth_encoder: DepthEncoderConfig | None = None,
encoder_threads: int | None = None,
image_writer_processes: int = 0,
@@ -793,8 +793,8 @@ class LeRobotDataset(torch.utils.data.Dataset):
video_backend: Video decoding backend for reading back data.
batch_encoding_size: Number of episodes to accumulate before
batch-encoding videos.
camera_encoder: Video encoder settings for cameras (codec, quality, etc.).
When ``None``, :func:`~lerobot.configs.video.camera_encoder_defaults` is used.
rgb_encoder: Video encoder settings for cameras (codec, quality, etc.).
When ``None``, :func:`~lerobot.configs.video.rgb_encoder_defaults` is used.
depth_encoder: Video encoder settings for depth cameras (codec, quality, etc.).
When ``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used.
encoder_threads: Number of encoder threads (global). ``None``
@@ -844,12 +844,12 @@ class LeRobotDataset(torch.utils.data.Dataset):
streaming_enc = None
if streaming_encoding and len(obj.meta.video_keys) > 0:
streaming_enc = cls._build_streaming_encoder(
obj.meta.fps, camera_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads
obj.meta.fps, rgb_encoder, depth_encoder, encoder_queue_maxsize, encoder_threads
)
obj.writer = DatasetWriter(
meta=obj.meta,
root=obj.root,
camera_encoder=camera_encoder,
rgb_encoder=rgb_encoder,
depth_encoder=depth_encoder,
encoder_threads=encoder_threads,
batch_encoding_size=batch_encoding_size,
src/lerobot/datasets/video_utils.py
+10 -10
View File
@@ -41,8 +41,8 @@ from PIL import Image
from lerobot.configs import (
DepthEncoderConfig,
VideoEncoderConfig,
camera_encoder_defaults,
depth_encoder_defaults,
rgb_encoder_defaults,
)
from lerobot.utils.import_utils import get_safe_default_video_backend
@@ -449,7 +449,7 @@ def encode_video_frames(
video_path: Output path for the encoded ``.mp4`` file.
fps: Frame rate of the output video.
video_encoder: Encoder settings (codec, pixel format, quality, ...). When
``None``, :func:`camera_encoder_defaults` is used. Pass a
``None``, :func:`rgb_encoder_defaults` is used. Pass a
:class:`~lerobot.configs.video.DepthEncoderConfig` to encode depth frames.
encoder_threads: Per-encoder thread count forwarded to the codec. ``None``
lets the codec decide.
@@ -459,7 +459,7 @@ def encode_video_frames(
log a warning. When ``True``, re-encode and replace the existing file.
"""
if video_encoder is None:
video_encoder = camera_encoder_defaults()
video_encoder = rgb_encoder_defaults()
vcodec = video_encoder.vcodec
pix_fmt = video_encoder.pix_fmt
@@ -547,7 +547,7 @@ def reencode_video(
Args:
input_video_path: Existing video file to read.
output_video_path: Path for the re-encoded file.
video_encoder: Encoder configuration. Defaults to :func:`camera_encoder_defaults`.
video_encoder: Encoder configuration. Defaults to :func:`rgb_encoder_defaults`.
encoder_threads: Optional thread count forwarded to :meth:`VideoEncoderConfig.get_codec_options`.
log_level: libav log level while encoding, or ``None`` to leave logging unchanged. Defaults to WARNING.
overwrite: When ``False`` and ``output_video_path`` already exists, skip and log a warning.
@@ -555,7 +555,7 @@ def reencode_video(
end_time_s: When set, trim the output to end at this timestamp (seconds, exclusive).
"""
video_encoder = video_encoder or camera_encoder_defaults()
video_encoder = video_encoder or rgb_encoder_defaults()
if (start_time_s is not None and start_time_s < 0) or (end_time_s is not None and end_time_s < 0):
raise ValueError(f"Trim times must be non-negative, got start={start_time_s}, end={end_time_s}.")
@@ -886,7 +886,7 @@ class StreamingVideoEncoder:
def __init__(
self,
fps: int,
camera_encoder: VideoEncoderConfig | None = None,
rgb_encoder: VideoEncoderConfig | None = None,
depth_encoder: DepthEncoderConfig | None = None,
queue_maxsize: int = 30,
encoder_threads: int | None = None,
@@ -894,8 +894,8 @@ class StreamingVideoEncoder:
"""
Args:
fps: Frames per second for the output videos.
camera_encoder: Video encoder settings applied to all RGB cameras.
When ``None``, :func:`camera_encoder_defaults` is used.
rgb_encoder: Video encoder settings applied to all RGB cameras.
When ``None``, :func:`rgb_encoder_defaults` is used.
depth_encoder: Video encoder settings applied to all depth cameras,
including the depth quantization parameters. When ``None``,
:func:`depth_encoder_defaults` is used.
@@ -905,7 +905,7 @@ class StreamingVideoEncoder:
``None`` lets the codec decide.
"""
self.fps = fps
self._camera_encoder = camera_encoder or camera_encoder_defaults()
self._rgb_encoder = rgb_encoder or rgb_encoder_defaults()
self._depth_encoder = depth_encoder or depth_encoder_defaults()
self._encoder_threads = encoder_threads
self.queue_maxsize = queue_maxsize
@@ -946,7 +946,7 @@ class StreamingVideoEncoder:
temp_video_dir = Path(tempfile.mkdtemp(dir=temp_dir))
video_path = temp_video_dir / f"{video_key.replace('/', '_')}_streaming.mp4"
encoder = self._depth_encoder if video_key in depth_video_keys else self._camera_encoder
encoder = self._depth_encoder if video_key in depth_video_keys else self._rgb_encoder
encoder_thread = _CameraEncoderThread(
video_path=video_path,
fps=self.fps,
src/lerobot/rollout/context.py
+2 -2
View File
@@ -332,7 +332,7 @@ def build_rollout_context(
cfg.dataset.repo_id,
root=cfg.dataset.root,
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
camera_encoder=cfg.dataset.camera_encoder,
rgb_encoder=cfg.dataset.rgb_encoder,
depth_encoder=cfg.dataset.depth_encoder,
streaming_encoding=cfg.dataset.streaming_encoding,
encoder_queue_maxsize=cfg.dataset.encoder_queue_maxsize,
@@ -368,7 +368,7 @@ def build_rollout_context(
image_writer_threads=cfg.dataset.num_image_writer_threads_per_camera
* len(robot.cameras if hasattr(robot, "cameras") else []),
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
camera_encoder=cfg.dataset.camera_encoder,
rgb_encoder=cfg.dataset.rgb_encoder,
depth_encoder=cfg.dataset.depth_encoder,
streaming_encoding=cfg.dataset.streaming_encoding,
encoder_queue_maxsize=cfg.dataset.encoder_queue_maxsize,
src/lerobot/scripts/lerobot_edit_dataset.py
+14 -14
View File
@@ -199,17 +199,17 @@ Re-encode all videos in a dataset (saves to lerobot/pusht_reencoded by default):
lerobot-edit-dataset \
--repo_id lerobot/pusht \
--operation.type reencode_videos \
--operation.camera_encoder.vcodec h264 \
--operation.camera_encoder.pix_fmt yuv420p \
--operation.camera_encoder.crf 23
--operation.rgb_encoder.vcodec h264 \
--operation.rgb_encoder.pix_fmt yuv420p \
--operation.rgb_encoder.crf 23
Re-encode videos into a new dataset using 4 parallel processes:
lerobot-edit-dataset \
--repo_id lerobot/pusht \
--new_repo_id lerobot/pusht_h264 \
--operation.type reencode_videos \
--operation.camera_encoder.vcodec h264 \
--operation.camera_encoder.crf 23 \
--operation.rgb_encoder.vcodec h264 \
--operation.rgb_encoder.crf 23 \
--operation.num_workers 4
Re-encode videos in-place (overwrites original dataset):
@@ -217,14 +217,14 @@ Re-encode videos in-place (overwrites original dataset):
--repo_id lerobot/pusht \
--new_repo_id lerobot/pusht \
--operation.type reencode_videos \
--operation.camera_encoder.vcodec h264 \
--operation.rgb_encoder.vcodec h264 \
--operation.overwrite true
Re-encode both RGB and depth videos in a dataset (depth quantization params are preserved):
lerobot-edit-dataset \
--repo_id lerobot/pusht_depth \
--operation.type reencode_videos \
--operation.camera_encoder.vcodec libx264 \
--operation.rgb_encoder.vcodec libx264 \
--operation.depth_encoder.vcodec ffv1
Using JSON config file:
@@ -243,10 +243,10 @@ import draccus
from lerobot.configs import (
DepthEncoderConfig,
VideoEncoderConfig,
camera_encoder_defaults,
RGBEncoderConfig,
depth_encoder_defaults,
parser,
rgb_encoder_defaults,
)
from lerobot.datasets import (
LeRobotDataset,
@@ -309,7 +309,7 @@ class ModifyTasksConfig(OperationConfig):
@dataclass
class ConvertImageToVideoConfig(OperationConfig):
output_dir: str | None = None
camera_encoder: VideoEncoderConfig = field(default_factory=camera_encoder_defaults)
rgb_encoder: RGBEncoderConfig = field(default_factory=rgb_encoder_defaults)
depth_encoder: DepthEncoderConfig = field(default_factory=depth_encoder_defaults)
episode_indices: list[int] | None = None
num_workers: int = 4
@@ -331,7 +331,7 @@ class RecomputeStatsConfig(OperationConfig):
@OperationConfig.register_subclass("reencode_videos")
@dataclass
class ReencodeVideosConfig(OperationConfig):
camera_encoder: VideoEncoderConfig = field(default_factory=camera_encoder_defaults)
rgb_encoder: RGBEncoderConfig = field(default_factory=rgb_encoder_defaults)
depth_encoder: DepthEncoderConfig = field(default_factory=depth_encoder_defaults)
num_workers: int = 0
encoder_threads: int | None = None
@@ -625,7 +625,7 @@ def handle_convert_image_to_video(cfg: EditDatasetConfig) -> None:
dataset=dataset,
output_dir=output_dir,
repo_id=output_repo_id,
camera_encoder=getattr(cfg.operation, "camera_encoder", None) or camera_encoder_defaults(),
rgb_encoder=getattr(cfg.operation, "rgb_encoder", None) or rgb_encoder_defaults(),
depth_encoder=getattr(cfg.operation, "depth_encoder", None) or depth_encoder_defaults(),
episode_indices=getattr(cfg.operation, "episode_indices", None),
num_workers=getattr(cfg.operation, "num_workers", 4),
@@ -745,12 +745,12 @@ def handle_reencode_videos(cfg: EditDatasetConfig) -> None:
dataset = LeRobotDataset(output_repo_id, root=output_root)
logging.info(
f"Re-encoding videos in {output_repo_id} with RGB encoder {cfg.operation.camera_encoder} "
f"Re-encoding videos in {output_repo_id} with RGB encoder {cfg.operation.rgb_encoder} "
f"and depth encoder {cfg.operation.depth_encoder}"
)
reencode_dataset(
dataset,
camera_encoder=cfg.operation.camera_encoder,
rgb_encoder=cfg.operation.rgb_encoder,
depth_encoder=cfg.operation.depth_encoder,
encoder_threads=cfg.operation.encoder_threads,
num_workers=cfg.operation.num_workers,
src/lerobot/scripts/lerobot_record.py
+6 -6
View File
@@ -79,9 +79,9 @@ lerobot-record \\
--dataset.single_task="Grab the cube" \\
--dataset.streaming_encoding=true \\
--dataset.encoder_threads=2 \\
--dataset.camera_encoder.vcodec=h264 \\
--dataset.camera_encoder.preset=fast \\
--dataset.camera_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \\
--dataset.rgb_encoder.vcodec=h264 \\
--dataset.rgb_encoder.preset=fast \\
--dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2} \\
--display_data=true
```
"""
@@ -403,7 +403,7 @@ def record(
cfg.dataset.repo_id,
root=cfg.dataset.root,
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
camera_encoder=cfg.dataset.camera_encoder,
rgb_encoder=cfg.dataset.rgb_encoder,
depth_encoder=cfg.dataset.depth_encoder,
encoder_threads=cfg.dataset.encoder_threads,
streaming_encoding=cfg.dataset.streaming_encoding,
@@ -433,7 +433,7 @@ def record(
image_writer_processes=cfg.dataset.num_image_writer_processes,
image_writer_threads=cfg.dataset.num_image_writer_threads_per_camera * len(robot.cameras),
batch_encoding_size=cfg.dataset.video_encoding_batch_size,
camera_encoder=cfg.dataset.camera_encoder,
rgb_encoder=cfg.dataset.rgb_encoder,
depth_encoder=cfg.dataset.depth_encoder,
encoder_threads=cfg.dataset.encoder_threads,
streaming_encoding=cfg.dataset.streaming_encoding,
@@ -448,7 +448,7 @@ def record(
if not cfg.dataset.streaming_encoding:
logging.info(
"Streaming encoding is disabled. If you have capable hardware, consider enabling it for way faster episode saving. --dataset.streaming_encoding=true --dataset.encoder_threads=2 # --dataset.camera_encoder.vcodec=auto. More info in the documentation: https://huggingface.co/docs/lerobot/streaming_video_encoding"
"Streaming encoding is disabled. If you have capable hardware, consider enabling it for way faster episode saving. --dataset.streaming_encoding=true --dataset.encoder_threads=2 # --dataset.rgb_encoder.vcodec=auto. More info in the documentation: https://huggingface.co/docs/lerobot/streaming_video_encoding"
)
with VideoEncodingManager(dataset):
src/lerobot/scripts/lerobot_rollout.py
+3 -3
View File
@@ -142,9 +142,9 @@ Usage examples
--robot.port=/dev/ttyACM0 \\
--task="pick up cube" --duration=60 \\
--display_data=true \\
--dataset.camera_encoder.vcodec=h264 \\
--dataset.camera_encoder.preset=fast \\
--dataset.camera_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2}
--dataset.rgb_encoder.vcodec=h264 \\
--dataset.rgb_encoder.preset=fast \\
--dataset.rgb_encoder.extra_options={"tune": "film", "profile:v": "high", "bf": 2}
"""
import logging
tests/datasets/test_dataset_tools.py
+8 -8
View File
@@ -24,7 +24,7 @@ import torch
pytest.importorskip("datasets", reason="datasets is required (install lerobot[dataset])")
from lerobot.configs import DepthEncoderConfig, VideoEncoderConfig
from lerobot.configs import DepthEncoderConfig, RGBEncoderConfig
from lerobot.datasets.dataset_tools import (
add_features,
convert_image_to_video_dataset,
@@ -1253,7 +1253,7 @@ def test_convert_image_to_video_dataset(tmp_path):
dataset=source_dataset,
output_dir=output_dir,
repo_id="lerobot/pusht_video",
camera_encoder=VideoEncoderConfig(
rgb_encoder=RGBEncoderConfig(
vcodec="libsvtav1",
pix_fmt="yuv420p",
g=2,
@@ -1396,7 +1396,7 @@ def test_convert_image_to_video_dataset_depth(tmp_path, empty_lerobot_dataset_fa
dataset=source_dataset,
output_dir=output_dir,
repo_id="dummy/depth_video",
camera_encoder=VideoEncoderConfig(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
rgb_encoder=RGBEncoderConfig(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
depth_encoder=depth_encoder,
num_workers=1,
)
@@ -1466,12 +1466,12 @@ def test_reencode_dataset_multi_key_multiprocessing(
):
"""Re-encode a two-camera dataset with num_workers=2 and verify metadata refresh."""
features = features_factory(use_videos=True)
initial_cfg = VideoEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12)
initial_cfg = RGBEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12)
dataset = empty_lerobot_dataset_factory(
root=tmp_path / "ds",
features=features,
use_videos=True,
camera_encoder=initial_cfg,
rgb_encoder=initial_cfg,
)
add_frames(dataset, num_frames=4)
@@ -1482,13 +1482,13 @@ def test_reencode_dataset_multi_key_multiprocessing(
assert len(dataset.meta.video_keys) == 2
target_cfg = VideoEncoderConfig(vcodec="h264", g=6, crf=23, pix_fmt="yuv420p")
target_cfg = RGBEncoderConfig(vcodec="h264", g=6, crf=23, pix_fmt="yuv420p")
result = reencode_dataset(dataset, camera_encoder=target_cfg, num_workers=2)
result = reencode_dataset(dataset, rgb_encoder=target_cfg, num_workers=2)
assert result is dataset
persisted_info = load_info(dataset.root)
for vk in dataset.meta.video_keys:
persisted_encoder = VideoEncoderConfig.from_video_info(persisted_info.features[vk].get("info", {}))
persisted_encoder = RGBEncoderConfig.from_video_info(persisted_info.features[vk].get("info", {}))
assert persisted_encoder == target_cfg
tests/datasets/test_streaming_video_encoder.py
+8 -8
View File
@@ -173,7 +173,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.laptop"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(
rgb_encoder=self._make_encoder_config(
vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30, preset=13
),
)
@@ -205,7 +205,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.laptop", f"{OBS_IMAGES}.phone"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
)
encoder.start_episode(video_keys, tmp_path)
@@ -231,7 +231,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.cam"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
)
for ep in range(3):
@@ -257,7 +257,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.cam"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
)
encoder.start_episode(video_keys, tmp_path)
@@ -303,7 +303,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.cam"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(
rgb_encoder=self._make_encoder_config(
vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30, preset=13
),
)
@@ -340,7 +340,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.cam1", f"{OBS_IMAGES}.cam2"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
rgb_encoder=self._make_encoder_config(vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30),
)
encoder.start_episode(video_keys, tmp_path)
@@ -377,7 +377,7 @@ class TestStreamingVideoEncoder:
)
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=cfg,
rgb_encoder=cfg,
encoder_threads=2,
)
assert encoder._encoder_threads == 2
@@ -417,7 +417,7 @@ class TestStreamingVideoEncoder:
video_keys = [f"{OBS_IMAGES}.cam"]
encoder = StreamingVideoEncoder(
fps=30,
camera_encoder=self._make_encoder_config(
rgb_encoder=self._make_encoder_config(
vcodec="libsvtav1", pix_fmt="yuv420p", g=2, crf=30, preset=13
),
queue_maxsize=1,
tests/datasets/test_video_encoding.py
+2 -2
View File
@@ -595,7 +595,7 @@ class TestEncoderConfigPersistence:
def test_first_episode_save_persists_encoder_config(self, tmp_path, empty_lerobot_dataset_factory):
cfg = VideoEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12)
dataset = empty_lerobot_dataset_factory(
root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, camera_encoder=cfg
root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, rgb_encoder=cfg
)
add_frames(dataset, num_frames=4)
@@ -618,7 +618,7 @@ class TestEncoderConfigPersistence:
def test_second_episode_does_not_overwrite_encoder_fields(self, tmp_path, empty_lerobot_dataset_factory):
cfg = VideoEncoderConfig(vcodec="libsvtav1", g=2, crf=30, preset=12)
dataset = empty_lerobot_dataset_factory(
root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, camera_encoder=cfg
root=tmp_path / "ds", features=DUMMY_VIDEO_FEATURES, use_videos=True, rgb_encoder=cfg
)
add_frames(dataset, num_frames=4)