mirror of
https://github.com/huggingface/lerobot.git
synced 2026-08-08 17:39:44 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e79aa3fd3a | |||
| 99806fec8d | |||
| 31d16c8fd8 | |||
| 629bf08e4a |
@@ -24,24 +24,19 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
|
||||
# Triggers on pushes to main that touch the docs or the sources the API reference is generated from.
|
||||
# `src/**` is included because the API reference is built from docstrings via `[[autodoc]]`: without it,
|
||||
# published API pages would go stale as soon as a docstring changed.
|
||||
# Triggers the workflow on push events to main for the docs folder
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "src/**"
|
||||
|
||||
# Same for pull requests, so a docstring change gets a preview build and a broken `[[autodoc]]` path
|
||||
# fails the PR rather than main.
|
||||
# Triggers the workflow on pull request events targeting main for the docs folder
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "src/**"
|
||||
|
||||
release:
|
||||
types: [published]
|
||||
@@ -64,21 +59,12 @@ jobs:
|
||||
with:
|
||||
commit_sha: ${{ github.sha }}
|
||||
package: lerobot
|
||||
# doc-builder ships a mock-deps registry entry for lerobot, so the reusable workflow takes its
|
||||
# "light install" path: `pip install ./lerobot --no-deps` plus a handful of real dependencies.
|
||||
# That is not enough to import lerobot — draccus runs `register_subclass` at import time and
|
||||
# `processor/converters.py` calls `functools.singledispatch.register(torch.Tensor)`, neither of
|
||||
# which works against a mock. Install the package for real before the build.
|
||||
pre_command: uv pip install "./lerobot[dataset]"
|
||||
# `--version main` is load-bearing: without `--not_python_module`, doc-builder falls back to
|
||||
# `lerobot.__version__` and only maps that to the default branch when it contains "dev". Our main
|
||||
# branch carries a release version (0.6.2), so omitting this would publish the main docs to
|
||||
# /lerobot/v0.6.2/ instead of /lerobot/main/ and disable notebook building.
|
||||
additional_args: >-
|
||||
--not_python_module
|
||||
${{
|
||||
(github.event_name == 'release' && format('--version {0}', github.event.release.tag_name)) ||
|
||||
(inputs.version != '' && format('--version {0}', inputs.version)) ||
|
||||
'--version main'
|
||||
''
|
||||
}}
|
||||
secrets:
|
||||
token: ${{ secrets.HUGGINGFACE_PUSH }}
|
||||
@@ -97,6 +83,4 @@ jobs:
|
||||
commit_sha: ${{ github.event.pull_request.head.sha }}
|
||||
pr_number: ${{ github.event.number }}
|
||||
package: lerobot
|
||||
# See the comment on build_main_docs. The PR workflow passes its own `--version pr_<n>`, so no
|
||||
# additional_args are needed here.
|
||||
pre_command: uv pip install "./lerobot[dataset]"
|
||||
additional_args: --not_python_module
|
||||
|
||||
@@ -56,41 +56,3 @@ jobs:
|
||||
uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
|
||||
with:
|
||||
extra_args: --all-files --show-diff-on-failure --color=always
|
||||
|
||||
# This job runs the examples in our docstrings and validates the doctest allowlist.
|
||||
# See docs/source/writing_docstrings.mdx for the standard these enforce.
|
||||
doc-checks:
|
||||
name: Run Documentation Checks (Doctests)
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Examples that need a physical robot, a serial port or a Hub download are skipped by content.
|
||||
# Everything else has to actually run. See src/lerobot/utils/doctest_utils.py.
|
||||
SKIP_HARDWARE_DOCTEST: "1"
|
||||
SKIP_CUDA_DOCTEST: "1"
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup uv and Python
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
enable-cache: true
|
||||
version: "0.11.30"
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked --extra test --extra dataset
|
||||
|
||||
- name: Check the doctest list is sorted and its paths exist
|
||||
run: make check-doctest-list
|
||||
|
||||
- name: Check documented arguments match their signatures
|
||||
run: make check-docstrings
|
||||
|
||||
- name: Check docstring coverage has not regressed
|
||||
run: uv run --with interrogate interrogate --config=pyproject.toml
|
||||
|
||||
- name: Run doctests
|
||||
run: make doctest
|
||||
|
||||
+2
-11
@@ -67,11 +67,7 @@ repos:
|
||||
args: [--prose-wrap=preserve]
|
||||
# Jinja2 model-card templates use a .md extension but contain {% ... %} /
|
||||
# {{ ... }} tags that prettier's Markdown formatter mangles (e.g. table loops).
|
||||
#
|
||||
# docs/source/api/ holds the generated API reference. Its `[[autodoc]]` blocks restrict output
|
||||
# to an indented `- member` list, which prettier reads as a lazy paragraph continuation and
|
||||
# joins onto one line — silently turning a member list into part of the directive.
|
||||
exclude: ^(src/lerobot/templates/.*\.md|docs/source/api/.*\.mdx)$
|
||||
exclude: ^src/lerobot/templates/.*\.md$
|
||||
|
||||
##### Security #####
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
@@ -108,13 +104,8 @@ repos:
|
||||
# args: ["--docstring-style", "google", "-v", "2"]
|
||||
# exclude: ^tests/.*$
|
||||
|
||||
# interrogate runs in CI (quality.yml, doc-checks job) rather than here. Its 1.7.0 release still imports
|
||||
# the deprecated `py` package, which resolves against whatever `py` happens to be importable in
|
||||
# pre-commit's isolated env — on a machine with miniconda on the path that is a stray `py.py` and the
|
||||
# hook dies before it reads any config. The gate is the same either way; the CI step is just reliable.
|
||||
# - repo: https://github.com/econchick/interrogate
|
||||
# rev: 1.7.0
|
||||
# hooks:
|
||||
# - id: interrogate
|
||||
# args: ["--config=pyproject.toml"]
|
||||
# pass_filenames: false
|
||||
# args: ["-vv", "--config=pyproject.toml"]
|
||||
|
||||
@@ -50,10 +50,6 @@ To run checks manually on all files:
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
### Docstrings
|
||||
|
||||
The API reference is generated from the docstrings in `src/lerobot/`. If you add or change anything public, follow the [docstring standard](https://huggingface.co/docs/lerobot/writing_docstrings) — the format is parsed by the renderer and checked in CI.
|
||||
|
||||
### Running Tests
|
||||
|
||||
We use `pytest`. First, ensure you have test artifacts by installing **git-lfs**:
|
||||
|
||||
@@ -184,29 +184,3 @@ test-smolvla-ete-eval:
|
||||
# backend, so it does not require a real model checkpoint or GPU.
|
||||
annotation-e2e:
|
||||
uv run python -m tests.annotations.run_e2e_smoke
|
||||
|
||||
# Docstring & doctest checks. See docs/source/writing_docstrings.mdx for the standard these enforce.
|
||||
|
||||
# Run the examples in the docstrings listed in utils/documentation_tests.txt. Hardware and GPU examples are
|
||||
# skipped by content (see src/lerobot/utils/doctest_utils.py); CI sets both flags.
|
||||
doctest:
|
||||
@files=$$(grep -v '^\s*#' utils/documentation_tests.txt | grep -v '^\s*$$'); \
|
||||
if [ -z "$$files" ]; then \
|
||||
echo "utils/documentation_tests.txt lists no files; nothing to run."; \
|
||||
else \
|
||||
SKIP_HARDWARE_DOCTEST=1 uv run pytest --doctest-modules --no-header -q $$files; \
|
||||
fi
|
||||
|
||||
check-doctest-list:
|
||||
uv run python utils/check_doctest_list.py
|
||||
|
||||
fix-doctest-list:
|
||||
uv run python utils/check_doctest_list.py --fix_and_overwrite
|
||||
|
||||
check-docstrings:
|
||||
uv run python utils/check_docstrings.py
|
||||
uv run python utils/check_config_docstrings.py
|
||||
|
||||
fix-docstrings:
|
||||
uv run python utils/check_docstrings.py --fix_and_overwrite
|
||||
uv run python utils/check_doctest_list.py --fix_and_overwrite
|
||||
|
||||
-60
@@ -1,60 +0,0 @@
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Root conftest: makes doctest collection use LeRobot's parser.
|
||||
|
||||
This only affects `--doctest-modules` runs (see `make doctest`). The test suite itself is configured by
|
||||
`tests/conftest.py`.
|
||||
"""
|
||||
|
||||
import doctest
|
||||
|
||||
import _pytest.doctest
|
||||
|
||||
from lerobot.utils.doctest_utils import LeRobotDoctestModule, LeRobotDocTestParser
|
||||
|
||||
# Lets an example opt out of output comparison with `# doctest: +IGNORE_RESULT`, for calls whose output is
|
||||
# a progress bar or otherwise not reproducible.
|
||||
IGNORE_RESULT = doctest.register_optionflag("IGNORE_RESULT")
|
||||
|
||||
OutputChecker = doctest.OutputChecker
|
||||
|
||||
|
||||
class CustomOutputChecker(OutputChecker):
|
||||
"""An output checker that honours the `IGNORE_RESULT` flag."""
|
||||
|
||||
def check_output(self, want, got, optionflags):
|
||||
"""Return `True` when `IGNORE_RESULT` is set, otherwise defer to stdlib.
|
||||
|
||||
Args:
|
||||
want (`str`):
|
||||
The expected output.
|
||||
got (`str`):
|
||||
The actual output.
|
||||
optionflags (`int`):
|
||||
Bitmask of active doctest option flags.
|
||||
|
||||
Returns:
|
||||
`bool`: Whether the output is considered a match.
|
||||
"""
|
||||
if IGNORE_RESULT & optionflags:
|
||||
return True
|
||||
return OutputChecker.check_output(self, want, got, optionflags)
|
||||
|
||||
|
||||
# Reassigning these module attributes is how doctest behaviour is customised; mypy sees it as assigning to
|
||||
# a type, which is exactly what is intended here.
|
||||
doctest.OutputChecker = CustomOutputChecker # type: ignore[misc]
|
||||
_pytest.doctest.DoctestModule = LeRobotDoctestModule
|
||||
doctest.DocTestParser = LeRobotDocTestParser # type: ignore[misc]
|
||||
@@ -191,28 +191,6 @@
|
||||
- sections:
|
||||
- local: contributing
|
||||
title: Contribute to LeRobot
|
||||
- local: writing_docstrings
|
||||
title: Writing docstrings
|
||||
- local: backwardcomp
|
||||
title: Backward compatibility
|
||||
title: "About"
|
||||
- sections:
|
||||
- local: api/robots
|
||||
title: Robots
|
||||
- local: api/teleoperators
|
||||
title: Teleoperators
|
||||
- local: api/cameras
|
||||
title: Cameras
|
||||
- local: api/motors
|
||||
title: Motors
|
||||
- local: api/datasets
|
||||
title: Datasets
|
||||
- local: api/policies
|
||||
title: Policies
|
||||
- local: api/processor
|
||||
title: Processors
|
||||
- local: api/envs
|
||||
title: Environments
|
||||
- local: api/configs
|
||||
title: Configuration
|
||||
title: "API Reference"
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# Cameras
|
||||
|
||||
Cameras supply the image observations a policy sees. Every backend — OpenCV, Intel RealSense, Reachy 2 —
|
||||
implements the [`Camera`] interface, so swapping hardware does not change the code that reads frames.
|
||||
|
||||
See the [Cameras guide](../cameras) for choosing and configuring a camera, and
|
||||
[Third-Party Cameras & Sensors](../third_party_sensors) for devices outside the core set.
|
||||
|
||||
## Camera
|
||||
|
||||
[[autodoc]] lerobot.cameras.Camera
|
||||
- connect
|
||||
- disconnect
|
||||
- read
|
||||
- async_read
|
||||
- find_cameras
|
||||
|
||||
## CameraConfig
|
||||
|
||||
[[autodoc]] lerobot.cameras.CameraConfig
|
||||
|
||||
## make_cameras_from_configs
|
||||
|
||||
[[autodoc]] lerobot.cameras.make_cameras_from_configs
|
||||
@@ -1,27 +0,0 @@
|
||||
# Configuration
|
||||
|
||||
LeRobot configuration is plain dataclasses parsed by [draccus](https://github.com/dlwh/draccus), so every
|
||||
field is settable from the CLI. [`TrainPipelineConfig`] is the top-level object for `lerobot-train`.
|
||||
|
||||
Polymorphic configs (policies, robots, environments) use `draccus.ChoiceRegistry`: a subclass registers
|
||||
itself with `@register_subclass("name")` and is then selectable by that name on the command line.
|
||||
|
||||
## TrainPipelineConfig
|
||||
|
||||
[[autodoc]] lerobot.configs.train.TrainPipelineConfig
|
||||
|
||||
## PreTrainedConfig
|
||||
|
||||
[[autodoc]] lerobot.configs.PreTrainedConfig
|
||||
|
||||
## DatasetConfig
|
||||
|
||||
[[autodoc]] lerobot.configs.DatasetConfig
|
||||
|
||||
## EvalConfig
|
||||
|
||||
[[autodoc]] lerobot.configs.EvalConfig
|
||||
|
||||
## WandBConfig
|
||||
|
||||
[[autodoc]] lerobot.configs.WandBConfig
|
||||
@@ -1,57 +0,0 @@
|
||||
# Datasets
|
||||
|
||||
[`LeRobotDataset`] is the format every LeRobot script reads and writes. It is episode-aware, decodes video
|
||||
observations on the fly, and round-trips to the Hugging Face Hub.
|
||||
|
||||
See [Using LeRobotDataset](../lerobot-dataset-v3) for the format and the common operations,
|
||||
[Porting Large Datasets](../porting_datasets_v3) for migration, and [Tools](../tools) for the CLI.
|
||||
|
||||
## LeRobotDataset
|
||||
|
||||
[[autodoc]] lerobot.datasets.LeRobotDataset
|
||||
|
||||
## LeRobotDatasetMetadata
|
||||
|
||||
[[autodoc]] lerobot.datasets.LeRobotDatasetMetadata
|
||||
|
||||
## MultiLeRobotDataset
|
||||
|
||||
[[autodoc]] lerobot.datasets.MultiLeRobotDataset
|
||||
|
||||
## StreamingLeRobotDataset
|
||||
|
||||
[[autodoc]] lerobot.datasets.StreamingLeRobotDataset
|
||||
|
||||
## EpisodeAwareSampler
|
||||
|
||||
[[autodoc]] lerobot.datasets.sampler.EpisodeAwareSampler
|
||||
|
||||
## Editing a dataset
|
||||
|
||||
Functions in `lerobot.datasets.dataset_tools` for editing an existing `LeRobotDataset` on disk: adding,
|
||||
removing, or modifying features; splitting, merging, or deleting episodes; re-encoding video; and
|
||||
recomputing statistics. Each returns a new dataset rather than mutating the source in place.
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.add_features
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.remove_feature
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.modify_features
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.modify_tasks
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.delete_episodes
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.split_dataset
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.merge_datasets
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.recompute_stats
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.reencode_dataset
|
||||
|
||||
[[autodoc]] lerobot.datasets.dataset_tools.convert_image_to_video_dataset
|
||||
|
||||
## Aggregating datasets
|
||||
|
||||
[[autodoc]] lerobot.datasets.aggregate.aggregate_datasets
|
||||
@@ -1,19 +0,0 @@
|
||||
# Environments
|
||||
|
||||
Simulation environments are configured through [`EnvConfig`] and built by [`make_env`]. Each subclass
|
||||
declares its `gym_kwargs` and how to construct the vectorised environments.
|
||||
|
||||
See [Environments from the Hub](../envhub) for using published environments and
|
||||
[Adding a New Benchmark](../adding_benchmarks) for contributing one.
|
||||
|
||||
## EnvConfig
|
||||
|
||||
[[autodoc]] lerobot.envs.EnvConfig
|
||||
|
||||
## make_env
|
||||
|
||||
[[autodoc]] lerobot.envs.make_env
|
||||
|
||||
## make_env_config
|
||||
|
||||
[[autodoc]] lerobot.envs.make_env_config
|
||||
@@ -1,23 +0,0 @@
|
||||
# Motors
|
||||
|
||||
`MotorsBus` is the low-level interface to a chain of servos on a serial bus. Robots use it to read positions
|
||||
and write goal positions; you rarely touch it directly unless you are adding hardware.
|
||||
|
||||
See [Bring Your Own Hardware](../integrate_hardware) for adding a new bus, and
|
||||
[Updating Feetech Firmware](../feetech) and [Damiao Motors and CAN Bus](../damiao) for device-specific notes.
|
||||
|
||||
## MotorsBus
|
||||
|
||||
[[autodoc]] lerobot.motors.motors_bus.MotorsBus
|
||||
|
||||
## Motor
|
||||
|
||||
[[autodoc]] lerobot.motors.Motor
|
||||
|
||||
## MotorCalibration
|
||||
|
||||
[[autodoc]] lerobot.motors.MotorCalibration
|
||||
|
||||
## MotorNormMode
|
||||
|
||||
[[autodoc]] lerobot.motors.MotorNormMode
|
||||
@@ -1,20 +0,0 @@
|
||||
# Policies
|
||||
|
||||
Every policy inherits [`PreTrainedPolicy`], which combines a `torch.nn.Module` with the Hub mixin, so any
|
||||
policy can be pushed to and loaded from the Hugging Face Hub with the same two calls.
|
||||
|
||||
Each policy has its own guide with training recipes and results — [ACT](../act), [SmolVLA](../smolvla),
|
||||
[π₀](../pi0), [π₀.₅](../pi05) and the rest are listed under Policies. To add one, see
|
||||
[Adding a Policy](../bring_your_own_policies).
|
||||
|
||||
## PreTrainedPolicy
|
||||
|
||||
[[autodoc]] lerobot.policies.pretrained.PreTrainedPolicy
|
||||
|
||||
## PreTrainedConfig
|
||||
|
||||
[[autodoc]] lerobot.configs.PreTrainedConfig
|
||||
|
||||
## make_policy
|
||||
|
||||
[[autodoc]] lerobot.policies.factory.make_policy
|
||||
@@ -1,20 +0,0 @@
|
||||
# Processors
|
||||
|
||||
Processors are the data transformation layer between a robot, a dataset and a policy. A pipeline is a chain
|
||||
of [`ProcessorStep`]s; each step declares how it transforms both the data and the feature contract.
|
||||
|
||||
See [Introduction to Robot Processors](../introduction_processors) for the concepts,
|
||||
[Implement your own processor](../implement_your_own_processor) to write a step, and
|
||||
[Debug your processor pipeline](../debug_processor_pipeline) when a pipeline misbehaves.
|
||||
|
||||
## ProcessorStep
|
||||
|
||||
[[autodoc]] lerobot.processor.pipeline.ProcessorStep
|
||||
|
||||
## DataProcessorPipeline
|
||||
|
||||
[[autodoc]] lerobot.processor.pipeline.DataProcessorPipeline
|
||||
|
||||
## PolicyProcessorPipeline
|
||||
|
||||
[[autodoc]] lerobot.processor.pipeline.PolicyProcessorPipeline
|
||||
@@ -1,147 +0,0 @@
|
||||
# Robots
|
||||
|
||||
Every robot in LeRobot implements the [`Robot`] interface: connect, read an observation, send an action,
|
||||
disconnect. Writing a policy or a recording script against that interface means it works with any supported
|
||||
arm without change.
|
||||
|
||||
This page is the generated reference. For wiring, calibration and first-run instructions, start with the
|
||||
hardware guides — [SO-101](../so101), [LeKiwi](../lekiwi), [Hope Jr](../hope_jr), [Reachy 2](../reachy2),
|
||||
[OpenArm](../openarm) — or [Imitation Learning for Robots](../il_robots) for the end-to-end workflow. To add
|
||||
a robot of your own, see [Bring Your Own Hardware](../integrate_hardware).
|
||||
|
||||
## Robot
|
||||
|
||||
The abstract base class. Subclasses implement every method below; the contract described here is what a
|
||||
policy or recording loop can rely on.
|
||||
|
||||
[[autodoc]] lerobot.robots.Robot
|
||||
- connect
|
||||
- disconnect
|
||||
- configure
|
||||
- calibrate
|
||||
- get_observation
|
||||
- send_action
|
||||
- observation_features
|
||||
- action_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
## RobotConfig
|
||||
|
||||
[[autodoc]] lerobot.robots.RobotConfig
|
||||
|
||||
## make_robot_from_config
|
||||
|
||||
[[autodoc]] lerobot.robots.make_robot_from_config
|
||||
|
||||
## SO-100 and SO-101 followers
|
||||
|
||||
`SO100Follower` and `SO101Follower` are aliases of the same `SOFollower` class; the two arms differ in their
|
||||
configuration, not their control code. `SO100FollowerConfig` and `SO101FollowerConfig` are likewise aliases
|
||||
of `SOFollowerRobotConfig`.
|
||||
|
||||
[[autodoc]] lerobot.robots.so_follower.SOFollower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.so_follower.SOFollowerRobotConfig
|
||||
|
||||
## BiSOFollower
|
||||
|
||||
Two SO followers driven as one bimanual robot.
|
||||
|
||||
[[autodoc]] lerobot.robots.bi_so_follower.BiSOFollower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.bi_so_follower.BiSOFollowerConfig
|
||||
|
||||
## KochFollower
|
||||
|
||||
[[autodoc]] lerobot.robots.koch_follower.KochFollower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.koch_follower.KochFollowerConfig
|
||||
|
||||
## LeKiwi
|
||||
|
||||
`LeKiwi` runs on the robot itself. `LeKiwiClient` is the host-side proxy that talks to it over the network
|
||||
and presents the same [`Robot`] interface.
|
||||
|
||||
[[autodoc]] lerobot.robots.lekiwi.LeKiwi
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.lekiwi.LeKiwiConfig
|
||||
|
||||
[[autodoc]] lerobot.robots.lekiwi.LeKiwiClient
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.lekiwi.LeKiwiClientConfig
|
||||
|
||||
## OpenArmFollower
|
||||
|
||||
[[autodoc]] lerobot.robots.openarm_follower.OpenArmFollower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.openarm_follower.OpenArmFollowerConfig
|
||||
|
||||
## BiOpenArmFollower
|
||||
|
||||
[[autodoc]] lerobot.robots.bi_openarm_follower.BiOpenArmFollower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.bi_openarm_follower.BiOpenArmFollowerConfig
|
||||
|
||||
## OmxFollower
|
||||
|
||||
[[autodoc]] lerobot.robots.omx_follower.OmxFollower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.omx_follower.OmxFollowerConfig
|
||||
|
||||
## Reachy2Robot
|
||||
|
||||
[[autodoc]] lerobot.robots.reachy2.Reachy2Robot
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.reachy2.Reachy2RobotConfig
|
||||
|
||||
## UnitreeG1
|
||||
|
||||
[[autodoc]] lerobot.robots.unitree_g1.UnitreeG1
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.unitree_g1.UnitreeG1Config
|
||||
|
||||
## Hope Jr
|
||||
|
||||
The Hope Jr humanoid is exposed as two independent robots, an arm and a hand.
|
||||
|
||||
[[autodoc]] lerobot.robots.hope_jr.HopeJrArm
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.hope_jr.HopeJrArmConfig
|
||||
|
||||
[[autodoc]] lerobot.robots.hope_jr.HopeJrHand
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.hope_jr.HopeJrHandConfig
|
||||
|
||||
## RebotB601Follower
|
||||
|
||||
[[autodoc]] lerobot.robots.rebot_b601_follower.RebotB601Follower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.rebot_b601_follower.RebotB601FollowerRobotConfig
|
||||
|
||||
## BiRebotB601Follower
|
||||
|
||||
[[autodoc]] lerobot.robots.bi_rebot_b601_follower.BiRebotB601Follower
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.bi_rebot_b601_follower.BiRebotB601FollowerConfig
|
||||
|
||||
## EarthRoverMiniPlus
|
||||
|
||||
[[autodoc]] lerobot.robots.earthrover_mini_plus.EarthRoverMiniPlus
|
||||
- all
|
||||
|
||||
[[autodoc]] lerobot.robots.earthrover_mini_plus.EarthRoverMiniPlusConfig
|
||||
@@ -1,30 +0,0 @@
|
||||
# Teleoperators
|
||||
|
||||
A teleoperator produces actions for a robot to follow — a leader arm, a gamepad, a keyboard, a phone. All of
|
||||
them implement the [`Teleoperator`] interface, so a recording script written against it works with any input
|
||||
device.
|
||||
|
||||
See [Phone teleoperation](../phone_teleop) and [Isaac Teleop](../isaac_teleop) for setup guides, and
|
||||
[Imitation Learning for Robots](../il_robots) for the recording workflow.
|
||||
|
||||
## Teleoperator
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.Teleoperator
|
||||
- connect
|
||||
- disconnect
|
||||
- configure
|
||||
- calibrate
|
||||
- get_action
|
||||
- send_feedback
|
||||
- action_features
|
||||
- feedback_features
|
||||
- is_connected
|
||||
- is_calibrated
|
||||
|
||||
## TeleoperatorConfig
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.TeleoperatorConfig
|
||||
|
||||
## make_teleoperator_from_config
|
||||
|
||||
[[autodoc]] lerobot.teleoperators.make_teleoperator_from_config
|
||||
@@ -161,16 +161,6 @@ The methods called by the train/eval loops:
|
||||
|
||||
Batches are flat dictionaries keyed by the constants in [`lerobot.utils.constants`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/utils/constants.py): `OBS_STATE` (`observation.state.<motor>`), `OBS_IMAGES` (`observation.images.<camera>`), `OBS_LANGUAGE`, `ACTION`, etc. Reuse the constants — don't invent new prefixes.
|
||||
|
||||
If your model is large enough to warrant [sharded multi-GPU training](./multi_gpu_training#sharded-training-fsdp), also declare its FSDP wrap units — the repeated block classes sharding operates on:
|
||||
|
||||
```python
|
||||
class MyPolicy(PreTrainedPolicy):
|
||||
...
|
||||
_fsdp_wrap_modules = ["MyTransformerBlock"]
|
||||
```
|
||||
|
||||
With this one declaration, `--parallelism.dp_shard=N` works out of the box for your policy (users can still override it with `--accelerator.fsdp.wrap_modules`). Without any wrap source, sharded runs fail at startup by design.
|
||||
|
||||
### Processor functions
|
||||
|
||||
LeRobot uses `PolicyProcessorPipeline`s to normalize inputs and de-normalize outputs around your policy. For a concrete reference, see [`processor_act.py`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/policies/act/processor_act.py) or [`processor_diffusion.py`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/policies/diffusion/processor_diffusion.py).
|
||||
@@ -310,7 +300,7 @@ The file names are load-bearing: the factory does lazy imports by name, and the
|
||||
Two places need to know about your policy. All by name.
|
||||
|
||||
1. **`policies/__init__.py`** — re-export `MyPolicyConfig` and add it to `__all__`. This import is what registers your policy: `@PreTrainedConfig.register_subclass("my_policy")` runs, and from then on the factory resolves everything by convention. **Don't** re-export the modeling class; it loads lazily through the factory (so `import lerobot` stays fast).
|
||||
2. **`templates/lerobot_modelcard_template.md` and the root `README.md`** — the template is what the end-of-training publisher renders into the model card of every checkpoint trained with your policy: add a one-line description of your policy in the `model_name` branches, map it in `policy_docs` so cards link to your MDX guide, and optionally add an architecture image to `diagrams`. Then add your policy to the models table in the root `README.md`, under the right category, linking to your doc page.
|
||||
2. **`templates/lerobot_modelcard_template.md` and the root `README.md`** — the template is what `push_model_to_hub` renders into the model card of every checkpoint trained with your policy: add a one-line description of your policy in the `model_name` branches, map it in `policy_docs` so cards link to your MDX guide, and optionally add an architecture image to `diagrams`. Then add your policy to the models table in the root `README.md`, under the right category, linking to your doc page.
|
||||
|
||||
Mirror an existing policy that's structurally similar to yours; the diff is small.
|
||||
|
||||
@@ -354,7 +344,7 @@ A new policy is much easier to review — and far more useful — when it ships
|
||||
|
||||
**Pick at least one in-tree benchmark.** LeRobot ships sim benchmarks with per-benchmark Docker images (LIBERO, LIBERO-plus, Meta-World, RoboTwin 2.0, RoboCasa365, RoboCerebra, RoboMME, VLABench and more). Pick the one that matches your policy's modality — VLAs usually go to LIBERO or VLABench; image-only BC to LIBERO or Meta-World. The full list lives under [Benchmarks](./libero) in the docs sidebar.
|
||||
|
||||
**Push the checkpoint & processors** to the Hub under `lerobot/<policy>_<benchmark>` (or your namespace if you don't have write access; a maintainer can mirror it). The easiest way is training with `--policy.repo_id=<namespace>/<repo>` and `--policy.push_to_hub=true`: `lerobot-train` publishes the model, both processors, and a model card at the end of the run. To publish an existing checkpoint after the fact, upload its `pretrained_model/` directory (e.g. `huggingface-cli upload`), or use `lerobot-convert-dcp --push_to_hub=...` for sharded-format checkpoints.
|
||||
**Push the checkpoint & processors** to the Hub under `lerobot/<policy>_<benchmark>` (or your namespace if you don't have write access; a maintainer can mirror it). Use `PreTrainedPolicy.push_model_to_hub` so the repo gets `config.json`, `model.safetensors`, and a model card.
|
||||
|
||||
**Report results in your policy's MDX**, with the exact `lerobot-eval` command and hardware so anyone can re-run:
|
||||
|
||||
|
||||
@@ -230,12 +230,13 @@ lerobot-rollout \
|
||||
--device=cuda
|
||||
```
|
||||
|
||||
| Flag | Description |
|
||||
| ------------------------------------------- | -------------------------------------------------------------- |
|
||||
| `--inference.rtc.execution_horizon` | Steps to blend with previous chunk (default: varies by policy) |
|
||||
| `--inference.rtc.max_guidance_weight` | Consistency enforcement strength (default: varies by policy) |
|
||||
| `--inference.rtc.prefix_attention_schedule` | Blend schedule: `LINEAR`, `EXP`, `ONES`, `ZEROS` |
|
||||
| `--inference.queue_threshold` | Max queue size before backpressure (default: 30) |
|
||||
| Flag | Description |
|
||||
| ------------------------------------------- | ------------------------------------------------------------------------------ |
|
||||
| `--inference.rtc.execution_horizon` | Steps to blend with previous chunk (default: varies by policy) |
|
||||
| `--inference.rtc.mode` | `guided` (default) or trained-prefix `trained` for compatible Pi05 checkpoints |
|
||||
| `--inference.rtc.max_guidance_weight` | Consistency enforcement strength (default: varies by policy) |
|
||||
| `--inference.rtc.prefix_attention_schedule` | Blend schedule: `LINEAR`, `EXP`, `ONES`, `ZEROS` |
|
||||
| `--inference.queue_threshold` | Backpressure threshold; trained RTC requires at least its maximum delay |
|
||||
|
||||
See the [Real-Time Chunking](./rtc) guide for details on tuning RTC parameters.
|
||||
|
||||
|
||||
+129
-125
@@ -1,29 +1,28 @@
|
||||
# Multi-GPU Training
|
||||
|
||||
LeRobot trains on multiple GPUs through [Hugging Face Accelerate](https://huggingface.co/docs/accelerate). Three data-parallel layouts are supported:
|
||||
|
||||
| Layout | What it does | Config |
|
||||
| -------- | ------------------------------------------------------------- | ------------------------------------------------------- |
|
||||
| **DDP** | Replicates the full model on every GPU | default on any multi-GPU launch |
|
||||
| **FSDP** | Shards parameters, gradients, and optimizer state across GPUs | `--parallelism.dp_shard=N` |
|
||||
| **HSDP** | Shards within groups of GPUs, replicates across groups | `--parallelism.dp_replicate=R --parallelism.dp_shard=S` |
|
||||
This guide shows you how to train policies on multiple GPUs using [Hugging Face Accelerate](https://huggingface.co/docs/accelerate).
|
||||
|
||||
## Installation
|
||||
|
||||
`accelerate` is included in the `training` extra:
|
||||
`accelerate` is included in the `training` extra. Install it with:
|
||||
|
||||
```bash
|
||||
pip install 'lerobot[training]'
|
||||
```
|
||||
|
||||
## Launching
|
||||
## Training with Multiple GPUs
|
||||
|
||||
Distributed training can be launched through both `torchrun` and `accelerate launch`. Accelerate is used as a plain launcher: it does not manage the training configuration, and every distributed training setting lives in LeRobot's own config system.
|
||||
You can launch training in two ways:
|
||||
|
||||
With `torchrun`:
|
||||
### Option 1: Without config (specify parameters directly)
|
||||
|
||||
You can specify all parameters directly in the command without running `accelerate config`:
|
||||
|
||||
```bash
|
||||
torchrun --nproc-per-node=2 $(which lerobot-train) \
|
||||
accelerate launch \
|
||||
--multi_gpu \
|
||||
--num_processes=2 \
|
||||
$(which lerobot-train) \
|
||||
--dataset.repo_id=${HF_USER}/my_dataset \
|
||||
--policy.type=act \
|
||||
--policy.repo_id=${HF_USER}/my_trained_policy \
|
||||
@@ -32,145 +31,150 @@ torchrun --nproc-per-node=2 $(which lerobot-train) \
|
||||
--wandb.enable=true
|
||||
```
|
||||
|
||||
With `accelerate launch` (as a plain launcher):
|
||||
**Key accelerate parameters:**
|
||||
|
||||
- `--multi_gpu`: Enable multi-GPU training
|
||||
- `--num_processes=2`: Number of GPUs to use
|
||||
- `--mixed_precision=fp16`: Use fp16 mixed precision (or `bf16` if supported)
|
||||
|
||||
### Option 2: Using accelerate config
|
||||
|
||||
If you prefer to save your configuration, you can optionally configure accelerate for your hardware setup by running:
|
||||
|
||||
```bash
|
||||
accelerate config
|
||||
```
|
||||
|
||||
This interactive setup will ask you questions about your training environment (number of GPUs, mixed precision settings, etc.) and saves the configuration for future use. For a simple multi-GPU setup on a single machine, you can use these recommended settings:
|
||||
|
||||
- Compute environment: This machine
|
||||
- Number of machines: 1
|
||||
- Number of processes: (number of GPUs you want to use)
|
||||
- GPU ids to use: (leave empty to use all)
|
||||
- Mixed precision: fp16 or bf16 (recommended for faster training)
|
||||
|
||||
Then launch training with:
|
||||
|
||||
```bash
|
||||
accelerate launch $(which lerobot-train) \
|
||||
--dataset.repo_id=${HF_USER}/my_dataset \
|
||||
--policy.type=act \
|
||||
--policy.repo_id=${HF_USER}/my_trained_policy \
|
||||
--output_dir=outputs/train/act_multi_gpu \
|
||||
--job_name=act_multi_gpu \
|
||||
--wandb.enable=true
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
When you launch training with accelerate:
|
||||
|
||||
1. **Automatic detection**: LeRobot automatically detects if it's running under accelerate
|
||||
2. **Data distribution**: Your batch is automatically split across GPUs
|
||||
3. **Gradient synchronization**: Gradients are synchronized across GPUs during backpropagation
|
||||
4. **Single process logging**: Only the main process logs to wandb and saves checkpoints
|
||||
|
||||
## Learning Rate and Training Steps Scaling
|
||||
|
||||
**Important:** LeRobot does **NOT** automatically scale learning rates or training steps based on the number of GPUs. This gives you full control over your training hyperparameters.
|
||||
|
||||
### Why No Automatic Scaling?
|
||||
|
||||
Many distributed training frameworks automatically scale the learning rate by the number of GPUs (e.g., `lr = base_lr × num_gpus`).
|
||||
However, LeRobot keeps the learning rate exactly as you specify it.
|
||||
|
||||
### When and How to Scale
|
||||
|
||||
If you want to scale your hyperparameters when using multiple GPUs, you should do it manually:
|
||||
|
||||
**Learning Rate Scaling:**
|
||||
|
||||
```bash
|
||||
# Example: 2 GPUs with linear LR scaling
|
||||
# Base LR: 1e-4, with 2 GPUs -> 2e-4
|
||||
accelerate launch --num_processes=2 $(which lerobot-train) \
|
||||
--dataset.repo_id=${HF_USER}/my_dataset \
|
||||
--policy.type=act \
|
||||
--policy.repo_id=${HF_USER}/my_trained_policy \
|
||||
--output_dir=outputs/train/act_multi_gpu \
|
||||
--job_name=act_multi_gpu \
|
||||
--wandb.enable=true
|
||||
--optimizer.lr=2e-4 \
|
||||
--dataset.repo_id=lerobot/pusht \
|
||||
--policy.type=act
|
||||
```
|
||||
|
||||
With no `--parallelism.*` flags, a multi-process launch runs plain DDP. Multi-node runs use the standard `torchrun --nnodes/--node-rank/--rdzv-endpoint` flags (or `accelerate launch --num_machines/--machine_rank/--main_process_ip`).
|
||||
**Training Steps Scaling:**
|
||||
|
||||
> [!WARNING]
|
||||
> Accelerate's YAML config files (`accelerate launch --config_file some.yaml`, `accelerate config`) are not supported. They configure the engine through environment variables, bypassing LeRobot's configuration system, so `train_config.json` would no longer describe the settings a run actually used. `lerobot-train` therefore refuses to start when [accelerate environment variables](https://huggingface.co/docs/accelerate/usage_guides/fsdp) are set. Put the settings in `--parallelism.*` / `--accelerator.*` flags instead, or set `LEROBOT_ALLOW_ACCELERATE_ENV=1` to acknowledge the override and proceed anyway.
|
||||
|
||||
## Batch semantics, learning rate, and steps
|
||||
|
||||
Each of the `dp_replicate × dp_shard` data-parallel workers loads its own `--batch_size` micro-batch every step, so one training step consumes `batch_size × dp_world_size` samples, and `× gradient_accumulation_steps` of those go into each optimizer update:
|
||||
|
||||
```
|
||||
effective_batch_size = batch_size × dp_world_size × gradient_accumulation_steps
|
||||
```
|
||||
|
||||
The training banner prints this factorization at startup. `--steps` counts loop steps (micro-batches per worker), not optimizer updates.
|
||||
|
||||
Gradient accumulation is a first-class flag:
|
||||
Since the effective batch size `bs` increases with multiple GPUs (batch_size × num_gpus), you may want to reduce the number of training steps proportionally:
|
||||
|
||||
```bash
|
||||
torchrun --nproc-per-node=2 $(which lerobot-train) \
|
||||
--batch_size=8 --accelerator.gradient_accumulation.steps=4 ...
|
||||
# Example: 2 GPUs with effective batch size 2x larger
|
||||
# Original: batch_size=8, steps=100000
|
||||
# With 2 GPUs: batch_size=8 (16 in total), steps=50000
|
||||
accelerate launch --num_processes=2 $(which lerobot-train) \
|
||||
--batch_size=8 \
|
||||
--steps=50000 \
|
||||
--dataset.repo_id=lerobot/pusht \
|
||||
--policy.type=act
|
||||
```
|
||||
|
||||
**LeRobot does not auto-scale the learning rate or the number of steps** when the effective batch size grows. If you scale out and want equivalent training, please adjust manually, e.g. with 2 GPUs: double `--optimizer.lr` (linear scaling), or halve `--steps`.
|
||||
## Training Large Models with FSDP
|
||||
|
||||
## Sharded training (FSDP)
|
||||
DDP replicates the full model on every GPU, so a model that doesn't fit on one GPU won't fit under
|
||||
DDP either. For large models, use **FSDP** (Fully Sharded Data Parallel), which shards parameters,
|
||||
gradients, and optimizer state across GPUs. See the [accelerate FSDP guide](https://huggingface.co/docs/accelerate/usage_guides/fsdp) for background.
|
||||
|
||||
If a model is too large to train with DDP, shard it with FSDP2:
|
||||
An example on how to launch LeRobot training with FSDP across 4 GPUs (1 machine):
|
||||
|
||||
```bash
|
||||
torchrun --nproc-per-node=4 $(which lerobot-train) \
|
||||
accelerate launch --config_file fsdp.yaml --num_processes=4 $(which lerobot-train) \
|
||||
--dataset.repo_id=${HF_USER}/my_dataset \
|
||||
--policy.type=<your_policy> \
|
||||
--parallelism.dp_shard=4 \
|
||||
--accelerator.mixed_precision=bf16 \
|
||||
--output_dir=outputs/train/my_policy_fsdp
|
||||
```
|
||||
|
||||
`--parallelism.dp_shard=-1` shards over however many processes the launcher started.
|
||||
A minimal `fsdp.yaml` (FSDP1; shards params/grads/optimizer — ZeRO-3-equivalent):
|
||||
|
||||
### Wrap units
|
||||
|
||||
FSDP shards the model in units (typically the repeated transformer block) and gathers one unit at a time during forward/backward. Policies declare their wrap units via `_fsdp_wrap_modules` on the policy class. For example, ACT declares `["ACTEncoderLayer", "ACTDecoderLayer"]` and FastWAM declares `["MoTLayer"]`. For a policy without a `_fsdp_wrap_modules` declaration, pass one of the flags below. You can specify the module class name explicitly, or use a size-based policy instead:
|
||||
|
||||
```bash
|
||||
--accelerator.fsdp.wrap_modules='["MyTransformerBlock"]' # explicit class names
|
||||
--accelerator.fsdp.min_num_params=1000000 # or: wrap every submodule above 1M params
|
||||
```yaml
|
||||
compute_environment: LOCAL_MACHINE
|
||||
distributed_type: FSDP
|
||||
mixed_precision: bf16
|
||||
num_machines: 1
|
||||
num_processes: 4
|
||||
fsdp_config:
|
||||
fsdp_version: 1
|
||||
fsdp_sharding_strategy: FULL_SHARD # params + grads + optimizer (ZeRO-3)
|
||||
fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
|
||||
fsdp_transformer_layer_cls_to_wrap: <YourTransformerBlock> # repeated block class to shard
|
||||
fsdp_use_orig_params: true # required: optimizer is built pre-prepare
|
||||
fsdp_state_dict_type: FULL_STATE_DICT
|
||||
```
|
||||
|
||||
If a policy doesn't declare `_fsdp_wrap_modules` and no `--accelerator.fsdp.wrap_modules` or `--accelerator.fsdp.min_num_params` is passed, the run fails at startup rather than silently wrapping only the root module (which would forfeit all sharding memory savings).
|
||||
Set `fsdp_transformer_layer_cls_to_wrap` to your model's repeated transformer-block class so each
|
||||
block is sharded as its own unit. `fsdp_use_orig_params: true` is required because LeRobot builds the
|
||||
optimizer before `accelerator.prepare()`.
|
||||
|
||||
Other sharding settings:
|
||||
### FSDP checkpoints
|
||||
|
||||
- `--accelerator.fsdp.reshard_after_forward`: whether to keep each unit's parameters resident after forward.
|
||||
- `--accelerator.fsdp.cpu_offload`: keeps parameters, gradients and optimizer states on CPU.
|
||||
- `--accelerator.fsdp.ignored_modules`: a regex of module paths to keep unsharded.
|
||||
LeRobot gathers the full state dict across all ranks and the main process writes it as a single
|
||||
`model.safetensors`, loadable as usual with `Policy.from_pretrained(...)`. Two things to look out for:
|
||||
|
||||
### HSDP
|
||||
|
||||
Hybrid Sharded Data Parallel: parameters, gradients and optimizer states are sharded across `dp_shard` ranks, and that sharding is replicated `dp_replicate` times. Parameter all-gathers and gradient reduce-scatters stay inside a shard group; only the all-reduce that synchronizes the replicas crosses between groups. The two degrees must multiply to the world size:
|
||||
|
||||
```bash
|
||||
# 16 GPUs = 2 nodes × 8: shard within each node, replicate across nodes
|
||||
torchrun --nnodes=2 --nproc-per-node=8 ... $(which lerobot-train) \
|
||||
--parallelism.dp_replicate=2 --parallelism.dp_shard=8 ...
|
||||
```
|
||||
|
||||
## Checkpoints
|
||||
|
||||
Every checkpoint contains a `pretrained_model/` directory and a `training_state/` directory:
|
||||
|
||||
```text
|
||||
005000/ # the training step at that checkpoint
|
||||
├── pretrained_model/
|
||||
│ ├── config.json # policy config
|
||||
│ ├── train_config.json # the full training config
|
||||
│ ├── model.safetensors # full weights (checkpoint_format ∈ {safetensors, safetensors_dcp}, or any non-sharded run)
|
||||
│ ├── pytorch_model_fsdp_0/ # DCP weight shards (checkpoint_format ∈ {dcp, safetensors_dcp})
|
||||
│ ├── policy_preprocessor.json # preprocessor config (when the run has a preprocessor)
|
||||
│ ├── policy_preprocessor_step_*.safetensors # state of the stateful preprocessor steps
|
||||
│ ├── policy_postprocessor.json # postprocessor config (when the run has a postprocessor)
|
||||
│ └── policy_postprocessor_step_*.safetensors # state of the stateful postprocessor steps
|
||||
└── training_state/
|
||||
├── training_step.json # step counter, topology, and batch semantics
|
||||
├── rng_state.safetensors # rng states
|
||||
├── scheduler_state.json # scheduler state (when the run has a scheduler)
|
||||
├── optimizer_state.safetensors # full optimizer state (non-sharded runs)
|
||||
├── optimizer_param_groups.json # optimizer param groups (non-sharded runs)
|
||||
└── optimizer_0/ # DCP optimizer shards (sharded runs)
|
||||
```
|
||||
|
||||
During single-GPU or DDP training, the pipeline serializes each state dict into a single file: `model.safetensors` for the model and `optimizer_state.safetensors` for the optimizer.
|
||||
|
||||
During sharded training, the optimizer state is saved as DCP shards under `training_state/optimizer_0/`, and the layout of the model under `pretrained_model/` can be configured through `--checkpoint_format`:
|
||||
|
||||
| `--checkpoint_format` | Weights artifact | Use when |
|
||||
| ------------------------- | -------------------------------------------- | --------------------------------------------------------------------- |
|
||||
| `safetensors` _(default)_ | single `model.safetensors` only | you want every checkpoint immediately loadable with `from_pretrained` |
|
||||
| `dcp` | `pytorch_model_fsdp_0/` shard directory only | gathering the full weights makes saves and resumes too slow |
|
||||
| `safetensors_dcp` | both | you want fast resume _and_ immediately loadable checkpoints |
|
||||
|
||||
Two things to know about gathered (`safetensors`) checkpoints from sharded runs:
|
||||
|
||||
- **They store fp32 weights.** Under mixed precision training, FSDP keeps an fp32 master copy, and the checkpoint saves the master copy to make sure training resumes consistently.
|
||||
- The gather is collective (all ranks participate) but only the main process writes.
|
||||
|
||||
### Converting DCP checkpoints
|
||||
|
||||
`lerobot-convert-dcp` merges a DCP shard directory into a regular `model.safetensors`, offline and without GPUs:
|
||||
|
||||
```bash
|
||||
lerobot-convert-dcp --checkpoint_dir=outputs/train/run/checkpoints/005000
|
||||
lerobot-convert-dcp --checkpoint_dir=... --delete_dcp=true --push_to_hub=${HF_USER}/my_policy
|
||||
```
|
||||
|
||||
`--push_to_hub` publishes the converted directory as a model repo.
|
||||
|
||||
### Resuming
|
||||
|
||||
Resume with `--resume=true --config_path=.../checkpoints/last/pretrained_model/train_config.json`. Resuming from a DCP checkpoint supports resharding the model and optimizer state to the _current_ topology, which means you can resume with a different `dp_replicate/dp_shard` split. The data sampler can always resume at the right epoch and offset, but is only _sample-exact_ when the world size and batch size match the original run (a warning is logged otherwise).
|
||||
|
||||
> [!NOTE]
|
||||
> FSDP checkpoints written by LeRobot 0.6.x and earlier used a different on-disk layout (a gathered full optimizer state) and **cannot be resumed**.
|
||||
- **Checkpoints store fp32 weights.** Under mixed precision (`bf16`/`fp16`) FSDP keeps an fp32 master
|
||||
copy, and the checkpoint saves it (~2× the bf16 size on disk) so training can resume consistently
|
||||
with the fp32 optimizer state; `from_pretrained` casts back to the policy dtype on load. FSDP-specific
|
||||
caveat: an fp32 checkpoint is materialized in full precision on the target device _before_ casting,
|
||||
so loading it for inference on a tight GPU can OOM even when the bf16 model would fit — load on CPU
|
||||
first, or cast `model.safetensors` to the deployment dtype offline.
|
||||
- The sharded optimizer state is gathered into a full (world-size-independent) state dict and saved
|
||||
alongside the model in the same `optimizer_state.safetensors` / `optimizer_param_groups.json`
|
||||
format as single-GPU training, so **resume-from-checkpoint is supported** with `--resume=true`.
|
||||
Resume reshards both the model and the optimizer state to the _current_ FSDP topology, so you can
|
||||
resume an FSDP checkpoint on a different number of GPUs. Note that the data sampler is only
|
||||
sample-exact when the world size and batch size match the original run (a warning is logged
|
||||
otherwise); the optimizer/model state itself is unaffected.
|
||||
|
||||
## Notes
|
||||
|
||||
- Checkpoint saves and end-of-training publishes are collective (every rank enters them). Gathered weights, sidecar files and Hub uploads are written by the main process alone.
|
||||
- Metrics are reduced across ranks before logging: losses are averaged, and `samples/s` reports cluster-wide throughput.
|
||||
- Learning-rate scheduling is stepped once per training step regardless of the number of processes (`step_scheduler_with_optimizer=False` is baked in).
|
||||
- The `--policy.use_amp` flag in `lerobot-train` is only used when **not** running with accelerate. When using accelerate, mixed precision is controlled by accelerate's configuration.
|
||||
- Training logs, checkpoints, and hub uploads are only done by the main process to avoid conflicts. Non-main processes have console logging disabled to prevent duplicate output.
|
||||
- The effective batch size is `batch_size × num_gpus`. If you use 4 GPUs with `--batch_size=8`, your effective batch size is 32.
|
||||
- Learning rate scheduling is handled correctly across multiple processes—LeRobot sets `step_scheduler_with_optimizer=False` to prevent accelerate from adjusting scheduler steps based on the number of processes.
|
||||
- When saving or pushing models, LeRobot automatically unwraps the model from accelerate's distributed wrapper to ensure compatibility.
|
||||
- WandB integration automatically initializes only on the main process, preventing multiple runs from being created.
|
||||
|
||||
For background on the underlying machinery, see the [Accelerate FSDP guide](https://huggingface.co/docs/accelerate/usage_guides/fsdp). To go deeper on large-scale training, check out the [Ultrascale Playbook](https://huggingface.co/spaces/nanotron/ultrascale-playbook).
|
||||
For more advanced configurations and troubleshooting, see the [Accelerate documentation](https://huggingface.co/docs/accelerate). If you want to learn more about how to train on a large number of GPUs, checkout this awesome guide: [Ultrascale Playbook](https://huggingface.co/spaces/nanotron/ultrascale-playbook).
|
||||
|
||||
@@ -164,6 +164,56 @@ lerobot-train \
|
||||
- [lerobot/pi05_base](https://huggingface.co/lerobot/pi05_base)
|
||||
- [lerobot/pi05_libero_base](https://huggingface.co/lerobot/pi05_libero_base) (specifically trained on the Libero dataset)
|
||||
|
||||
### Optional training-time RTC
|
||||
|
||||
Pi05 can optionally learn action-prefix conditioning for efficient
|
||||
[Real-Time Chunking](./rtc), following
|
||||
[Training-Time Action Conditioning for Efficient Real-Time Chunking](https://arxiv.org/abs/2512.05964).
|
||||
Set `policy.rtc_training_max_delay` to the largest expected inference delay in
|
||||
controller steps:
|
||||
|
||||
```bash
|
||||
lerobot-train \
|
||||
--dataset.repo_id=${HF_USER}/my_dataset \
|
||||
--policy.type=pi05 \
|
||||
--policy.pretrained_path=lerobot/pi05_base \
|
||||
--policy.rtc_training_max_delay=10 \
|
||||
--policy.dtype=bfloat16 \
|
||||
--policy.device=cuda \
|
||||
--batch_size=8 \
|
||||
--steps=30000 \
|
||||
--output_dir=outputs/pi05_rtc \
|
||||
--job_name=pi05_rtc
|
||||
```
|
||||
|
||||
The default is `0`, which disables training-time RTC and leaves the standard
|
||||
Pi05 objective unchanged. The configured delay must be smaller than
|
||||
`chunk_size`. During training, Pi05 samples a clean prefix length independently
|
||||
for each example and computes flow loss only on the remaining postfix.
|
||||
|
||||
Use the resulting checkpoint with trained RTC explicitly:
|
||||
|
||||
```bash
|
||||
lerobot-rollout \
|
||||
--strategy.type=base \
|
||||
--policy.path=outputs/pi05_rtc/checkpoints/last/pretrained_model \
|
||||
--inference.type=rtc \
|
||||
--inference.rtc.mode=trained \
|
||||
--inference.rtc.execution_horizon=10 \
|
||||
--robot.type=so100_follower \
|
||||
--robot.port=/dev/ttyACM0 \
|
||||
--task="pick up the cube" \
|
||||
--fps=50 \
|
||||
--device=cuda
|
||||
```
|
||||
|
||||
Set `inference.rtc.execution_horizon` between the checkpoint's
|
||||
`rtc_training_max_delay` and `chunk_size - rtc_training_max_delay`, and
|
||||
`inference.queue_threshold` to at least `rtc_training_max_delay`. A one-off
|
||||
latency spike past the trained range discards that chunk and retries; rollout
|
||||
stops with an explicit error only if the delay stays out of range. The default
|
||||
`--inference.rtc.mode=guided` remains available for ordinary checkpoints.
|
||||
|
||||
### Loading a checkpoint
|
||||
|
||||
The two forms are not interchangeable:
|
||||
|
||||
+37
-1
@@ -1,6 +1,6 @@
|
||||
# Real-Time Chunking (RTC)
|
||||
|
||||
Real-Time Chunking (RTC) is an inference-time method that allows large, flow-matching based robotic policies, such as [Pi0](./pi0), [Pi0.5](./pi05), and [SmolVLA](./smolvla), to produce smooth, continuous, and reactive motion despite having high inference latency.
|
||||
Real-Time Chunking (RTC) allows large, flow-matching based robotic policies, such as [Pi0](./pi0), [Pi0.5](./pi05), and [SmolVLA](./smolvla), to produce smooth, continuous, and reactive motion despite having high inference latency. LeRobot provides the original inference-time guided mode and, for compatible Pi05 checkpoints, training-time action conditioning with cheap hard-prefix inference.
|
||||
|
||||
These policies generate chunks of future actions (e.g., 50 steps at a time) instead of single actions.
|
||||
Because the models are large, producing each chunk takes longer than the time it takes the robot to execute it.
|
||||
@@ -92,6 +92,17 @@ for step in range(num_steps):
|
||||
|
||||
`RTCConfig` has the following parameters to tune:
|
||||
|
||||
**`mode`** selects the action-prefix conditioning method:
|
||||
|
||||
- `guided` (default) applies the original Jacobian guidance during denoising and works with ordinary flow-matching checkpoints.
|
||||
- `trained` hard-inpaints the previous chunk's prefix with per-action flow timesteps. It currently requires a Pi05 checkpoint trained with `policy.rtc_training_max_delay > 0` and avoids the guidance backward pass.
|
||||
|
||||
For trained mode, both `execution_horizon` and the rollout backend's
|
||||
`inference.queue_threshold` must be at least the checkpoint's
|
||||
`rtc_training_max_delay`, and `execution_horizon` must not exceed
|
||||
`chunk_size - rtc_training_max_delay` (the `d <= s <= H - d` bound from the RTC
|
||||
paper); rollout validates this before connecting the robot.
|
||||
|
||||
**`execution_horizon`**: How many timesteps from the previous chunk to maintain consistency with. Higher values mean smoother transitions but potentially less reactivity.
|
||||
|
||||
Typical values: 8-12 steps
|
||||
@@ -124,6 +135,10 @@ python examples/rtc/eval_dataset.py \
|
||||
--device=cuda
|
||||
```
|
||||
|
||||
Add `--rtc.mode=trained` when evaluating a compatible training-time RTC Pi05
|
||||
checkpoint. Unsupported policies reject trained mode instead of falling back to
|
||||
guided RTC.
|
||||
|
||||
The script generates a visualization of the denoising process, comparing standard generation (left) with RTC (right). In the RTC plots, you can see how the first few steps (blue/purple lines) are guided to match the red ground truth trajectory (previous chunk's tail), ensuring a smooth transition between chunks.
|
||||
|
||||
<p align="center">
|
||||
@@ -141,6 +156,7 @@ lerobot-rollout \
|
||||
--strategy.type=base \
|
||||
--policy.path=${HF_USERNAME}/policy_repo_id \
|
||||
--inference.type=rtc \
|
||||
--inference.rtc.mode=guided \
|
||||
--inference.rtc.execution_horizon=10 \
|
||||
--inference.rtc.max_guidance_weight=10.0 \
|
||||
--robot.type=so100_follower \
|
||||
@@ -151,6 +167,24 @@ lerobot-rollout \
|
||||
--device=cuda
|
||||
```
|
||||
|
||||
For a training-time RTC Pi05 checkpoint, change the mode to `trained`. The
|
||||
checkpoint records its maximum supported delay, and rollout validates measured
|
||||
latency against it:
|
||||
|
||||
```bash
|
||||
lerobot-rollout \
|
||||
--strategy.type=base \
|
||||
--policy.path=${HF_USERNAME}/pi05_training_rtc \
|
||||
--inference.type=rtc \
|
||||
--inference.rtc.mode=trained \
|
||||
--inference.rtc.execution_horizon=10 \
|
||||
--robot.type=so100_follower \
|
||||
--robot.port=/dev/tty.usbmodem58FA0834591 \
|
||||
--task="Move green small object into the purple platform" \
|
||||
--duration=120 \
|
||||
--device=cuda
|
||||
```
|
||||
|
||||
## How It Differs from the Async Inference in LeRobot
|
||||
|
||||
Both RTC and [async inference](./async) improve real-time robot control, but they solve different problems.
|
||||
@@ -189,3 +223,5 @@ See `examples/rtc/eval_dataset.py` for a complete example of offline RTC visuali
|
||||
- [Smooth-As-Butter Robot Policies](https://alexander-soare.github.io/robotics/2025/08/05/smooth-as-butter-robot-policies.html) - Excellent technical explanation with real robot results
|
||||
- [Physical Intelligence - Real-Time Chunking](https://www.physicalintelligence.company/research/real_time_chunking) - Original paper and research
|
||||
- [Kinetix RTC Implementation](https://github.com/Physical-Intelligence/real-time-chunking-kinetix) - Reference implementation from Physical Intelligence
|
||||
- [Training-Time Action Conditioning](https://arxiv.org/abs/2512.05964) - Efficient RTC with clean-prefix conditioning during training
|
||||
- [RLDX-1](https://github.com/RLWRLD/RLDX-1) - PyTorch reference used for the training-time RTC integration
|
||||
|
||||
@@ -40,15 +40,3 @@ lerobot-eval \
|
||||
```
|
||||
|
||||
However, in most cases, presence of an accelerator is detected automatically and `policy.device` parameter can be omitted from CLI commands.
|
||||
|
||||
## Mixed precision
|
||||
|
||||
Training precision is owned by `--accelerator.mixed_precision`, which accepts `no` (default) and `bf16`:
|
||||
|
||||
```bash
|
||||
lerobot-train \
|
||||
--policy.type=act \
|
||||
--accelerator.mixed_precision=bf16 ...
|
||||
```
|
||||
|
||||
`bf16` requires an accelerator that supports it.
|
||||
|
||||
@@ -1,287 +0,0 @@
|
||||
# Writing docstrings
|
||||
|
||||
LeRobot's API reference is generated directly from the docstrings in `src/lerobot/`. A docstring is not a
|
||||
comment — it is the published documentation for that object, and the format below is what the renderer and
|
||||
the CI checks parse.
|
||||
|
||||
This page is the contract. If you are adding or editing anything public in `src/lerobot/`, follow it.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **An undocumented public method is an invisible one.** `[[autodoc]]` silently skips members that have no
|
||||
> docstring — no warning, no error, it simply does not appear on the rendered page. Coverage and
|
||||
> API-reference completeness are the same problem.
|
||||
|
||||
## The format in one example
|
||||
|
||||
Google section headers, Hugging Face type formatting. Both, not one or the other.
|
||||
|
||||
````python
|
||||
def send_action(self, action: RobotAction, rate_hz: float = 30.0) -> RobotAction:
|
||||
"""Command the robot to move to a target joint configuration.
|
||||
|
||||
Values are clipped by the configured maximum relative target before reaching the motors, so the
|
||||
returned action may differ from the requested one.
|
||||
|
||||
Args:
|
||||
action (`dict[str, float]`):
|
||||
Target values keyed by motor name, e.g. `{"shoulder_pan.pos": 0.0}`. Keys must match the
|
||||
robot's action features.
|
||||
rate_hz (`float`, *optional*, defaults to `30.0`):
|
||||
Control loop frequency.
|
||||
|
||||
Returns:
|
||||
`dict[str, float]`: The action actually written to the motors after safety clipping.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot has not been connected.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.robots.so_follower import SO101Follower, SO101FollowerConfig
|
||||
>>> robot = SO101Follower(SO101FollowerConfig(port="/dev/ttyACM0")) # doctest: +SKIP
|
||||
>>> robot.connect() # doctest: +SKIP
|
||||
>>> robot.send_action({"shoulder_pan.pos": 0.0}) # doctest: +SKIP
|
||||
```
|
||||
"""
|
||||
````
|
||||
|
||||
Cross-references are omitted from the examples on this page — see [Cross-references](#cross-references) for
|
||||
their syntax and why they cannot be shown inside a code block.
|
||||
|
||||
## Rules
|
||||
|
||||
### Sections
|
||||
|
||||
`Args:` · `Returns:` · `Raises:` · `Yields:` · `Example:` · `Note:`
|
||||
|
||||
In that order. No other section headers. A one-line summary comes first, then an optional free-form
|
||||
description, then the sections.
|
||||
|
||||
### The `Args:` line is machine-parsed
|
||||
|
||||
```
|
||||
name (`type`, *optional*, defaults to `X`):
|
||||
Description, indented on its own line.
|
||||
```
|
||||
|
||||
The `*optional*, defaults to` clause is **checked against the real signature default** by
|
||||
`make check-docstrings`. It is not decorative — if you write a default that has drifted from the code, CI
|
||||
fails. Omit the clause entirely for required parameters:
|
||||
|
||||
```python
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to, e.g. `/dev/ttyACM0`.
|
||||
max_relative_target (`float | dict[str, float]`, *optional*):
|
||||
Caps the magnitude of the relative positional target vector. `None` disables clipping.
|
||||
use_degrees (`bool`, *optional*, defaults to `True`):
|
||||
Keep `True` for backward compatibility with existing policies and datasets.
|
||||
```
|
||||
|
||||
Types go in backticks. Use `*optional*` with no `defaults to` when the default is `None` or is otherwise not
|
||||
worth restating.
|
||||
|
||||
### `Returns:` is type-first
|
||||
|
||||
One indented line, type first, then a colon, then the description:
|
||||
|
||||
```python
|
||||
Returns:
|
||||
`dict[str, float]`: The action actually written to the motors after safety clipping.
|
||||
```
|
||||
|
||||
`Yields:` takes the same shape.
|
||||
|
||||
### `**Attributes**:`, never `Attributes:`
|
||||
|
||||
doc-builder parses a bare `Attributes:` as a **synonym for `Parameters:`**, so your attributes get rendered
|
||||
as constructor arguments. This is silent and wrong. Whenever the attributes differ from the constructor
|
||||
parameters, use the bold form with a `--` separator:
|
||||
|
||||
```python
|
||||
class Robot(abc.ABC):
|
||||
"""The base abstract class for all LeRobot-compatible robots.
|
||||
|
||||
**Attributes**:
|
||||
- **config_class** (`type[RobotConfig]`) -- The expected configuration class for this robot.
|
||||
- **name** (`str`) -- The unique robot name used to identify this robot type.
|
||||
"""
|
||||
```
|
||||
|
||||
Note `--`, not `:`.
|
||||
|
||||
### Cross-references
|
||||
|
||||
Use doc-builder's bracket syntax: a square-bracketed backtick-quoted path. **Sphinx roles (`:pymeth:`,
|
||||
`:pyattr:`) are not supported** and render as literal text on the page.
|
||||
|
||||
| Want | Write |
|
||||
| ---------------------------- | ----------------------------------- |
|
||||
| Class in the main package | [`Robot`] |
|
||||
| Method, show the full path | [`Robot.connect`] |
|
||||
| Method, show the bare name | [`~Robot.connect`] |
|
||||
| Nested path | [`~robots.Robot.connect`] |
|
||||
| Object in another HF library | [`~accelerate.Accelerator`] |
|
||||
|
||||
The `~` strips the path from the **link text only**; the link still resolves to the full path.
|
||||
|
||||
> [!NOTE]
|
||||
> doc-builder resolves this syntax everywhere in a page — including inside fenced code blocks. That is why
|
||||
> the docstring examples on this page use plain prose instead of cross-references: a code block containing
|
||||
> one would render the resolved link rather than the syntax you need to type. In your own docstrings, use
|
||||
> cross-references freely; this restriction only affects documentation _about_ the syntax.
|
||||
|
||||
### Callouts
|
||||
|
||||
Use GitHub-style blockquotes:
|
||||
|
||||
```markdown
|
||||
> [!TIP]
|
||||
> Call this once at startup — it takes about two seconds.
|
||||
|
||||
> [!WARNING]
|
||||
> Torque is disabled on disconnect. The arm will drop if it is holding a load.
|
||||
```
|
||||
|
||||
The `<Tip>` component is legacy per doc-builder; don't add new ones.
|
||||
|
||||
### Examples must be fenced
|
||||
|
||||
An example lives inside a fenced ` ```python ` block containing `>>> `. The fence is what makes it render
|
||||
as a code block, and it is what the doctest preprocessor's regex looks for:
|
||||
|
||||
````python
|
||||
Example:
|
||||
```python
|
||||
>>> from lerobot.robots.so_follower import SO101FollowerConfig
|
||||
>>> cfg = SO101FollowerConfig(port="/dev/ttyACM0")
|
||||
>>> cfg.use_degrees
|
||||
True
|
||||
```
|
||||
````
|
||||
|
||||
> [!WARNING]
|
||||
> An unfenced `>>>` is still collected — doctest finds prompts anywhere in a docstring. What you lose is the
|
||||
> rendering, so it shows up as a wall of prose on the page. Every example needs the fence.
|
||||
|
||||
Every example either executes in CI or carries `# doctest: +SKIP`. Anything that touches hardware, a GPU, or
|
||||
downloads from the Hub gets `+SKIP`:
|
||||
|
||||
````python
|
||||
Example:
|
||||
```python
|
||||
>>> robot.connect() # doctest: +SKIP
|
||||
>>> policy = ACTPolicy.from_pretrained("lerobot/act_aloha_sim_transfer_cube_human") # doctest: +SKIP
|
||||
```
|
||||
````
|
||||
|
||||
Add files containing runnable examples to `utils/documentation_tests.txt`.
|
||||
|
||||
Put examples on the three to five genuine entry points of a module. Examples on trivial accessors are noise.
|
||||
|
||||
## Three patterns you will hit constantly
|
||||
|
||||
### Config dataclasses
|
||||
|
||||
Configuration fields are historically documented with `#` comments above each field. **doc-builder cannot
|
||||
see inline comments** — such a class renders with every field listed and not a single description. Move them
|
||||
into an `Args:` block on the class docstring:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SOFollowerConfig:
|
||||
"""Configuration for SO-family follower arms.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to, e.g. `/dev/ttyACM0`.
|
||||
max_relative_target (`float | dict[str, float]`, *optional*):
|
||||
Caps the magnitude of the relative positional target vector. A scalar applies to all motors;
|
||||
a dict maps motor name to a per-motor cap. `None` disables clipping.
|
||||
use_degrees (`bool`, *optional*, defaults to `True`):
|
||||
Keep `True` for backward compatibility with existing policies and datasets.
|
||||
"""
|
||||
|
||||
port: str
|
||||
max_relative_target: float | dict[str, float] | None = None
|
||||
use_degrees: bool = True
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **doc-builder does not inherit docstrings from base classes.** LeRobot's registered config classes are
|
||||
> often thin multiple-inheritance shims:
|
||||
>
|
||||
> ```python
|
||||
> @RobotConfig.register_subclass("so101_follower")
|
||||
> @dataclass
|
||||
> class SOFollowerRobotConfig(RobotConfig, SOFollowerConfig):
|
||||
> pass
|
||||
> ```
|
||||
>
|
||||
> That class renders **every** field — including the ones it inherits — with no descriptions at all, no
|
||||
> matter how well the bases are documented. The `Args:` block must live on the concrete class that
|
||||
> `[[autodoc]]` names, and it must cover inherited fields too.
|
||||
|
||||
### Base class, then concrete subclass
|
||||
|
||||
The abstract base carries the canonical contract. Subclasses document only what deviates — port semantics,
|
||||
calibration quirks, motor layout, supported feature keys. Do not copy the base contract into every subclass.
|
||||
|
||||
`Robot`, `Teleoperator`, `Camera`, `MotorsBus`, `ProcessorStep`, and `PreTrainedPolicy` all follow this
|
||||
shape.
|
||||
|
||||
### Module-level aliases
|
||||
|
||||
Several public names are aliases rather than distinct classes:
|
||||
|
||||
```python
|
||||
SO100FollowerConfig = SOFollowerRobotConfig
|
||||
SO101FollowerConfig = SOFollowerRobotConfig
|
||||
```
|
||||
|
||||
`[[autodoc]]` resolves the alias and renders the **canonical** class name, so a `## SO101FollowerConfig`
|
||||
heading will show `class lerobot.robots.so_follower.SOFollowerRobotConfig` in the body. Document the
|
||||
canonical class once, and mention the aliases in the page's prose rather than giving each alias its own
|
||||
autodoc block.
|
||||
|
||||
## What not to document
|
||||
|
||||
- **Private members.** Anything starting with `_` is not part of the public API.
|
||||
- **The type annotation restated as prose.** `port (`str`): A string.` adds nothing. Say what it is for.
|
||||
- **Vendored upstream code.** `src/lerobot/policies/molmoact2/molmoact2_hf_model/` is vendored from
|
||||
`transformers` and already carries upstream-style docstrings. Leave it alone — restyling it only creates
|
||||
conflicts on the next sync. It is excluded from the API reference and from the docstring checks.
|
||||
|
||||
## How this is enforced
|
||||
|
||||
| Check | What it catches |
|
||||
| ------------------------- | ---------------------------------------------------------------------------------------------------------- |
|
||||
| `make check-docstrings` | An `Args:` entry that doesn't match the signature; a documented default that has drifted from the real one |
|
||||
| `make doctest` | Examples that no longer run |
|
||||
| `make check-doctest-list` | Stale or unsorted entries in `utils/documentation_tests.txt` |
|
||||
| `ruff` (`D` rules) | Google-convention style violations |
|
||||
| `interrogate` | Docstring coverage falling below the current threshold |
|
||||
| doc-builder | A `[[autodoc]]` path that points at something that doesn't exist — this breaks the docs build |
|
||||
|
||||
Run them together before opening a PR:
|
||||
|
||||
```bash
|
||||
make check-docstrings && make doctest && pre-commit run --all-files
|
||||
```
|
||||
|
||||
Then render the page and actually look at it:
|
||||
|
||||
```bash
|
||||
doc-builder build lerobot docs/source/ --build_dir /tmp/doc-build
|
||||
```
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Every public member you touched has a docstring.
|
||||
- [ ] Every `Args:` entry matches the signature, including the `*optional*, defaults to` clause.
|
||||
- [ ] `Returns:` is type-first on one indented line.
|
||||
- [ ] No bare `Attributes:` — use `**Attributes**:` with `--` separators.
|
||||
- [ ] No Sphinx roles — cross-references use [`~module.Class.method`].
|
||||
- [ ] Examples are inside a fenced ` ```python ` block, and either run in CI or carry `# doctest: +SKIP`.
|
||||
- [ ] Config dataclass fields are in an `Args:` block on the concrete class, not `#` comments.
|
||||
- [ ] The rendered page has been eyeballed.
|
||||
@@ -306,6 +306,7 @@ class RTCEvaluator:
|
||||
# Configure RTC
|
||||
rtc_config = RTCConfig(
|
||||
enabled=rtc_enabled,
|
||||
mode=self.cfg.rtc.mode,
|
||||
execution_horizon=self.cfg.rtc.execution_horizon,
|
||||
max_guidance_weight=self.cfg.rtc.max_guidance_weight,
|
||||
prefix_attention_schedule=self.cfg.rtc.prefix_attention_schedule,
|
||||
|
||||
+17
-80
@@ -346,7 +346,6 @@ lerobot-record="lerobot.scripts.lerobot_record:main"
|
||||
lerobot-replay="lerobot.scripts.lerobot_replay:main"
|
||||
lerobot-setup-motors="lerobot.scripts.lerobot_setup_motors:main"
|
||||
lerobot-teleoperate="lerobot.scripts.lerobot_teleoperate:main"
|
||||
lerobot-convert-dcp="lerobot.scripts.lerobot_convert_dcp:main"
|
||||
lerobot-eval="lerobot.scripts.lerobot_eval:main"
|
||||
lerobot-train="lerobot.scripts.lerobot_train:main"
|
||||
lerobot-train-tokenizer="lerobot.scripts.lerobot_train_tokenizer:main"
|
||||
@@ -401,63 +400,19 @@ exclude = ["tests/artifacts/**/*.safetensors", "*_pb2.py", "*_pb2_grpc.py"]
|
||||
# N: pep8-naming
|
||||
# TODO: Uncomment rules when ready to use
|
||||
select = [
|
||||
"E", "W", "F", "I", "B", "C4", "T20", "N", "UP", "SIM", "D" #, "A", "S", "RUF"
|
||||
"E", "W", "F", "I", "B", "C4", "T20", "N", "UP", "SIM" #, "A", "S", "D", "RUF"
|
||||
]
|
||||
ignore = [
|
||||
"E501", # Line too long
|
||||
"T201", # Print statement found
|
||||
"T203", # Pprint statement found
|
||||
"B008", # Perform function call in argument defaults
|
||||
# D100/D104: module- and package-level docstrings. The API reference is generated from class and
|
||||
# function docstrings; a banner at the top of every file and every __init__.py would not appear on any
|
||||
# rendered page. Coverage of the things that do get rendered is enforced by interrogate instead.
|
||||
"D100",
|
||||
"D104",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"__init__.py" = ["F401", "F403", "E402", "D104"]
|
||||
"__init__.py" = ["F401", "F403", "E402"]
|
||||
# E402: conditional-import guards (TYPE_CHECKING / is_package_available) must precede the imports they protect
|
||||
"src/lerobot/scripts/convert_dataset_v21_to_v30.py" = ["E402"]
|
||||
|
||||
# D (pydocstyle) is enabled globally, but only holds for code that has been converted to the docstring
|
||||
# standard in docs/source/writing_docstrings.mdx. Every module below is still on the old style; each entry
|
||||
# is deleted as that module is converted, and this block can be removed once it is empty.
|
||||
#
|
||||
# Not part of the API reference and not planned for conversion: tests, examples, benchmarks, templates,
|
||||
# CI helper scripts and the packaging shim.
|
||||
"tests/**" = ["D"]
|
||||
"examples/**" = ["D"]
|
||||
"benchmarks/**" = ["D"]
|
||||
"scripts/**" = ["D"]
|
||||
"setup.py" = ["D"]
|
||||
"src/lerobot/templates/**" = ["D"]
|
||||
# Vendored from transformers; keeps its upstream docstring style so syncs stay clean.
|
||||
"src/lerobot/policies/molmoact2/molmoact2_hf_model/**" = ["D"]
|
||||
# Awaiting conversion, one PR per module.
|
||||
"src/lerobot/annotations/**" = ["D"]
|
||||
"src/lerobot/async_inference/**" = ["D"]
|
||||
"src/lerobot/cameras/**" = ["D"]
|
||||
"src/lerobot/common/**" = ["D"]
|
||||
"src/lerobot/configs/**" = ["D"]
|
||||
"src/lerobot/data_processing/**" = ["D"]
|
||||
"src/lerobot/distributed/**" = ["D"]
|
||||
"src/lerobot/envs/**" = ["D"]
|
||||
"src/lerobot/jobs/**" = ["D"]
|
||||
"src/lerobot/model/**" = ["D"]
|
||||
"src/lerobot/motors/**" = ["D"]
|
||||
"src/lerobot/optim/**" = ["D"]
|
||||
"src/lerobot/policies/**" = ["D"]
|
||||
"src/lerobot/processor/**" = ["D"]
|
||||
"src/lerobot/rewards/**" = ["D"]
|
||||
"src/lerobot/rl/**" = ["D"]
|
||||
"src/lerobot/rollout/**" = ["D"]
|
||||
"src/lerobot/scripts/**" = ["D"]
|
||||
"src/lerobot/teleoperators/**" = ["D"]
|
||||
"src/lerobot/transforms/**" = ["D"]
|
||||
"src/lerobot/transport/**" = ["D"]
|
||||
"src/lerobot/utils/**" = ["D"]
|
||||
"src/lerobot/lerobot_types.py" = ["D"]
|
||||
[tool.ruff.lint.isort]
|
||||
combine-as-imports = true
|
||||
known-first-party = ["lerobot"]
|
||||
@@ -501,34 +456,25 @@ default.extend-ignore-identifiers-re = [
|
||||
"seperated_timestep",
|
||||
]
|
||||
|
||||
# Docstring coverage gate. `fail-under` is a RATCHET, not a target: it is set just below the currently
|
||||
# measured coverage so it passes today, and is raised in the same PR that documents a module. Never set it
|
||||
# to a value that fails on main. The destination is 100; see docs/source/writing_docstrings.mdx.
|
||||
[tool.interrogate]
|
||||
ignore-init-module = true
|
||||
ignore-init-method = true
|
||||
ignore-nested-functions = false
|
||||
ignore-magic = false
|
||||
ignore-semiprivate = false
|
||||
ignore-private = false
|
||||
ignore-property-decorators = false
|
||||
ignore-module = false
|
||||
ignore-setters = false
|
||||
fail-under = 55
|
||||
output-format = "term-missing"
|
||||
color = true
|
||||
paths = ["src/lerobot"]
|
||||
exclude = ["src/lerobot/policies/molmoact2/molmoact2_hf_model"]
|
||||
# TODO: Uncomment when ready to use
|
||||
# [tool.interrogate]
|
||||
# ignore-init-module = true
|
||||
# ignore-init-method = true
|
||||
# ignore-nested-functions = false
|
||||
# ignore-magic = false
|
||||
# ignore-semiprivate = false
|
||||
# ignore-private = false
|
||||
# ignore-property-decorators = false
|
||||
# ignore-module = false
|
||||
# ignore-setters = false
|
||||
# fail-under = 80
|
||||
# output-format = "term-missing"
|
||||
# color = true
|
||||
# paths = ["src/lerobot"]
|
||||
|
||||
# TODO: Enable mypy gradually module by module across multiple PRs
|
||||
# Uncomment [tool.mypy] first, then uncomment individual module overrides as they get proper type annotations
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
markers = [
|
||||
"multigpu: distributed tests needing 2-4 GPUs (CI: docker_publish.yml lane)",
|
||||
"multigpu_heavy: 8-GPU sweeps and soak tests; never run in CI",
|
||||
]
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
ignore_missing_imports = true
|
||||
@@ -575,15 +521,6 @@ disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
check_untyped_defs = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "lerobot.distributed.*"
|
||||
ignore_errors = false
|
||||
|
||||
# extra strictness for the distributed engine
|
||||
disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
check_untyped_defs = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "lerobot.optim.*"
|
||||
ignore_errors = false
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""LeRobot -- PyTorch library for real-world robotics.
|
||||
"""
|
||||
LeRobot -- PyTorch library for real-world robotics.
|
||||
|
||||
Provides datasets, pretrained policies, and tools for training, evaluation,
|
||||
data collection, and robot control. Integrates with Hugging Face Hub for
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""To enable `lerobot.__version__`."""
|
||||
"""To enable `lerobot.__version__`"""
|
||||
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
|
||||
@@ -33,10 +33,10 @@ class Camera(abc.ABC):
|
||||
- Connection/disconnection
|
||||
- Frame capture (sync/async/latest)
|
||||
|
||||
**Attributes**:
|
||||
- **fps** (`int | None`) -- Configured frames per second.
|
||||
- **width** (`int | None`) -- Frame width in pixels.
|
||||
- **height** (`int | None`) -- Frame height in pixels.
|
||||
Attributes:
|
||||
fps (int | None): Configured frames per second
|
||||
width (int | None): Frame width in pixels
|
||||
height (int | None): Frame height in pixels
|
||||
"""
|
||||
|
||||
def __init__(self, config: CameraConfig):
|
||||
|
||||
@@ -40,20 +40,17 @@ class OpenCVCameraConfig(CameraConfig):
|
||||
OpenCVCameraConfig(0, 30, 1280, 720, fourcc="YUYV") # With YUYV format
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **index_or_path** (`int | Path`) -- Either an integer representing the camera device index, or a
|
||||
Path object pointing to a video file.
|
||||
- **fps** -- Requested frames per second for the color stream.
|
||||
- **width** -- Requested frame width in pixels for the color stream.
|
||||
- **height** -- Requested frame height in pixels for the color stream.
|
||||
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no
|
||||
rotation.
|
||||
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds)
|
||||
- **fourcc** (`str | None`) -- FOURCC code for video format (e.g., "MJPG", "YUYV", "I420"). Defaults
|
||||
to None (auto-detect).
|
||||
- **backend** (`Cv2Backends`) -- OpenCV backend identifier
|
||||
(https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html). Defaults to ANY.
|
||||
Attributes:
|
||||
index_or_path: Either an integer representing the camera device index,
|
||||
or a Path object pointing to a video file.
|
||||
fps: Requested frames per second for the color stream.
|
||||
width: Requested frame width in pixels for the color stream.
|
||||
height: Requested frame height in pixels for the color stream.
|
||||
color_mode: Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
rotation: Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no rotation.
|
||||
warmup_s: Time reading frames before returning from connect (in seconds)
|
||||
fourcc: FOURCC code for video format (e.g., "MJPG", "YUYV", "I420"). Defaults to None (auto-detect).
|
||||
backend: OpenCV backend identifier (https://docs.opencv.org/3.4/d4/d15/group__videoio__flags__base.html). Defaults to ANY.
|
||||
|
||||
Note:
|
||||
- Only 3-channel color output (RGB/BGR) is currently supported.
|
||||
|
||||
@@ -43,16 +43,16 @@ class Reachy2CameraConfig(CameraConfig):
|
||||
) # Left teleop camera, 640x480 @ 30FPS
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **name** (`str`) -- Name of the camera device. Can be "teleop" or "depth".
|
||||
- **image_type** (`str`) -- Type of image stream. For "teleop" camera, can be "left" or "right". For
|
||||
"depth" camera, can be "rgb" or "depth". (depth is not supported yet)
|
||||
- **fps** -- Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
|
||||
- **width** -- Requested frame width in pixels for the color stream.
|
||||
- **height** -- Requested frame height in pixels for the color stream.
|
||||
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
- **ip_address** (`str | None`) -- IP address of the robot. Defaults to "localhost".
|
||||
- **port** (`int`) -- Port number for the camera server. Defaults to 50065.
|
||||
Attributes:
|
||||
name: Name of the camera device. Can be "teleop" or "depth".
|
||||
image_type: Type of image stream. For "teleop" camera, can be "left" or "right".
|
||||
For "depth" camera, can be "rgb" or "depth". (depth is not supported yet)
|
||||
fps: Requested frames per second for the color stream. Not configurable for Reachy 2 cameras.
|
||||
width: Requested frame width in pixels for the color stream.
|
||||
height: Requested frame height in pixels for the color stream.
|
||||
color_mode: Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
ip_address: IP address of the robot. Defaults to "localhost".
|
||||
port: Port number for the camera server. Defaults to 50065.
|
||||
|
||||
Note:
|
||||
- Only 3-channel color output (RGB/BGR) is currently supported.
|
||||
|
||||
@@ -36,28 +36,27 @@ class RealSenseCameraConfig(CameraConfig):
|
||||
RealSenseCameraConfig("0123456789", 30, 640, 480, rotation=Cv2Rotation.ROTATE_90) # With 90° rotation
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **fps** -- Requested frames per second for the color stream.
|
||||
- **width** -- Requested frame width in pixels for the color stream.
|
||||
- **height** -- Requested frame height in pixels for the color stream.
|
||||
- **serial_number_or_name** (`str`) -- Unique serial number or human-readable name to identify the
|
||||
camera.
|
||||
- **color_mode** (`ColorMode`) -- Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
- **use_rgb** (`bool`) -- Whether to enable the color stream. Defaults to True.
|
||||
- **use_depth** (`bool`) -- Whether to enable depth stream. Defaults to False.
|
||||
- **rotation** (`Cv2Rotation`) -- Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no
|
||||
rotation.
|
||||
- **warmup_s** (`int`) -- Time reading frames before returning from connect (in seconds)
|
||||
- **exposure** (`int | None`) -- Manual exposure value for the color sensor. When set, auto-exposure
|
||||
is disabled and this fixed value is used. Valid ranges are camera-model specific and reported if the
|
||||
value is rejected. Defaults to None (leave unchanged).
|
||||
- **gain** (`int | None`) -- Manual gain value for the color sensor. When set, auto-exposure is
|
||||
disabled and this fixed gain is used, which also freezes exposure at its current value when no
|
||||
exposure is configured. Valid ranges are camera-model specific and reported if the value is
|
||||
rejected. Defaults to None (leave unchanged).
|
||||
- **white_balance** (`int | None`) -- Manual white balance value for the color sensor. When set, auto
|
||||
white balance is disabled and this fixed value is used. Valid ranges are camera-model specific and
|
||||
reported if the value is rejected. Defaults to None (leave unchanged).
|
||||
Attributes:
|
||||
fps: Requested frames per second for the color stream.
|
||||
width: Requested frame width in pixels for the color stream.
|
||||
height: Requested frame height in pixels for the color stream.
|
||||
serial_number_or_name: Unique serial number or human-readable name to identify the camera.
|
||||
color_mode: Color mode for image output (RGB or BGR). Defaults to RGB.
|
||||
use_rgb: Whether to enable the color stream. Defaults to True.
|
||||
use_depth: Whether to enable depth stream. Defaults to False.
|
||||
rotation: Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no rotation.
|
||||
warmup_s: Time reading frames before returning from connect (in seconds)
|
||||
exposure: Manual exposure value for the color sensor. When set, auto-exposure is
|
||||
disabled and this fixed value is used. Valid ranges are camera-model specific
|
||||
and reported if the value is rejected. Defaults to None (leave unchanged).
|
||||
gain: Manual gain value for the color sensor. When set, auto-exposure is disabled
|
||||
and this fixed gain is used, which also freezes exposure at its current value
|
||||
when no exposure is configured. Valid ranges are camera-model specific and
|
||||
reported if the value is rejected. Defaults to None (leave unchanged).
|
||||
white_balance: Manual white balance value for the color sensor. When set, auto
|
||||
white balance is disabled and this fixed value is used. Valid ranges are
|
||||
camera-model specific and reported if the value is rejected. Defaults to None
|
||||
(leave unchanged).
|
||||
|
||||
Note:
|
||||
- Either name or serial_number must be specified.
|
||||
|
||||
+173
-603
@@ -13,41 +13,16 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Training-output persistence: checkpoints, two-phase resume, and hub publishing.
|
||||
|
||||
Rank discipline: every function here that can
|
||||
contain a collective is documented as such and must run on ALL ranks; rank-0-only file writes
|
||||
sit under one grouped ``is_main_process()`` gate per contiguous region, placed below all
|
||||
collectives. The leaf save/load helpers carry no rank gates of their own — the exception is
|
||||
``PreTrainedPolicy._save_pretrained``, whose gate is internal because its collective gather and
|
||||
its writes live in the same method.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch.distributed as dist
|
||||
from huggingface_hub import HfApi, ModelCard, ModelCardData, snapshot_download
|
||||
from huggingface_hub import HfApi, snapshot_download
|
||||
from torch.optim import Optimizer
|
||||
from torch.optim.lr_scheduler import LRScheduler
|
||||
|
||||
from lerobot.__version__ import __version__
|
||||
from lerobot.configs.policies import PreTrainedConfig
|
||||
from lerobot.configs.rewards import RewardModelConfig
|
||||
from lerobot.configs.train import TrainPipelineConfig
|
||||
from lerobot.distributed.checkpoint import (
|
||||
is_sharded_module,
|
||||
load_sharded_model,
|
||||
load_sharded_optimizer,
|
||||
save_sharded_model,
|
||||
save_sharded_optimizer,
|
||||
)
|
||||
from lerobot.distributed.utils import is_main_process
|
||||
from lerobot.optim import (
|
||||
load_optimizer_state,
|
||||
load_optimizer_state_dict,
|
||||
load_scheduler_state,
|
||||
save_optimizer_state,
|
||||
save_scheduler_state,
|
||||
@@ -65,39 +40,14 @@ from lerobot.utils.hub import find_latest_hub_checkpoint
|
||||
from lerobot.utils.io_utils import load_json, write_json
|
||||
from lerobot.utils.random_utils import load_rng_state, save_rng_state
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from accelerate import Accelerator
|
||||
|
||||
from lerobot.datasets.dataset_metadata import LeRobotDatasetMetadata
|
||||
from lerobot.rewards.pretrained import PreTrainedRewardModel
|
||||
|
||||
|
||||
def get_step_identifier(step: int, total_steps: int) -> str:
|
||||
"""Format a step number as the zero-padded identifier used for checkpoint directory names.
|
||||
|
||||
Args:
|
||||
step (int): The training step to format.
|
||||
total_steps (int): The total number of training steps; sets the padding width
|
||||
(minimum 6 digits).
|
||||
|
||||
Returns:
|
||||
str: The zero-padded step identifier, e.g. `"005000"`.
|
||||
"""
|
||||
num_digits = max(6, len(str(total_steps)))
|
||||
return f"{step:0{num_digits}d}"
|
||||
|
||||
|
||||
def get_step_checkpoint_dir(output_dir: Path, total_steps: int, step: int) -> Path:
|
||||
"""Returns the checkpoint sub-directory corresponding to the step number.
|
||||
|
||||
Args:
|
||||
output_dir (Path): The training run's output directory.
|
||||
total_steps (int): The total number of training steps; sets the identifier padding.
|
||||
step (int): The training step of the checkpoint.
|
||||
|
||||
Returns:
|
||||
Path: The checkpoint step directory, `output_dir/checkpoints/<step-identifier>`.
|
||||
"""
|
||||
"""Returns the checkpoint sub-directory corresponding to the step number."""
|
||||
step_identifier = get_step_identifier(step, total_steps)
|
||||
return output_dir / CHECKPOINTS_DIR / step_identifier
|
||||
|
||||
@@ -113,15 +63,37 @@ def should_save_checkpoint(step: int, save_freq: int, total_steps: int) -> bool:
|
||||
return (save_freq > 0 and step % save_freq == 0) or step == total_steps
|
||||
|
||||
|
||||
def update_last_checkpoint(checkpoint_dir: Path) -> None:
|
||||
"""Point the `last` symlink in the checkpoints directory at the given checkpoint.
|
||||
def save_training_step(
|
||||
step: int, save_dir: Path, num_processes: int | None = None, batch_size: int | None = None
|
||||
) -> None:
|
||||
state: dict = {"step": step}
|
||||
# num_processes and batch_size are recorded so a resumed run can detect a changed world size or
|
||||
# batch size: the sampler's resume offset is computed from the (num_processes, batch_size) that
|
||||
# produced `step`, since both scale how many sampler positions a step consumes (see
|
||||
# compute_sampler_state).
|
||||
if num_processes is not None:
|
||||
state["num_processes"] = num_processes
|
||||
if batch_size is not None:
|
||||
state["batch_size"] = batch_size
|
||||
write_json(state, save_dir / TRAINING_STEP)
|
||||
|
||||
Any existing `last` symlink is replaced. The link target is relative to the checkpoints
|
||||
directory, so the tree stays valid when the run directory is moved.
|
||||
|
||||
Args:
|
||||
checkpoint_dir (Path): The checkpoint step directory the `last` link should target.
|
||||
"""
|
||||
def load_training_step(save_dir: Path) -> int:
|
||||
training_step = load_json(save_dir / TRAINING_STEP)
|
||||
return training_step["step"]
|
||||
|
||||
|
||||
def load_training_num_processes(checkpoint_dir: Path) -> int | None:
|
||||
"""World size recorded at checkpoint time, or None for checkpoints written before it was stored."""
|
||||
return load_json(checkpoint_dir / TRAINING_STATE_DIR / TRAINING_STEP).get("num_processes")
|
||||
|
||||
|
||||
def load_training_batch_size(checkpoint_dir: Path) -> int | None:
|
||||
"""Per-process batch size recorded at checkpoint time, or None for older checkpoints."""
|
||||
return load_json(checkpoint_dir / TRAINING_STATE_DIR / TRAINING_STEP).get("batch_size")
|
||||
|
||||
|
||||
def update_last_checkpoint(checkpoint_dir: Path) -> Path:
|
||||
last_checkpoint_dir = checkpoint_dir.parent / LAST_CHECKPOINT_LINK
|
||||
if last_checkpoint_dir.is_symlink():
|
||||
last_checkpoint_dir.unlink()
|
||||
@@ -129,68 +101,6 @@ def update_last_checkpoint(checkpoint_dir: Path) -> None:
|
||||
last_checkpoint_dir.symlink_to(relative_target)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# training_step.json
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_training_metadata(step: int, save_dir: Path, cfg: TrainPipelineConfig) -> None:
|
||||
"""Record the step counter plus everything a resume needs to reason about topology changes.
|
||||
|
||||
`step` counts loop iterations (= micro-batches), so
|
||||
the sampler resume offset is `step x batch_size x dp_world_size` with no grad-accum factor.
|
||||
`grad_accum_steps` and the parallelism snapshot are recorded so a resume can warn precisely
|
||||
when the optimizer-update cadence or the sharding topology changed.
|
||||
|
||||
Args:
|
||||
step (int): The training step (micro-batch counter) to record.
|
||||
save_dir (Path): The `training_state/` directory to write `training_step.json` into.
|
||||
cfg (TrainPipelineConfig): The training config whose batch size, gradient-accumulation,
|
||||
and parallelism settings are snapshotted alongside the step.
|
||||
"""
|
||||
state: dict[str, Any] = {
|
||||
"step": step,
|
||||
"dp_world_size": cfg.parallelism.dp_world_size,
|
||||
"batch_size": cfg.batch_size,
|
||||
"grad_accum_steps": cfg.accelerator.gradient_accumulation.steps,
|
||||
"parallelism": {
|
||||
"dp_replicate": cfg.parallelism.dp_replicate,
|
||||
"dp_shard": cfg.parallelism.dp_shard,
|
||||
"ring_degree": cfg.parallelism.context_parallel.ring_degree,
|
||||
"ulysses_degree": cfg.parallelism.context_parallel.ulysses_degree,
|
||||
},
|
||||
}
|
||||
write_json(state, save_dir / TRAINING_STEP)
|
||||
|
||||
|
||||
def load_training_metadata(training_state_dir: Path) -> dict[str, Any]:
|
||||
"""Read everything `save_training_metadata` recorded, in a single pass.
|
||||
|
||||
Every key is always present: fields a checkpoint predates come back as None, so a caller
|
||||
reading `metadata["batch_size"]` gets a KeyError on a typo rather than a silent None.
|
||||
|
||||
Args:
|
||||
training_state_dir (Path): The checkpoint's `training_state/` directory.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: `step` plus the `dp_world_size`, `batch_size`, `grad_accum_steps` and
|
||||
`parallelism` snapshot recorded alongside it (None where not recorded).
|
||||
"""
|
||||
state = load_json(training_state_dir / TRAINING_STEP)
|
||||
return {
|
||||
"step": int(state["step"]),
|
||||
"dp_world_size": state.get("dp_world_size", state.get("num_processes")),
|
||||
"batch_size": state.get("batch_size"),
|
||||
"grad_accum_steps": state.get("grad_accum_steps"),
|
||||
"parallelism": state.get("parallelism"),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Checkpoint save
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_checkpoint(
|
||||
checkpoint_dir: Path,
|
||||
step: int,
|
||||
@@ -200,301 +110,192 @@ def save_checkpoint(
|
||||
scheduler: LRScheduler | None = None,
|
||||
preprocessor: PolicyProcessorPipeline | None = None,
|
||||
postprocessor: PolicyProcessorPipeline | None = None,
|
||||
accelerator: "Accelerator | None" = None,
|
||||
num_processes: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
model_state_dict: dict | None = None,
|
||||
optim_state_dict: dict | None = None,
|
||||
) -> None:
|
||||
"""This function creates the following directory structure:
|
||||
|
||||
005000/ # training step at checkpoint
|
||||
├── pretrained_model/
|
||||
│ ├── config.json # policy config
|
||||
│ ├── model.safetensors # policy weights (checkpoint_format ∈ {safetensors, safetensors_dcp}, or any non-sharded run)
|
||||
│ ├── pytorch_model_fsdp_0/ # DCP model shards (checkpoint_format ∈ {dcp, safetensors_dcp})
|
||||
│ ├── model.safetensors # policy weights
|
||||
│ ├── train_config.json # train config
|
||||
│ ├── policy_preprocessor.json # preprocessor config (if preprocessor provided)
|
||||
│ ├── policy_preprocessor_step_*.safetensors # state of the stateful preprocessor steps
|
||||
│ ├── policy_postprocessor.json # postprocessor config (if postprocessor provided)
|
||||
│ └── policy_postprocessor_step_*.safetensors # state of the stateful postprocessor steps
|
||||
│ ├── processor.json # processor config (if preprocessor provided)
|
||||
│ └── step_*.safetensors # processor state files (if any)
|
||||
└── training_state/
|
||||
├── optimizer_param_groups.json # optimizer param groups (non-sharded runs)
|
||||
├── optimizer_state.safetensors # optimizer state (non-sharded runs)
|
||||
├── optimizer_0/ # DCP optimizer shards (sharded runs)
|
||||
├── optimizer_param_groups.json # optimizer param groups
|
||||
├── optimizer_state.safetensors # optimizer state
|
||||
├── rng_state.safetensors # rng states
|
||||
├── scheduler_state.json # scheduler state (if scheduler provided)
|
||||
└── training_step.json # training step + dp_world_size/batch_size/grad_accum + topology
|
||||
|
||||
Collective: MUST be called on every rank. Rank-0-only writes are gated internally, so the
|
||||
call site needs no rank branches.
|
||||
├── scheduler_state.json # scheduler state
|
||||
└── training_step.json # training step
|
||||
|
||||
Args:
|
||||
checkpoint_dir (Path): The checkpoint step directory to write (e.g. `.../checkpoints/005000`).
|
||||
step (int): The training step at that checkpoint.
|
||||
cfg (TrainPipelineConfig): The training config used for this run.
|
||||
step (int): The training step at that checkpoint.
|
||||
policy (PreTrainedPolicy): The policy to save.
|
||||
optimizer (Optimizer): The optimizer to save the state from.
|
||||
optimizer (Optimizer | None, optional): The optimizer to save the state from. Defaults to None.
|
||||
scheduler (LRScheduler | None, optional): The scheduler to save the state from. Defaults to None.
|
||||
preprocessor (PolicyProcessorPipeline | None, optional): The preprocessor/pipeline to save.
|
||||
preprocessor: The preprocessor/pipeline to save. Defaults to None.
|
||||
postprocessor: The postprocessor/pipeline to save. Defaults to None.
|
||||
num_processes (int | None, optional): Distributed world size to record for sample-exact
|
||||
resume. Defaults to None (not recorded).
|
||||
batch_size (int | None, optional): Per-process batch size to record for sample-exact
|
||||
resume. Defaults to None (not recorded).
|
||||
model_state_dict: Pre-gathered full (unsharded) model state dict. Required under FSDP,
|
||||
where `policy.state_dict()` would return sharded tensors; the caller gathers it via a
|
||||
cross-rank collective and passes it here so rank 0 can write it directly. It holds
|
||||
FSDP's fp32 master weights and is saved as-is (the loader casts to the policy dtype on
|
||||
read). When None (DDP / single-GPU), the model is saved the normal way. Defaults to None.
|
||||
optim_state_dict: Pre-gathered full (unsharded) optimizer state dict. Required under FSDP
|
||||
(gathered alongside `model_state_dict` via `gather_fsdp_state_dicts`); saved in the same
|
||||
safetensors format as the single-GPU path. When None, `optimizer.state_dict()` is used.
|
||||
Defaults to None.
|
||||
postprocessor (PolicyProcessorPipeline | None, optional): The postprocessor/pipeline to save.
|
||||
Defaults to None.
|
||||
accelerator (Accelerator | None, optional): The accelerator the policy was prepared with;
|
||||
used to unwrap the model and required on sharded runs, where it owns the DCP save
|
||||
channels. Defaults to None (plain single-process saves).
|
||||
"""
|
||||
pretrained_dir = checkpoint_dir / PRETRAINED_MODEL_DIR
|
||||
fmt = cfg.checkpoint_format
|
||||
policy_to_save = accelerator.unwrap_model(policy) if accelerator is not None else policy
|
||||
sharded = is_sharded_module(policy_to_save)
|
||||
|
||||
# -- model artifact(s): the two collective-capable calls ----------------------------------
|
||||
policy.save_pretrained(pretrained_dir, state_dict=model_state_dict)
|
||||
cfg.save_pretrained(pretrained_dir)
|
||||
if cfg.peft is not None:
|
||||
# PeftModel.save_pretrained is an external API with no internal rank gate, and the
|
||||
# adapters are replicated (PEFT x sharded is rejected at validation): main rank writes.
|
||||
if is_main_process():
|
||||
policy_to_save.save_pretrained(pretrained_dir)
|
||||
elif fmt.wants_safetensors or not sharded:
|
||||
# Collective when sharded (full gather); writes happen on the main process only in all
|
||||
# multi-rank layouts (the gate lives inside _save_pretrained, next to its collective gather).
|
||||
policy_to_save.save_pretrained(pretrained_dir)
|
||||
if fmt.wants_dcp and sharded:
|
||||
save_sharded_model(accelerator, policy_to_save, pretrained_dir)
|
||||
|
||||
# -- sidecar configs: ONE gate for the whole contiguous rank-0-only region ----------------
|
||||
if is_main_process():
|
||||
if fmt.wants_dcp and not fmt.wants_safetensors:
|
||||
# save_pretrained did not run: keep the DCP-only checkpoint self-describing.
|
||||
policy_to_save.config.save_pretrained(pretrained_dir)
|
||||
cfg.save_pretrained(pretrained_dir)
|
||||
if cfg.peft is not None:
|
||||
# PEFT's save_pretrained writes only adapter weights + config; the policy config
|
||||
# needed to reload the base model is written explicitly.
|
||||
policy_to_save.config.save_pretrained(pretrained_dir)
|
||||
if preprocessor is not None:
|
||||
preprocessor.save_pretrained(pretrained_dir)
|
||||
if postprocessor is not None:
|
||||
postprocessor.save_pretrained(pretrained_dir)
|
||||
|
||||
# When using PEFT, policy.save_pretrained will only write the adapter weights + config, not the
|
||||
# policy config which we need for loading the model. In this case we'll write it ourselves.
|
||||
policy.config.save_pretrained(pretrained_dir)
|
||||
if preprocessor is not None:
|
||||
preprocessor.save_pretrained(pretrained_dir)
|
||||
if postprocessor is not None:
|
||||
postprocessor.save_pretrained(pretrained_dir)
|
||||
save_training_state(
|
||||
checkpoint_dir, step, cfg, optimizer, scheduler, accelerator, sharded=sharded, model=policy_to_save
|
||||
checkpoint_dir,
|
||||
step,
|
||||
optimizer,
|
||||
scheduler,
|
||||
num_processes=num_processes,
|
||||
batch_size=batch_size,
|
||||
optim_state_dict=optim_state_dict,
|
||||
)
|
||||
if accelerator is not None:
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
|
||||
def save_training_state(
|
||||
checkpoint_dir: Path,
|
||||
step: int,
|
||||
cfg: TrainPipelineConfig,
|
||||
optimizer: Optimizer | dict[str, Optimizer] | None = None,
|
||||
train_step: int,
|
||||
optimizer: Optimizer | None = None,
|
||||
scheduler: LRScheduler | None = None,
|
||||
accelerator: "Accelerator | None" = None,
|
||||
*,
|
||||
sharded: bool = False,
|
||||
model: PreTrainedPolicy | None = None,
|
||||
num_processes: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
optim_state_dict: dict | None = None,
|
||||
) -> None:
|
||||
"""Write training_state/. Collective under sharding: call on every rank.
|
||||
"""
|
||||
Saves the training step, optimizer state, scheduler state, and rng state.
|
||||
|
||||
Args:
|
||||
checkpoint_dir (Path): The checkpoint step directory; `training_state/` is created inside it.
|
||||
step (int): The training step at that checkpoint.
|
||||
cfg (TrainPipelineConfig): The training config used for this run (its topology and
|
||||
accumulation settings are recorded in `training_step.json`).
|
||||
optimizer (Optimizer | dict[str, Optimizer] | None, optional): The optimizer(s) to save
|
||||
the state from. Defaults to None.
|
||||
scheduler (LRScheduler | None, optional): The scheduler to save the state from.
|
||||
save_dir (Path): The directory to save artifacts to.
|
||||
train_step (int): Current training step.
|
||||
optimizer (Optimizer | None, optional): The optimizer from which to save the state_dict.
|
||||
Defaults to None.
|
||||
accelerator (Accelerator | None, optional): Required when `sharded` is True — it owns
|
||||
the DCP optimizer save channel. Defaults to None.
|
||||
sharded (bool): The model's sharding state, computed once in `save_checkpoint` and
|
||||
threaded here so the two sites cannot disagree. Defaults to False.
|
||||
model (PreTrainedPolicy | None, optional): Required only for the sharded optimizer
|
||||
channel: torch's optimizer DCP APIs are model-coupled (the state dict is keyed by
|
||||
model FQNs), so accelerate's `save_fsdp_optimizer` needs the sharded module
|
||||
alongside the optimizer. Defaults to None.
|
||||
scheduler (LRScheduler | None, optional): The scheduler from which to save the state_dict.
|
||||
Defaults to None.
|
||||
num_processes (int | None, optional): Distributed world size to record. Defaults to None.
|
||||
batch_size (int | None, optional): Per-process batch size to record. Defaults to None.
|
||||
optim_state_dict: Pre-gathered full optimizer state dict (for FSDP). Saved instead of
|
||||
`optimizer.state_dict()` when provided. Defaults to None.
|
||||
"""
|
||||
save_dir = checkpoint_dir / TRAINING_STATE_DIR
|
||||
# All ranks: the directory must exist before the DCP optimizer collective writes into it
|
||||
# (exist_ok makes the concurrent mkdir race-free on shared filesystems).
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if optimizer is not None and sharded:
|
||||
if accelerator is None or model is None:
|
||||
raise ValueError("Saving a sharded optimizer state requires the accelerator and model.")
|
||||
# Collective — all ranks write their DCP shards into optimizer_0/.
|
||||
save_sharded_optimizer(accelerator, optimizer, model, save_dir)
|
||||
|
||||
if is_main_process(): # ONE grouped gate for the whole rank-0-only region
|
||||
save_training_metadata(step, save_dir, cfg)
|
||||
save_rng_state(save_dir)
|
||||
if scheduler is not None:
|
||||
save_scheduler_state(scheduler, save_dir)
|
||||
if optimizer is not None and not sharded:
|
||||
save_optimizer_state(optimizer, save_dir)
|
||||
save_training_step(train_step, save_dir, num_processes=num_processes, batch_size=batch_size)
|
||||
save_rng_state(save_dir)
|
||||
if optimizer is not None:
|
||||
save_optimizer_state(optimizer, save_dir, optim_state_dict=optim_state_dict)
|
||||
if scheduler is not None:
|
||||
save_scheduler_state(scheduler, save_dir)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Two-phase resume
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resume_before_prepare(cfg: TrainPipelineConfig) -> int:
|
||||
"""Phase 1 — before `accelerator.prepare()`: restore RNG and return the step counter.
|
||||
|
||||
Pure loaders only. The sampler resume offset is *derived* from the returned step inside the
|
||||
dataloader factory, and everything bound to sharded objects (model DCP shards, optimizer,
|
||||
scheduler) loads in `resume_after_prepare`.
|
||||
def load_training_state(
|
||||
checkpoint_dir: Path, optimizer: Optimizer, scheduler: LRScheduler | None, load_optimizer: bool = True
|
||||
) -> tuple[int, Optimizer, LRScheduler | None]:
|
||||
"""
|
||||
Loads the training step, optimizer state, scheduler state, and rng state.
|
||||
This is used to resume a training run.
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The resumed training config; `cfg.checkpoint_path` locates
|
||||
the checkpoint to restore from.
|
||||
checkpoint_dir (Path): The checkpoint directory. Should contain a 'training_state' dir.
|
||||
optimizer (Optimizer): The optimizer to load the state_dict to.
|
||||
scheduler (LRScheduler | None): The scheduler to load the state_dict to (can be None).
|
||||
load_optimizer (bool, optional): Whether to load the optimizer state from disk. Defaults to
|
||||
True. Set to False under FSDP, where the sharded optimizer state must be loaded after
|
||||
`accelerator.prepare()` via `load_fsdp_optimizer_state` (the optimizer is returned
|
||||
untouched here).
|
||||
|
||||
Raises:
|
||||
NotADirectoryError: If 'checkpoint_dir' doesn't contain a 'training_state' dir
|
||||
|
||||
Returns:
|
||||
int: The training step recorded in the checkpoint (micro-batch counter).
|
||||
|
||||
Raises:
|
||||
NotADirectoryError: If the checkpoint has no `training_state/` directory.
|
||||
ValueError: If the resumed topology crosses the sharded/non-sharded boundary relative
|
||||
to the one recorded in the checkpoint.
|
||||
tuple[int, Optimizer, LRScheduler | None]: training step, optimizer and scheduler with their
|
||||
state_dict loaded.
|
||||
"""
|
||||
training_state_dir = cfg.checkpoint_path / TRAINING_STATE_DIR
|
||||
training_state_dir = checkpoint_dir / TRAINING_STATE_DIR
|
||||
if not training_state_dir.is_dir():
|
||||
raise NotADirectoryError(training_state_dir)
|
||||
metadata = load_training_metadata(training_state_dir)
|
||||
_guard_resume_changes(cfg, metadata)
|
||||
|
||||
load_rng_state(training_state_dir)
|
||||
return metadata["step"]
|
||||
|
||||
|
||||
def _guard_resume_changes(cfg: TrainPipelineConfig, metadata: dict[str, Any]) -> None:
|
||||
"""Check the resumed run settings against the ones recorded in the checkpoint.
|
||||
|
||||
Two tiers, both driven by the checkpoint's recorded parallelism snapshot:
|
||||
|
||||
- **Hard error** when the resume crosses the sharded/non-sharded boundary in either
|
||||
direction: the checkpoint's training-state artifacts only support resuming on the same
|
||||
kind of topology (resharding works across sizes, not across kinds). Checkpoints without
|
||||
a recorded snapshot skip this check.
|
||||
- **One warning** naming every other recorded setting that differs — those changes are
|
||||
legal (DCP reshards weights and optimizer state across topologies and the sampler offset
|
||||
adapts), but a changed ``grad_accum_steps`` shifts the optimizer-update cadence, so the
|
||||
resume says precisely what differs. The sampler-exactness warnings
|
||||
(``dp_world_size``/``batch_size``) live with the sampler math in the dataloader factory.
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The resumed training config, compared against the settings
|
||||
recorded in the checkpoint.
|
||||
metadata (dict[str, Any]): The checkpoint's recorded training metadata, as returned by
|
||||
`load_training_metadata`.
|
||||
|
||||
Raises:
|
||||
ValueError: If the checkpoint records a sharded topology and the resumed run is
|
||||
non-sharded, or vice versa.
|
||||
"""
|
||||
snapshot = metadata["parallelism"]
|
||||
|
||||
if snapshot is not None:
|
||||
recorded_sharded = (
|
||||
snapshot.get("dp_shard", 1) != 1
|
||||
or snapshot.get("ring_degree", 1) * snapshot.get("ulysses_degree", 1) > 1
|
||||
)
|
||||
if recorded_sharded != cfg.parallelism.is_sharded:
|
||||
raise ValueError(
|
||||
f"Cannot resume: the checkpoint was written with a "
|
||||
f"{'sharded' if recorded_sharded else 'non-sharded'} topology "
|
||||
f"(dp_replicate={snapshot.get('dp_replicate')}, dp_shard={snapshot.get('dp_shard')}) "
|
||||
f"but this run is {'sharded' if cfg.parallelism.is_sharded else 'non-sharded'} "
|
||||
f"(dp_replicate={cfg.parallelism.dp_replicate}, dp_shard={cfg.parallelism.dp_shard})."
|
||||
)
|
||||
|
||||
recorded = {
|
||||
"grad_accum_steps": (
|
||||
metadata["grad_accum_steps"],
|
||||
cfg.accelerator.gradient_accumulation.steps,
|
||||
),
|
||||
}
|
||||
if snapshot is not None:
|
||||
recorded.update(
|
||||
{
|
||||
"dp_replicate": (snapshot.get("dp_replicate"), cfg.parallelism.dp_replicate),
|
||||
"dp_shard": (snapshot.get("dp_shard"), cfg.parallelism.dp_shard),
|
||||
"ring_degree": (
|
||||
snapshot.get("ring_degree"),
|
||||
cfg.parallelism.context_parallel.ring_degree,
|
||||
),
|
||||
"ulysses_degree": (
|
||||
snapshot.get("ulysses_degree"),
|
||||
cfg.parallelism.context_parallel.ulysses_degree,
|
||||
),
|
||||
}
|
||||
)
|
||||
changed = [f"{key}: {was} -> {now}" for key, (was, now) in recorded.items() if was not in (None, now)]
|
||||
if changed and is_main_process():
|
||||
logging.warning(
|
||||
"Resuming with settings that differ from the checkpoint: " + "; ".join(changed) + ". "
|
||||
"Topology changes reshard safely via DCP; a changed grad_accum_steps shifts the "
|
||||
"optimizer-update cadence (the step counter keeps counting micro-batches)."
|
||||
)
|
||||
|
||||
|
||||
def resume_after_prepare(
|
||||
cfg: TrainPipelineConfig,
|
||||
accelerator: "Accelerator",
|
||||
policy: PreTrainedPolicy,
|
||||
optimizer: Optimizer | dict[str, Optimizer],
|
||||
scheduler: LRScheduler | None,
|
||||
) -> None:
|
||||
"""Phase 2 — after `accelerator.prepare()`: model (DCP) -> optimizer -> scheduler.
|
||||
|
||||
Collective under sharding: call on every rank. The model-weight source follows the
|
||||
checkpoint's own recorded `checkpoint_format` (on resume, `cfg` was parsed from the
|
||||
checkpoint's train_config.json): DCP-bearing formats load shards here into the prepared
|
||||
model (whose construction skipped the safetensors load); the safetensors format was already
|
||||
loaded by `from_pretrained` before sharding — no model step here.
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The resumed training config; `cfg.checkpoint_path` locates
|
||||
the checkpoint and `cfg.checkpoint_format` selects the model-weight source.
|
||||
accelerator (Accelerator): The accelerator the policy was prepared with; it unwraps the
|
||||
model and owns the DCP load channels.
|
||||
policy (PreTrainedPolicy): The prepared (possibly sharded) policy to load weights into.
|
||||
optimizer (Optimizer | dict[str, Optimizer]): The prepared optimizer(s) to restore.
|
||||
scheduler (LRScheduler | None): The scheduler to restore, or None if the run has none.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the checkpoint format declares DCP model shards but the shard
|
||||
directory is missing (e.g. it was pruned before upload).
|
||||
"""
|
||||
checkpoint_dir = cfg.checkpoint_path
|
||||
pretrained_dir = checkpoint_dir / PRETRAINED_MODEL_DIR
|
||||
training_state_dir = checkpoint_dir / TRAINING_STATE_DIR
|
||||
unwrapped = accelerator.unwrap_model(policy)
|
||||
sharded = is_sharded_module(unwrapped)
|
||||
|
||||
if cfg.checkpoint_format.wants_dcp:
|
||||
from accelerate.utils.constants import FSDP_MODEL_NAME
|
||||
|
||||
dcp_dir = pretrained_dir / f"{FSDP_MODEL_NAME}_0"
|
||||
if not dcp_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"checkpoint_format={cfg.checkpoint_format.value} declares DCP model shards, "
|
||||
f"but {dcp_dir} is missing. If the shards were pruned, convert what remains "
|
||||
"with `lerobot-convert-dcp` or resume from a safetensors checkpoint."
|
||||
)
|
||||
load_sharded_model(accelerator, unwrapped, pretrained_dir)
|
||||
|
||||
if sharded:
|
||||
# Requires the prepared optimizer: FSDP2's prepare rebinds param groups to DTensors but
|
||||
# never migrates optimizer.state — DCP reshards it here (works across topology changes).
|
||||
load_sharded_optimizer(accelerator, optimizer, unwrapped, training_state_dir)
|
||||
else:
|
||||
load_optimizer_state(optimizer, training_state_dir)
|
||||
|
||||
step = load_training_step(training_state_dir)
|
||||
if load_optimizer:
|
||||
optimizer = load_optimizer_state(optimizer, training_state_dir)
|
||||
if scheduler is not None:
|
||||
load_scheduler_state(scheduler, training_state_dir)
|
||||
scheduler = load_scheduler_state(scheduler, training_state_dir)
|
||||
|
||||
return step, optimizer, scheduler
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Hub: checkpoint push (resume artifact) and publishing (distribution artifact)
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
def gather_fsdp_state_dicts(model, optimizer) -> tuple[dict, dict]:
|
||||
"""Gather the full (unsharded) model and optimizer state dicts under FSDP.
|
||||
|
||||
`model.state_dict()` and `FSDP.optim_state_dict(...)` are cross-rank collectives, so this must be
|
||||
called on *every* rank with the prepared (FSDP-wrapped) `model` and `optimizer`. With
|
||||
`rank0_only=True` and `offload_to_cpu=True`, every rank runs the all-gather but only rank 0
|
||||
materializes the full dicts (the others get empty dicts) and they are kept on CPU to bound GPU
|
||||
memory. The returned optimizer state dict is keyed by parameter FQNs and is world-size
|
||||
independent; `load_fsdp_optimizer_state` reshards it on resume.
|
||||
|
||||
Returns:
|
||||
(model_state_dict, optim_state_dict): full dicts on rank 0, empty dicts on other ranks.
|
||||
"""
|
||||
from torch.distributed.fsdp import (
|
||||
FullOptimStateDictConfig,
|
||||
FullStateDictConfig,
|
||||
FullyShardedDataParallel as FSDP, # noqa F401
|
||||
StateDictType,
|
||||
)
|
||||
|
||||
state_cfg = FullStateDictConfig(offload_to_cpu=True, rank0_only=True)
|
||||
optim_cfg = FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=True)
|
||||
with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, state_cfg, optim_cfg):
|
||||
model_state_dict = model.state_dict()
|
||||
optim_state_dict = FSDP.optim_state_dict(model, optimizer)
|
||||
return model_state_dict, optim_state_dict
|
||||
|
||||
|
||||
def load_fsdp_optimizer_state(model, optimizer, checkpoint_dir: Path) -> None:
|
||||
"""Load the FSDP optimizer state (saved as safetensors) and reshard it into the optimizer.
|
||||
|
||||
This is a cross-rank collective and must be called on every rank *after* `accelerator.prepare()`
|
||||
with the prepared (FSDP-wrapped) `model` and `optimizer`. The saved state is the full,
|
||||
world-size-independent optimizer state (keyed by parameter FQNs); `FSDP.optim_state_dict_to_load`
|
||||
reshards it to the current FSDP topology, so resume on a different number of GPUs works.
|
||||
"""
|
||||
from torch.distributed.fsdp import (
|
||||
FullOptimStateDictConfig,
|
||||
FullStateDictConfig,
|
||||
FullyShardedDataParallel as FSDP, # noqa F401
|
||||
StateDictType,
|
||||
)
|
||||
|
||||
# Every rank reads the same full state from the (shared) checkpoint dir, so rank0_only=False.
|
||||
full_osd = load_optimizer_state_dict(checkpoint_dir / TRAINING_STATE_DIR)
|
||||
state_cfg = FullStateDictConfig(rank0_only=False)
|
||||
optim_cfg = FullOptimStateDictConfig(rank0_only=False)
|
||||
with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, state_cfg, optim_cfg):
|
||||
sharded_osd = FSDP.optim_state_dict_to_load(model=model, optim=optimizer, optim_state_dict=full_osd)
|
||||
optimizer.load_state_dict(sharded_osd)
|
||||
|
||||
|
||||
def push_checkpoint_to_hub(
|
||||
@@ -510,16 +311,6 @@ def push_checkpoint_to_hub(
|
||||
The model repo is created idempotently, and the commit is tagged with the
|
||||
checkpoint step so a checkpoint can be recovered with
|
||||
--policy.pretrained_revision=<step> instead of a commit sha.
|
||||
|
||||
The directory is uploaded verbatim — including DCP shards under the DCP formats: this tree
|
||||
exists for *resume*, not distribution, and `resolve_resume_checkpoint` downloads it back
|
||||
symmetrically.
|
||||
|
||||
Args:
|
||||
checkpoint_dir (Path): The local checkpoint step directory to upload.
|
||||
repo_id (str): The Hub model repo to push to (created idempotently if missing).
|
||||
private (bool | None): Whether a newly created repo should be private. Defaults to
|
||||
None (public unless the organization's default is private).
|
||||
"""
|
||||
api = HfApi()
|
||||
api.create_repo(repo_id=repo_id, repo_type="model", private=private, exist_ok=True)
|
||||
@@ -547,16 +338,6 @@ def resolve_resume_checkpoint(repo_id: str, output_dir: Path) -> Path:
|
||||
into `output_dir/checkpoints/<step>/`, recreate the local `last` symlink, and return that local
|
||||
checkpoint dir. Used to resume training from the Hub on a machine (or HF Jobs pod) that does not
|
||||
have the original local run dir.
|
||||
|
||||
Args:
|
||||
repo_id (str): The Hub model repo holding `checkpoints/<step>/` subtrees.
|
||||
output_dir (Path): The local run directory to download the checkpoint into.
|
||||
|
||||
Returns:
|
||||
Path: The local checkpoint step directory, `output_dir/checkpoints/<step>`.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the repo contains no checkpoints under `checkpoints/`.
|
||||
"""
|
||||
latest = find_latest_hub_checkpoint(repo_id)
|
||||
if latest is None:
|
||||
@@ -573,214 +354,3 @@ def resolve_resume_checkpoint(repo_id: str, output_dir: Path) -> Path:
|
||||
checkpoint_dir = output_dir / latest
|
||||
update_last_checkpoint(checkpoint_dir)
|
||||
return checkpoint_dir
|
||||
|
||||
|
||||
def publish_trained_model(
|
||||
cfg: TrainPipelineConfig,
|
||||
model: "PreTrainedPolicy | PreTrainedRewardModel",
|
||||
preprocessor: PolicyProcessorPipeline | None,
|
||||
postprocessor: PolicyProcessorPipeline | None,
|
||||
dataset_meta: "LeRobotDatasetMetadata | None",
|
||||
*,
|
||||
peft_model: Any | None = None,
|
||||
) -> None:
|
||||
"""Publish the complete training bundle as a distributable model repo.
|
||||
|
||||
Collective-safe: call on ALL ranks — the model commit gathers sharded weights through
|
||||
`save_pretrained`; uploads happen on the main process only (gated inside
|
||||
`HubMixin.push_to_hub` and here). Commits, in order: (1) the model (skipped for PEFT —
|
||||
adapters replace full weights), (2) the preprocessor, (3) the postprocessor, (4) the bundle
|
||||
sidecar: README.md model card + train_config.json (+ adapter weights and the wrapped
|
||||
policy's config in the PEFT case). Every commit uploads a freshly assembled directory, so
|
||||
a published repo carries only the distributable artifacts.
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The training config; saved as `train_config.json` and used
|
||||
to render the model card.
|
||||
model (PreTrainedPolicy | PreTrainedRewardModel): The trained model to publish; its
|
||||
config supplies the target repo id, visibility, license, and tags.
|
||||
preprocessor (PolicyProcessorPipeline | None): The preprocessor pipeline to publish
|
||||
alongside the model, if any.
|
||||
postprocessor (PolicyProcessorPipeline | None): The postprocessor pipeline to publish
|
||||
alongside the model, if any.
|
||||
dataset_meta (LeRobotDatasetMetadata | None): Dataset metadata for the model card, if
|
||||
available.
|
||||
peft_model (Any | None): The PEFT wrapper when training adapters; its adapter weights
|
||||
replace the full model weights in the published repo. Defaults to None.
|
||||
|
||||
Raises:
|
||||
ValueError: If the model config carries no repo id (`--policy.repo_id`).
|
||||
"""
|
||||
model_cfg = model.config
|
||||
repo_id = model_cfg.repo_id
|
||||
if not repo_id:
|
||||
raise ValueError("Publishing requires a repo id (--policy.repo_id).")
|
||||
ignore = ["*.tmp", "*.log"]
|
||||
|
||||
if peft_model is None:
|
||||
# Calls are made on the exact objects that own each method (never through PEFT's
|
||||
# attribute forwarding), so the peft branch below never touches this path.
|
||||
model.push_to_hub(repo_id, private=model_cfg.private, ignore_patterns=ignore)
|
||||
if preprocessor is not None:
|
||||
preprocessor.push_to_hub(repo_id, private=model_cfg.private)
|
||||
if postprocessor is not None:
|
||||
postprocessor.push_to_hub(repo_id, private=model_cfg.private)
|
||||
|
||||
if is_main_process():
|
||||
api = HfApi()
|
||||
repo_id = api.create_repo(repo_id=repo_id, private=model_cfg.private, exist_ok=True).repo_id
|
||||
with TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
|
||||
saved_path = Path(tmp) / repo_id
|
||||
saved_path.mkdir(parents=True, exist_ok=True)
|
||||
if peft_model is not None:
|
||||
peft_model.save_pretrained(saved_path) # adapter weights + adapter config
|
||||
model.config.save_pretrained(saved_path) # PEFT cannot write the policy config
|
||||
card = generate_model_card(model_cfg, cfg=cfg, dataset_meta=dataset_meta)
|
||||
card.save(str(saved_path / "README.md"))
|
||||
cfg.save_pretrained(saved_path) # train_config.json
|
||||
commit_info = api.upload_folder(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
folder_path=saved_path,
|
||||
commit_message="Upload model card and train config",
|
||||
allow_patterns=["*.safetensors", "*.json", "*.yaml", "*.md"],
|
||||
ignore_patterns=ignore,
|
||||
)
|
||||
# Contract: lerobot.jobs.hf.submit_to_hf watches for this exact "Model pushed to <url>"
|
||||
# line to end a remote run early. Keep the wording and URL format in sync.
|
||||
logging.info(f"Model pushed to {commit_info.repo_url.url}")
|
||||
|
||||
if dist.is_initialized():
|
||||
dist.barrier()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Model card
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
_BASE_MODEL_MAPPING = {
|
||||
"smolvla": "lerobot/smolvla_base",
|
||||
"pi0": "lerobot/pi0_base",
|
||||
"pi05": "lerobot/pi05_base",
|
||||
"pi0_fast": "lerobot/pi0fast-base",
|
||||
"xvla": "lerobot/xvla-base",
|
||||
}
|
||||
|
||||
|
||||
def build_card_context(
|
||||
cfg: TrainPipelineConfig | None,
|
||||
dataset_meta: "LeRobotDatasetMetadata | None",
|
||||
input_features: dict | None,
|
||||
output_features: dict | None,
|
||||
) -> dict:
|
||||
"""Collect optional data for the model-card template.
|
||||
|
||||
Returns plain values only (no Markdown) — the template in
|
||||
``lerobot/templates/lerobot_modelcard_template.md`` decides how and whether to show
|
||||
each one. Everything is best-effort: anything unavailable is left empty/None and the
|
||||
template simply skips that section, so this never breaks a Hub push.
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig | None): The training config supplying the training section,
|
||||
if available.
|
||||
dataset_meta (LeRobotDatasetMetadata | None): Dataset metadata supplying the dataset,
|
||||
robot-type, and camera sections, if available.
|
||||
input_features (dict | None): The policy's input feature declarations, if any.
|
||||
output_features (dict | None): The policy's output feature declarations, if any.
|
||||
|
||||
Returns:
|
||||
dict: Template context with `training`, `input_features`, `output_features`,
|
||||
`dataset`, `robot_type`, and `cameras` entries; unavailable pieces stay
|
||||
empty/None.
|
||||
"""
|
||||
context = {
|
||||
"training": None,
|
||||
"input_features": input_features or {},
|
||||
"output_features": output_features or {},
|
||||
"dataset": None,
|
||||
"robot_type": None,
|
||||
"cameras": [],
|
||||
}
|
||||
|
||||
if cfg is not None:
|
||||
optimizer = getattr(cfg, "optimizer", None)
|
||||
context["training"] = {
|
||||
"steps": cfg.steps,
|
||||
"batch_size": cfg.batch_size,
|
||||
"seed": cfg.seed,
|
||||
"optimizer": getattr(optimizer, "type", None) if optimizer else None,
|
||||
"lr": getattr(optimizer, "lr", None) if optimizer else None,
|
||||
"lerobot_version": __version__,
|
||||
}
|
||||
|
||||
if dataset_meta is not None:
|
||||
context["dataset"] = {
|
||||
"repo_id": dataset_meta.repo_id,
|
||||
"episodes": dataset_meta.total_episodes,
|
||||
"frames": dataset_meta.total_frames,
|
||||
"fps": dataset_meta.fps,
|
||||
"tasks": [str(task) for task in dataset_meta.tasks.index],
|
||||
}
|
||||
context["robot_type"] = dataset_meta.robot_type
|
||||
context["cameras"] = [key.split(".")[-1] for key in dataset_meta.camera_keys]
|
||||
|
||||
return context
|
||||
|
||||
|
||||
def generate_model_card(
|
||||
model_cfg: PreTrainedConfig | RewardModelConfig,
|
||||
cfg: TrainPipelineConfig | None = None,
|
||||
dataset_meta: "LeRobotDatasetMetadata | None" = None,
|
||||
) -> ModelCard:
|
||||
"""Render the LeRobot model card for a trained policy or reward model.
|
||||
|
||||
A free function on purpose: every template variable comes from arguments — the model
|
||||
config, the training config, and the dataset metadata — none from a live model, so a card
|
||||
can also be rendered from a checkpoint's `config.json` alone (see `lerobot-convert-dcp`).
|
||||
The config type selects the template: reward models get the reward-model card, policies the
|
||||
policy card with the training/dataset sections.
|
||||
|
||||
Args:
|
||||
model_cfg (PreTrainedConfig | RewardModelConfig): The model config providing type,
|
||||
license, tags, repo id, and — for policies — the feature declarations.
|
||||
cfg (TrainPipelineConfig | None, optional): The training config for the training and
|
||||
dataset card sections. Defaults to None.
|
||||
dataset_meta (LeRobotDatasetMetadata | None, optional): Dataset metadata for the
|
||||
dataset card sections. Defaults to None.
|
||||
|
||||
Returns:
|
||||
ModelCard: The rendered and validated LeRobot model card.
|
||||
"""
|
||||
model_type = model_cfg.type
|
||||
base_model = _BASE_MODEL_MAPPING.get(model_type)
|
||||
|
||||
if isinstance(model_cfg, RewardModelConfig):
|
||||
tags = {"robotics", "lerobot", "reward-model", model_type}
|
||||
template_card = (
|
||||
files("lerobot.templates")
|
||||
.joinpath("lerobot_rewardmodel_modelcard_template.md")
|
||||
.read_text("utf-8")
|
||||
)
|
||||
context: dict[str, Any] = {} # the reward template renders from card_data alone
|
||||
else:
|
||||
tags = {"robotics", "lerobot", model_type}
|
||||
template_card = (
|
||||
files("lerobot.templates").joinpath("lerobot_modelcard_template.md").read_text("utf-8")
|
||||
)
|
||||
context = build_card_context(cfg, dataset_meta, model_cfg.input_features, model_cfg.output_features)
|
||||
# Used by the template to pre-fill commands and the "Fine-tuned from" line.
|
||||
context["policy_repo_id"] = model_cfg.repo_id
|
||||
context["base_model"] = base_model
|
||||
|
||||
card_data = ModelCardData(
|
||||
license=model_cfg.license or "apache-2.0",
|
||||
library_name="lerobot",
|
||||
pipeline_tag="robotics",
|
||||
tags=list(tags.union(model_cfg.tags or [])),
|
||||
model_name=model_type,
|
||||
datasets=cfg.dataset.repo_id if cfg is not None else None,
|
||||
base_model=base_model,
|
||||
)
|
||||
card = ModelCard.from_template(card_data, template_str=template_card, **context)
|
||||
card.validate()
|
||||
return card
|
||||
|
||||
@@ -1,273 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Execution-runtime configuration: everything handed to (or applied by) the `Accelerator`.
|
||||
|
||||
Each sub-config mirrors the plain-typed subset of the corresponding accelerate object and
|
||||
builds it at runtime (the way ``OptimizerConfig.build()`` constructs a ``torch.optim.Optimizer``),
|
||||
so the whole tree round-trips through the CLI and ``train_config.json`` and parsing a config
|
||||
never imports accelerate.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from lerobot.configs.parallelism import ParallelismConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from accelerate import Accelerator
|
||||
from accelerate.utils import (
|
||||
DistributedDataParallelKwargs,
|
||||
FullyShardedDataParallelPlugin,
|
||||
GradientAccumulationPlugin,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FSDPConfig:
|
||||
"""Mirror of the `FullyShardedDataParallelPlugin` subset LeRobot supports (FSDP2 only).
|
||||
|
||||
Exactly one wrap policy applies: `wrap_modules` (module *class names* forming the FSDP
|
||||
units — and, later, the activation-checkpointing units) or `min_num_params` (size-based).
|
||||
When both are None, the policy's own `_fsdp_wrap_modules` declaration is used; a run where
|
||||
no wrap source exists at all fails loudly rather than silently wrapping only the root.
|
||||
"""
|
||||
|
||||
reshard_after_forward: bool = True
|
||||
wrap_modules: list[str] | None = None
|
||||
min_num_params: int | None = None
|
||||
cpu_offload: bool = False
|
||||
# Regex matched against module FQNs to exclude their parameters from sharding.
|
||||
ignored_modules: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate the wrap-policy fields.
|
||||
|
||||
Raises:
|
||||
ValueError: If both ``wrap_modules`` and ``min_num_params`` are set (they are
|
||||
mutually exclusive wrap policies), or if ``min_num_params`` is < 1.
|
||||
"""
|
||||
if self.wrap_modules is not None and self.min_num_params is not None:
|
||||
raise ValueError(
|
||||
"fsdp.wrap_modules and fsdp.min_num_params are mutually exclusive wrap policies."
|
||||
)
|
||||
if self.min_num_params is not None and self.min_num_params < 1:
|
||||
raise ValueError(f"fsdp.min_num_params must be >= 1, got {self.min_num_params}.")
|
||||
|
||||
def build_plugin(self) -> "FullyShardedDataParallelPlugin":
|
||||
"""Build the FSDP2 plugin for `Accelerator(fsdp_plugin=...)`.
|
||||
|
||||
Returns:
|
||||
FullyShardedDataParallelPlugin: FSDP2 (`fsdp_version=2`) plugin carrying the
|
||||
mirrored wrap policy, resharding, CPU-offload, and ignored-modules settings.
|
||||
"""
|
||||
from accelerate.utils import FullyShardedDataParallelPlugin
|
||||
|
||||
use_size_policy = self.min_num_params is not None
|
||||
return FullyShardedDataParallelPlugin(
|
||||
fsdp_version=2,
|
||||
reshard_after_forward=self.reshard_after_forward,
|
||||
auto_wrap_policy="size_based_wrap" if use_size_policy else "transformer_based_wrap",
|
||||
# May legitimately still be None here: the policy-declared default is applied right
|
||||
# before `accelerator.prepare()` (see lerobot.distributed.factory.set_fsdp_wrap_modules).
|
||||
transformer_cls_names_to_wrap=list(self.wrap_modules) if self.wrap_modules else None,
|
||||
min_num_params=self.min_num_params,
|
||||
cpu_offload=self.cpu_offload,
|
||||
ignored_modules=self.ignored_modules,
|
||||
# state_dict_type stays at the FSDP2 default (SHARDED_STATE_DICT) and is never
|
||||
# switched: full gathers go through torch's state-dict API, which does not consult
|
||||
# the plugin. activation_checkpointing stays False: AC is LeRobot-owned.
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DDPConfig:
|
||||
"""Mirror of the `DistributedDataParallelKwargs` subset LeRobot exposes."""
|
||||
|
||||
# Today's in-script default, kept for models with conditional computation.
|
||||
find_unused_parameters: bool = True
|
||||
gradient_as_bucket_view: bool = False
|
||||
static_graph: bool = False
|
||||
|
||||
def build_kwargs_handler(self) -> "DistributedDataParallelKwargs":
|
||||
"""Build the DDP kwargs handler for `Accelerator(kwargs_handlers=[...])`.
|
||||
|
||||
Returns:
|
||||
DistributedDataParallelKwargs: Handler carrying the mirrored DDP fields, applied
|
||||
by accelerate when it wraps the model in `DistributedDataParallel`.
|
||||
"""
|
||||
from accelerate.utils import DistributedDataParallelKwargs
|
||||
|
||||
return DistributedDataParallelKwargs(
|
||||
find_unused_parameters=self.find_unused_parameters,
|
||||
gradient_as_bucket_view=self.gradient_as_bucket_view,
|
||||
static_graph=self.static_graph,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GradientAccumulationConfig:
|
||||
"""Mirror of the `GradientAccumulationPlugin` subset LeRobot supports.
|
||||
|
||||
Only the step count is a knob. ``sync_with_dataloader`` is pinned to False by
|
||||
:meth:`build_plugin`: the training loop cycles a finite dataloader, so accelerate's default
|
||||
of syncing at every dataloader end would force an optimizer step at every dataset epoch
|
||||
boundary instead of every ``steps`` micro-batches.
|
||||
"""
|
||||
|
||||
steps: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate the accumulation step count.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``steps`` is < 1.
|
||||
"""
|
||||
if self.steps < 1:
|
||||
raise ValueError(f"gradient_accumulation.steps must be >= 1, got {self.steps}.")
|
||||
|
||||
def build_plugin(self) -> "GradientAccumulationPlugin":
|
||||
"""Build the plugin for `Accelerator(gradient_accumulation_plugin=...)`.
|
||||
|
||||
A named plugin argument, not a `kwargs_handlers` entry: accelerate consumes this object
|
||||
through its dedicated constructor parameter — the `KwargsHandler` base class only lends
|
||||
it `to_kwargs()`, so the consumption site, not the inheritance, decides its role.
|
||||
|
||||
Returns:
|
||||
GradientAccumulationPlugin: Carrying the mirrored step count, with
|
||||
``sync_with_dataloader=False`` pinned (see the class docstring).
|
||||
"""
|
||||
from accelerate.utils import GradientAccumulationPlugin
|
||||
|
||||
return GradientAccumulationPlugin(num_steps=self.steps, sync_with_dataloader=False)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompileConfig:
|
||||
"""torch.compile knobs — a configured placeholder: wiring lands in a later round.
|
||||
|
||||
The setup-order contract it will follow is already fixed: compile applies
|
||||
after CP dispatch install and activation checkpointing, before `fully_shard`, regionally
|
||||
(per wrap unit) — the only combination proven with FSDP2.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
backend: str = "inductor"
|
||||
mode: str | None = None
|
||||
regional: bool = True
|
||||
|
||||
|
||||
class ActivationCheckpointingMode(str, Enum):
|
||||
NONE = "none"
|
||||
FULL = "full"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActivationCheckpointingConfig:
|
||||
"""Activation-checkpointing knobs — a configured placeholder: wiring lands in a later round.
|
||||
|
||||
AC units will coincide with the FSDP wrap units (one declaration drives both), applied
|
||||
before torch.compile and `fully_shard` (the same ordering contract as CompileConfig).
|
||||
"""
|
||||
|
||||
mode: ActivationCheckpointingMode = ActivationCheckpointingMode.NONE
|
||||
|
||||
|
||||
@dataclass
|
||||
class AcceleratorConfig:
|
||||
"""Builds the `Accelerator` — the runtime counterpart of the `parallelism` topology.
|
||||
|
||||
`mixed_precision` selects accelerate-native AMP for DDP/single-GPU runs and the FSDP2
|
||||
`MixedPrecisionPolicy` for sharded runs (accelerate derives it). Sharded runs support
|
||||
"no" and "bf16" only; fp16's GradScaler-over-DTensor path is unverified and fails fast
|
||||
at config validation.
|
||||
"""
|
||||
|
||||
mixed_precision: str = "no"
|
||||
gradient_accumulation: GradientAccumulationConfig = field(default_factory=GradientAccumulationConfig)
|
||||
fsdp: FSDPConfig = field(default_factory=FSDPConfig)
|
||||
ddp: DDPConfig = field(default_factory=DDPConfig)
|
||||
compile: CompileConfig = field(default_factory=CompileConfig)
|
||||
activation_checkpointing: ActivationCheckpointingConfig = field(
|
||||
default_factory=ActivationCheckpointingConfig
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate the accelerate-facing scalar fields.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``mixed_precision`` is not one of ``"no"``, ``"fp16"``, ``"bf16"``.
|
||||
"""
|
||||
if self.mixed_precision not in ("no", "fp16", "bf16"):
|
||||
raise ValueError(
|
||||
f"mixed_precision must be one of 'no', 'fp16', 'bf16', got {self.mixed_precision!r}."
|
||||
)
|
||||
|
||||
def build(self, parallelism: ParallelismConfig, *, cpu: bool = False) -> "Accelerator":
|
||||
"""Translate the mirrored fields into a ready `Accelerator` (call once per process).
|
||||
|
||||
`parallelism` must already be resolved against the world size. The degradation matrix
|
||||
is encoded here and nowhere else: sharded -> FSDP2 (+HSDP via the accelerate
|
||||
`ParallelismConfig` mesh), replicated-only -> DDP kwargs, single process -> plain.
|
||||
|
||||
Args:
|
||||
parallelism (ParallelismConfig): The resolved process topology; selects which
|
||||
accelerate path (FSDP2 mesh, DDP kwargs handler, or plain) is configured.
|
||||
cpu (bool): Force CPU execution even when CUDA is available. Defaults to False.
|
||||
|
||||
Returns:
|
||||
Accelerator: The configured accelerate entry point for this process.
|
||||
"""
|
||||
from accelerate import Accelerator
|
||||
|
||||
kwargs: dict = {
|
||||
# LeRobot steps its scheduler manually once per training step; accelerate must not
|
||||
# rescale scheduler stepping by num_processes.
|
||||
"step_scheduler_with_optimizer": False,
|
||||
"gradient_accumulation_plugin": self.gradient_accumulation.build_plugin(),
|
||||
"mixed_precision": self.mixed_precision,
|
||||
"cpu": cpu,
|
||||
}
|
||||
if parallelism.is_sharded:
|
||||
kwargs["fsdp_plugin"] = self.fsdp.build_plugin()
|
||||
kwargs["parallelism_config"] = _accelerate_parallelism_config(parallelism)
|
||||
elif parallelism.is_replicated_only:
|
||||
kwargs["kwargs_handlers"] = [self.ddp.build_kwargs_handler()]
|
||||
return Accelerator(**kwargs)
|
||||
|
||||
|
||||
def _accelerate_parallelism_config(parallelism: ParallelismConfig) -> object:
|
||||
"""LeRobot topology -> accelerate `ParallelismConfig`.
|
||||
|
||||
CP is declared honestly (`cp_size = ring x ulysses`) so accelerate builds the canonical
|
||||
mesh, folds CP into the FSDP shard group (`dp_shard_cp`), and duplicates batches within CP
|
||||
groups. The ring/ulysses sub-structure stays private to `lerobot.distributed.ParallelDims`.
|
||||
|
||||
Args:
|
||||
parallelism (ParallelismConfig): The resolved LeRobot topology to translate.
|
||||
|
||||
Returns:
|
||||
object: The accelerate `ParallelismConfig` mirroring `dp_replicate`, `dp_shard`, and
|
||||
the collapsed `cp_size` (annotated as `object` so importing this module never
|
||||
imports accelerate).
|
||||
"""
|
||||
from accelerate.parallelism_config import ParallelismConfig as AccelerateParallelismConfig
|
||||
|
||||
return AccelerateParallelismConfig(
|
||||
dp_replicate_size=parallelism.dp_replicate,
|
||||
dp_shard_size=parallelism.dp_shard,
|
||||
cp_size=parallelism.cp_size,
|
||||
)
|
||||
@@ -1,190 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Declarative process topology for distributed training and inference.
|
||||
|
||||
The mesh convention (canonical row-major rank layout, outermost first)::
|
||||
|
||||
(dp_replicate, dp_shard, ring, ulysses)
|
||||
|
||||
- ``dp_replicate x dp_shard`` is the data-parallel world: HSDP replicates over
|
||||
``dp_replicate`` and shards parameters over ``dp_shard``. FSDP2's actual shard
|
||||
group folds context parallelism in (``dp_shard x ring x ulysses``), matching
|
||||
accelerate's ``dp_shard_cp`` flattening and torchtitan's ``fsdp`` axis.
|
||||
- ``ring`` is the outer and ``ulysses`` the inner context-parallel dim
|
||||
(diffusers convention: ulysses all-to-all exchanges run over adjacent, typically
|
||||
NVLink-connected ranks).
|
||||
- ``cfg_parallel`` (classifier-free-guidance parallelism) is a branch-parallel,
|
||||
inference-only dim that sits between dp and the sequence dims. It never
|
||||
affects weight sharding or checkpoints.
|
||||
|
||||
This module is pure configuration: plain-typed dataclasses that draccus can
|
||||
round-trip through the CLI and ``train_config.json``. Runtime objects (device
|
||||
meshes, process groups) live in :mod:`lerobot.distributed`.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContextParallelConfig:
|
||||
"""Ring x Ulysses context parallelism (sequence parallelism for attention).
|
||||
|
||||
Both degrees are configured placeholders in this release: the CP engine is not implemented
|
||||
yet, and enabling either degree > 1 fails fast at config validation. The fields exist now so
|
||||
that the CLI surface, checkpoint metadata, and mesh math are stable when the engine lands.
|
||||
"""
|
||||
|
||||
ring_degree: int = 1
|
||||
ulysses_degree: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate the declared context-parallel degrees.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``ring_degree`` or ``ulysses_degree`` is < 1.
|
||||
"""
|
||||
if self.ring_degree < 1 or self.ulysses_degree < 1:
|
||||
raise ValueError(
|
||||
f"Context-parallel degrees must be >= 1, got ring_degree={self.ring_degree}, "
|
||||
f"ulysses_degree={self.ulysses_degree}."
|
||||
)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
"""Total number of ranks a full sequence is sharded across."""
|
||||
return self.ring_degree * self.ulysses_degree
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParallelismConfig:
|
||||
"""Degrees of every parallelism dim. Invariant: their product equals the world size.
|
||||
|
||||
Degradations are expressed purely through the degrees (no mode flags):
|
||||
|
||||
- single process: all degrees 1;
|
||||
- DDP: ``dp_replicate == world_size`` (auto-filled when every sharding field is left at its
|
||||
default — plain ``torchrun`` keeps today's out-of-the-box behavior);
|
||||
- FSDP: ``dp_shard > 1`` (or ``-1`` to fill the remaining world into the shard dim);
|
||||
- HSDP: ``dp_replicate > 1`` and ``dp_shard > 1``.
|
||||
|
||||
``resolve()`` turns the declared degrees into concrete ones once the world size is known and
|
||||
is the single place the world-size equation is enforced. It is called by
|
||||
:func:`lerobot.distributed.factory.make_accelerator`; the config is inert until then.
|
||||
"""
|
||||
|
||||
dp_replicate: int = 1
|
||||
# -1 is an explicit opt-in sentinel: shard over world_size // (dp_replicate * cp).
|
||||
dp_shard: int = 1
|
||||
context_parallel: ContextParallelConfig = field(default_factory=ContextParallelConfig)
|
||||
# Classifier-free-guidance parallelism — inference-only (cosmos/vllm-omni precedent:
|
||||
# cond/uncond branches on different ranks). Reserved for the serving round; training
|
||||
# validates it to 1. Meaningful values are 1 or 2 (Cosmos3 has two CFG branches).
|
||||
cfg_parallel: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate the declared degrees (world-size-independent checks only).
|
||||
|
||||
Raises:
|
||||
ValueError: If ``dp_replicate`` is < 1, ``dp_shard`` is neither >= 1 nor the
|
||||
``-1`` infer sentinel, or ``cfg_parallel`` is not 1 or 2.
|
||||
"""
|
||||
if self.dp_replicate < 1:
|
||||
raise ValueError(f"dp_replicate must be >= 1, got {self.dp_replicate}.")
|
||||
if self.dp_shard < 1 and self.dp_shard != -1:
|
||||
raise ValueError(f"dp_shard must be >= 1, or -1 to infer, got {self.dp_shard}.")
|
||||
if self.cfg_parallel not in (1, 2):
|
||||
raise ValueError(f"cfg_parallel must be 1 or 2, got {self.cfg_parallel}.")
|
||||
|
||||
@property
|
||||
def cp_size(self) -> int:
|
||||
"""Total context-parallel size (``ring_degree * ulysses_degree``)."""
|
||||
return self.context_parallel.size
|
||||
|
||||
@property
|
||||
def is_sharded(self) -> bool:
|
||||
"""True when the run uses FSDP2 (parameters sharded); selects the sharded engine path."""
|
||||
return self.dp_shard != 1 or self.cp_size > 1
|
||||
|
||||
@property
|
||||
def is_replicated_only(self) -> bool:
|
||||
"""True for plain DDP (weights replicated, no sharding)."""
|
||||
return not self.is_sharded and self.dp_replicate > 1
|
||||
|
||||
@property
|
||||
def dp_world_size(self) -> int:
|
||||
"""Number of distinct data-parallel workers (batches are sharded this many ways).
|
||||
|
||||
Returns:
|
||||
int: ``dp_replicate * dp_shard``.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If accessed while ``dp_shard`` is still the ``-1`` sentinel, i.e.
|
||||
before :meth:`resolve` has bound the degrees to a world size.
|
||||
"""
|
||||
if self.dp_shard == -1:
|
||||
raise RuntimeError("dp_world_size is undefined before resolve() fills dp_shard=-1.")
|
||||
return self.dp_replicate * self.dp_shard
|
||||
|
||||
def resolve(self, world_size: int) -> None:
|
||||
"""Bind the declared degrees to a concrete world size (idempotent).
|
||||
|
||||
Fills the ``dp_shard=-1`` sentinel, auto-fills ``dp_replicate`` for the DDP degradation,
|
||||
and enforces ``dp_replicate * dp_shard * cp == world_size`` with every degree echoed on
|
||||
failure.
|
||||
|
||||
Args:
|
||||
world_size (int): Total number of launched processes (torchrun's ``WORLD_SIZE``).
|
||||
|
||||
Raises:
|
||||
ValueError: If a context-parallel degree is > 1 (the CP engine is not implemented
|
||||
yet), if ``dp_shard=-1`` cannot be inferred because ``world_size`` is not
|
||||
divisible by ``dp_replicate * cp``, or if the resolved degrees do not multiply
|
||||
to ``world_size``.
|
||||
"""
|
||||
if self.cp_size > 1:
|
||||
raise ValueError(
|
||||
"Context parallelism is not implemented yet: ring_degree and ulysses_degree "
|
||||
"must be 1. The fields are reserved for the CP engine round."
|
||||
)
|
||||
if self.is_sharded:
|
||||
if self.dp_shard == -1:
|
||||
self.dp_shard, remainder = divmod(world_size, self.dp_replicate * self.cp_size)
|
||||
if remainder or self.dp_shard < 1:
|
||||
raise ValueError(
|
||||
f"Cannot infer dp_shard: world_size={world_size} is not divisible by "
|
||||
f"dp_replicate={self.dp_replicate} * cp={self.cp_size}."
|
||||
)
|
||||
elif self.dp_replicate == 1:
|
||||
# Untouched config on a multi-process launch: fill the DDP degradation.
|
||||
self.dp_replicate = world_size
|
||||
total = self.dp_replicate * self.dp_shard * self.cp_size
|
||||
if total != world_size:
|
||||
raise ValueError(
|
||||
f"Parallelism degrees do not multiply to the world size: dp_replicate="
|
||||
f"{self.dp_replicate} * dp_shard={self.dp_shard} * ring="
|
||||
f"{self.context_parallel.ring_degree} * ulysses="
|
||||
f"{self.context_parallel.ulysses_degree} = {total} != WORLD_SIZE={world_size}."
|
||||
)
|
||||
|
||||
|
||||
def world_size_from_env() -> int:
|
||||
"""World size as set by torchrun (or 1 outside distributed launches).
|
||||
|
||||
Returns:
|
||||
int: The ``WORLD_SIZE`` environment variable, or 1 when unset.
|
||||
"""
|
||||
return int(os.environ.get("WORLD_SIZE", "1"))
|
||||
@@ -18,7 +18,6 @@ import multiprocessing
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -27,8 +26,6 @@ from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
|
||||
from lerobot import envs
|
||||
from lerobot.configs.accelerator import AcceleratorConfig, ActivationCheckpointingMode
|
||||
from lerobot.configs.parallelism import ParallelismConfig
|
||||
from lerobot.optim import LRSchedulerConfig, OptimizerConfig
|
||||
from lerobot.utils.constants import PRETRAINED_MODEL_DIR
|
||||
from lerobot.utils.hub import HubMixin, find_latest_hub_checkpoint
|
||||
@@ -42,34 +39,6 @@ from .rewards import RewardModelConfig
|
||||
TRAIN_CONFIG_NAME = "train_config.json"
|
||||
|
||||
|
||||
class CheckpointFormat(str, Enum):
|
||||
"""Model-artifact format inside training checkpoints.
|
||||
|
||||
Selects only the *model* artifact; the training_state layout is format-independent (the
|
||||
optimizer channel is always DCP under sharded runs, safetensors+json otherwise).
|
||||
|
||||
- SAFETENSORS (default): a full `model.safetensors` — maximum compatibility, one gather per
|
||||
save under sharding.
|
||||
- DCP: sharded `pytorch_model_fsdp_0/*.distcp` only — fastest save/resume; convert with
|
||||
`lerobot-convert-dcp` before distributing.
|
||||
- SAFETENSORS_AND_DCP: both artifacts, written independently.
|
||||
"""
|
||||
|
||||
SAFETENSORS = "safetensors"
|
||||
DCP = "dcp"
|
||||
SAFETENSORS_AND_DCP = "safetensors_dcp"
|
||||
|
||||
@property
|
||||
def wants_safetensors(self) -> bool:
|
||||
"""True when a full `model.safetensors` artifact should be written."""
|
||||
return self in (CheckpointFormat.SAFETENSORS, CheckpointFormat.SAFETENSORS_AND_DCP)
|
||||
|
||||
@property
|
||||
def wants_dcp(self) -> bool:
|
||||
"""True when sharded DCP model shards (`pytorch_model_fsdp_0/`) should be written."""
|
||||
return self in (CheckpointFormat.DCP, CheckpointFormat.SAFETENSORS_AND_DCP)
|
||||
|
||||
|
||||
def _migrate_legacy_rabc_fields(config: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Return migrated payload for legacy RA-BC fields, or None when no migration is needed."""
|
||||
legacy_fields = (
|
||||
@@ -152,16 +121,9 @@ class TrainPipelineConfig(HubMixin):
|
||||
# Checkpoint is saved every `save_freq` training iterations and after the last training step.
|
||||
# A non-positive value disables periodic saving, keeping only the final checkpoint.
|
||||
save_freq: int = 20_000
|
||||
# Model-artifact format inside checkpoints; non-default values require a sharded run.
|
||||
checkpoint_format: CheckpointFormat = CheckpointFormat.SAFETENSORS
|
||||
use_policy_training_preset: bool = True
|
||||
optimizer: OptimizerConfig | None = None
|
||||
scheduler: LRSchedulerConfig | None = None
|
||||
# Process topology: dp_replicate / dp_shard (HSDP) and context-parallel degree placeholders.
|
||||
parallelism: ParallelismConfig = field(default_factory=ParallelismConfig)
|
||||
# Execution runtime handed to the Accelerator: mixed precision, gradient accumulation,
|
||||
# FSDP/DDP tuning knobs, compile & activation-checkpointing placeholders.
|
||||
accelerator: AcceleratorConfig = field(default_factory=AcceleratorConfig)
|
||||
eval: EvalConfig = field(default_factory=EvalConfig)
|
||||
wandb: WandBConfig = field(default_factory=WandBConfig)
|
||||
peft: PeftConfig | None = None
|
||||
@@ -329,60 +291,6 @@ class TrainPipelineConfig(HubMixin):
|
||||
if self.save_checkpoint_to_hub and not (self.policy is not None and self.policy.repo_id):
|
||||
raise ValueError("save_checkpoint_to_hub requires --policy.repo_id.")
|
||||
|
||||
self._validate_distributed()
|
||||
|
||||
def _validate_distributed(self) -> None:
|
||||
"""Fail-fasts for the distributed-training scope.
|
||||
|
||||
Raises:
|
||||
ValueError: If the config requests anything outside the verified scope: context
|
||||
parallelism or CFG parallelism (reserved placeholders), the compile or
|
||||
activation-checkpointing placeholders, a DCP checkpoint format on a
|
||||
non-sharded run, or — under sharded training — fp16 mixed precision, PEFT,
|
||||
reward-model training, in-training environment evaluation, or multi-optimizer
|
||||
configs.
|
||||
"""
|
||||
if self.parallelism.cp_size > 1:
|
||||
raise ValueError(
|
||||
"Context parallelism is not implemented yet: --parallelism.context_parallel.* "
|
||||
"degrees must be 1 (reserved for the CP engine round)."
|
||||
)
|
||||
if self.parallelism.cfg_parallel != 1:
|
||||
raise ValueError(
|
||||
"CFG parallelism is inference-only and must be 1 for training "
|
||||
"(cfg_parallel is reserved for the serving round)."
|
||||
)
|
||||
if self.accelerator.compile.enabled:
|
||||
raise ValueError("--accelerator.compile is a placeholder and not wired yet.")
|
||||
if self.accelerator.activation_checkpointing.mode is not ActivationCheckpointingMode.NONE:
|
||||
raise ValueError("--accelerator.activation_checkpointing is a placeholder and not wired yet.")
|
||||
if self.checkpoint_format is not CheckpointFormat.SAFETENSORS and not self.parallelism.is_sharded:
|
||||
raise ValueError(
|
||||
f"checkpoint_format={self.checkpoint_format.value} requires a sharded run "
|
||||
"(--parallelism.dp_shard != 1); non-sharded checkpoints are always safetensors."
|
||||
)
|
||||
if self.parallelism.is_sharded:
|
||||
if self.accelerator.mixed_precision == "fp16":
|
||||
raise ValueError(
|
||||
"fp16 is not supported under sharded training (GradScaler over DTensor "
|
||||
"gradients is unverified); use bf16 or full precision."
|
||||
)
|
||||
if self.peft is not None:
|
||||
raise ValueError("PEFT is not supported under sharded training yet.")
|
||||
if self.is_reward_model_training:
|
||||
raise ValueError(
|
||||
"Reward-model training is not supported under sharded training yet "
|
||||
"(reward models declare no FSDP wrap units and have no sharded save path)."
|
||||
)
|
||||
if self.env is not None and self.env_eval_freq > 0:
|
||||
raise ValueError(
|
||||
"In-training environment evaluation is not supported under sharded training "
|
||||
"(a rank-0-only rollout of a sharded model deadlocks on collectives); set "
|
||||
"--env_eval_freq=0 and evaluate with lerobot-eval on saved checkpoints."
|
||||
)
|
||||
if self.optimizer is not None and self.optimizer.builds_multiple_optimizers:
|
||||
raise ValueError("Multi-optimizer configs are not supported under sharded training.")
|
||||
|
||||
@classmethod
|
||||
def __get_path_fields__(cls) -> list[str]:
|
||||
"""Keys for draccus pretrained-path loading."""
|
||||
|
||||
@@ -58,38 +58,12 @@ type ChunkFile = tuple[int, int]
|
||||
|
||||
|
||||
class IndexState(TypedDict):
|
||||
"""The current write cursor for a non-video (parquet) output stream during aggregation.
|
||||
|
||||
**Attributes**:
|
||||
- **chunk** (`int`) -- The chunk index currently being written to.
|
||||
- **file** (`int`) -- The file index, within `chunk`, currently being written to.
|
||||
- **src_to_dst** (`dict[ChunkFile, ChunkFile]`, *optional*) -- Maps each source dataset's
|
||||
`(chunk, file)` to the destination `(chunk, file)` its rows were merged into.
|
||||
"""
|
||||
|
||||
chunk: int
|
||||
file: int
|
||||
src_to_dst: NotRequired[dict[ChunkFile, ChunkFile]]
|
||||
|
||||
|
||||
class VideoIndex(TypedDict):
|
||||
"""The current write cursor for a video output stream during aggregation.
|
||||
|
||||
**Attributes**:
|
||||
- **chunk** (`int`) -- The chunk index currently being written to.
|
||||
- **file** (`int`) -- The file index, within `chunk`, currently being written to.
|
||||
- **latest_duration** (`float`) -- The duration, in seconds, appended to the current destination
|
||||
file so far.
|
||||
- **episode_duration** (`float`) -- The duration, in seconds, of the episode currently being
|
||||
concatenated.
|
||||
- **src_to_offset** (`dict[ChunkFile, float]`, *optional*) -- Maps each source `(chunk, file)` to
|
||||
the time offset, in seconds, at which it was appended into its destination file.
|
||||
- **src_to_dst** (`dict[ChunkFile, ChunkFile]`, *optional*) -- Maps each source `(chunk, file)` to
|
||||
the destination `(chunk, file)` its video was concatenated into.
|
||||
- **dst_file_durations** (`dict[ChunkFile, float]`, *optional*) -- The final total duration, in
|
||||
seconds, of each completed destination file.
|
||||
"""
|
||||
|
||||
chunk: int
|
||||
file: int
|
||||
latest_duration: float
|
||||
@@ -106,7 +80,7 @@ def merge_video_feature_info_for_aggregate(all_metadata: list[LeRobotDatasetMeta
|
||||
"""Create a merged video feature info dictionary for aggregation. The video encoder info is merged field-by-field: each key is kept only when every source agrees; otherwise that key is set to ``null`` (or ``{}`` for ``video.extra_options``) and a warning is logged.
|
||||
|
||||
Args:
|
||||
all_metadata (`list`): List of `LeRobotDatasetMetadata` objects to merge.
|
||||
all_metadata: List of LeRobotDatasetMetadata objects to merge.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary of merged video feature info.
|
||||
@@ -152,7 +126,7 @@ def validate_all_metadata(all_metadata: list[LeRobotDatasetMetadata]) -> tuple[i
|
||||
Video encoder info is not considered for validation but is merged during aggregation in ``merge_video_feature_info_for_aggregate``.
|
||||
|
||||
Args:
|
||||
all_metadata (`list`): List of `LeRobotDatasetMetadata` objects to validate.
|
||||
all_metadata: List of LeRobotDatasetMetadata objects to validate.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing (fps, robot_type, features) from the first metadata.
|
||||
@@ -161,6 +135,7 @@ def validate_all_metadata(all_metadata: list[LeRobotDatasetMetadata]) -> tuple[i
|
||||
ValueError: If any metadata has different fps, robot_type, or features
|
||||
than the first metadata in the list.
|
||||
"""
|
||||
|
||||
fps = all_metadata[0].fps
|
||||
robot_type = all_metadata[0].robot_type
|
||||
features = all_metadata[0].features
|
||||
@@ -189,13 +164,14 @@ def update_data_df(
|
||||
previously aggregated data in the destination dataset.
|
||||
|
||||
Args:
|
||||
df (`DataFrame`): DataFrame containing the data to be updated.
|
||||
src_meta (`LeRobotDatasetMetadata`): Source dataset metadata.
|
||||
dst_meta (`LeRobotDatasetMetadata`): Destination dataset metadata.
|
||||
df: DataFrame containing the data to be updated.
|
||||
src_meta: Source dataset metadata.
|
||||
dst_meta: Destination dataset metadata.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Updated DataFrame with adjusted indices.
|
||||
"""
|
||||
|
||||
df["episode_index"] = df["episode_index"] + dst_meta.info.total_episodes
|
||||
df["index"] = df["index"] + dst_meta.info.total_frames
|
||||
|
||||
@@ -221,15 +197,16 @@ def update_meta_data(
|
||||
to correctly map source file indices to their destination locations.
|
||||
|
||||
Args:
|
||||
df (`DataFrame`): DataFrame containing the metadata to be updated.
|
||||
dst_meta (`LeRobotDatasetMetadata`): Destination dataset metadata.
|
||||
meta_idx (`IndexState`): Dictionary containing current metadata chunk and file indices.
|
||||
data_idx (`IndexState`): Dictionary containing current data chunk and file indices.
|
||||
videos_idx (`VideoIndexState`): Dictionary containing current video indices and timestamps.
|
||||
df: DataFrame containing the metadata to be updated.
|
||||
dst_meta: Destination dataset metadata.
|
||||
meta_idx: Dictionary containing current metadata chunk and file indices.
|
||||
data_idx: Dictionary containing current data chunk and file indices.
|
||||
videos_idx: Dictionary containing current video indices and timestamps.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Updated DataFrame with adjusted indices and timestamps.
|
||||
"""
|
||||
|
||||
df["meta/episodes/chunk_index"] = df["meta/episodes/chunk_index"] + meta_idx["chunk"]
|
||||
df["meta/episodes/file_index"] = df["meta/episodes/file_index"] + meta_idx["file"]
|
||||
|
||||
@@ -390,21 +367,15 @@ def aggregate_datasets(
|
||||
4. Finalizing the aggregated dataset with proper statistics
|
||||
|
||||
Args:
|
||||
repo_ids (`list`): List of repository IDs for the datasets to aggregate.
|
||||
aggr_repo_id (`str`): Repository ID for the aggregated output dataset.
|
||||
roots (`list[pathlib.Path] | None`, *optional*): List of root paths for the source
|
||||
datasets.
|
||||
aggr_root (`pathlib.Path | None`, *optional*): Root path for the aggregated dataset.
|
||||
data_files_size_in_mb (`int | None`, *optional*): Maximum size for data files in MB. Falls
|
||||
back to `DEFAULT_DATA_FILE_SIZE_IN_MB` when not set.
|
||||
video_files_size_in_mb (`int | None`, *optional*): Maximum size for video files in MB. Falls
|
||||
back to `DEFAULT_VIDEO_FILE_SIZE_IN_MB` when not set.
|
||||
chunk_size (`int | None`, *optional*): Maximum number of files per chunk. Falls back to
|
||||
`DEFAULT_CHUNK_SIZE` when not set.
|
||||
concatenate_videos (`bool`, *optional*, defaults to `True`): When `False`, keep one mp4 per
|
||||
source file instead of packing into shards.
|
||||
concatenate_data (`bool`, *optional*, defaults to `True`): When `False`, keep one parquet per
|
||||
source file instead of packing into shards.
|
||||
repo_ids: List of repository IDs for the datasets to aggregate.
|
||||
aggr_repo_id: Repository ID for the aggregated output dataset.
|
||||
roots: Optional list of root paths for the source datasets.
|
||||
aggr_root: Optional root path for the aggregated dataset.
|
||||
data_files_size_in_mb: Maximum size for data files in MB (defaults to DEFAULT_DATA_FILE_SIZE_IN_MB)
|
||||
video_files_size_in_mb: Maximum size for video files in MB (defaults to DEFAULT_VIDEO_FILE_SIZE_IN_MB)
|
||||
chunk_size: Maximum number of files per chunk (defaults to DEFAULT_CHUNK_SIZE)
|
||||
concatenate_videos: When False, keep one mp4 per source file instead of packing into shards.
|
||||
concatenate_data: When False, keep one parquet per source file instead of packing into shards.
|
||||
"""
|
||||
logger.info("Start aggregate_datasets")
|
||||
|
||||
@@ -487,14 +458,12 @@ def aggregate_videos(
|
||||
Creates new video files when size limits are exceeded.
|
||||
|
||||
Args:
|
||||
src_meta (`LeRobotDatasetMetadata`): Source dataset metadata.
|
||||
dst_meta (`LeRobotDatasetMetadata`): Destination dataset metadata.
|
||||
videos_idx (`VideoIndexState`): Dictionary tracking video chunk and file indices.
|
||||
video_files_size_in_mb (`float`): Maximum size for video files in MB.
|
||||
chunk_size (`int`): Maximum number of files per chunk.
|
||||
concatenate_videos (`bool`, *optional*, defaults to `True`): When `False`, keep one mp4 per
|
||||
source file instead of packing into shards.
|
||||
|
||||
src_meta: Source dataset metadata.
|
||||
dst_meta: Destination dataset metadata.
|
||||
videos_idx: Dictionary tracking video chunk and file indices.
|
||||
video_files_size_in_mb: Maximum size for video files in MB (defaults to DEFAULT_VIDEO_FILE_SIZE_IN_MB)
|
||||
chunk_size: Maximum number of files per chunk (defaults to DEFAULT_CHUNK_SIZE)
|
||||
concatenate_videos: When False, keep one mp4 per source file instead of packing into shards.
|
||||
Returns:
|
||||
dict: Updated videos_idx with current chunk and file indices.
|
||||
"""
|
||||
@@ -612,13 +581,12 @@ def aggregate_data(
|
||||
have multiple data files (e.g., from a previous merge operation).
|
||||
|
||||
Args:
|
||||
src_meta (`LeRobotDatasetMetadata`): Source dataset metadata.
|
||||
dst_meta (`LeRobotDatasetMetadata`): Destination dataset metadata.
|
||||
data_idx (`IndexState`): Dictionary tracking data chunk and file indices.
|
||||
data_files_size_in_mb (`float`): Maximum size for data files in MB.
|
||||
chunk_size (`int`): Maximum number of files per chunk.
|
||||
concatenate_data (`bool`, *optional*, defaults to `True`): When `False`, keep one parquet per
|
||||
source file instead of packing into shards.
|
||||
src_meta: Source dataset metadata.
|
||||
dst_meta: Destination dataset metadata.
|
||||
data_idx: Dictionary tracking data chunk and file indices.
|
||||
data_files_size_in_mb: Maximum size for data files in MB.
|
||||
chunk_size: Maximum number of files per chunk.
|
||||
concatenate_data: When False, keep one parquet per source file instead of packing into shards.
|
||||
|
||||
Returns:
|
||||
dict: Updated data_idx with current chunk and file indices.
|
||||
@@ -692,11 +660,11 @@ def aggregate_metadata(
|
||||
and writes them to the destination with proper file rotation.
|
||||
|
||||
Args:
|
||||
src_meta (`LeRobotDatasetMetadata`): Source dataset metadata.
|
||||
dst_meta (`LeRobotDatasetMetadata`): Destination dataset metadata.
|
||||
meta_idx (`IndexState`): Dictionary tracking metadata chunk and file indices.
|
||||
data_idx (`IndexState`): Dictionary tracking data chunk and file indices.
|
||||
videos_idx (`VideoIndexState`): Dictionary tracking video indices and timestamps.
|
||||
src_meta: Source dataset metadata.
|
||||
dst_meta: Destination dataset metadata.
|
||||
meta_idx: Dictionary tracking metadata chunk and file indices.
|
||||
data_idx: Dictionary tracking data chunk and file indices.
|
||||
videos_idx: Dictionary tracking video indices and timestamps.
|
||||
|
||||
Returns:
|
||||
dict: Updated meta_idx with current chunk and file indices.
|
||||
@@ -759,22 +727,18 @@ def append_or_create_parquet_file(
|
||||
from becoming too large. Handles both regular parquet files and those containing images.
|
||||
|
||||
Args:
|
||||
df (`DataFrame`): DataFrame to write to the parquet file.
|
||||
src_path (`Path`): Path to the source file, used for size estimation.
|
||||
idx (`IndexState`): Dictionary containing current `chunk` and `file` indices.
|
||||
max_mb (`float`): Maximum allowed file size in MB before rotation.
|
||||
chunk_size (`int`): Maximum number of files per chunk before incrementing the chunk index.
|
||||
default_path (`str`): Format string for generating file paths.
|
||||
contains_images (`bool`, *optional*, defaults to `False`): Whether the data contains images
|
||||
requiring special handling.
|
||||
aggr_root (`pathlib.Path | None`, *optional*): Root path for the aggregated dataset.
|
||||
hf_features (`datasets.features.features.Features | None`, *optional*): HuggingFace Features
|
||||
schema used for proper image typing.
|
||||
concatenate (`bool`, *optional*, defaults to `True`): When `False`, always rotate to a new
|
||||
file instead of appending to the current one.
|
||||
one_row_group_per_episode (`bool`, *optional*, defaults to `False`): Whether to emit one
|
||||
parquet row group per episode. Set to `True` for data parquet files; left `False` for the
|
||||
episodes-metadata parquet, which already has one row per episode.
|
||||
df: DataFrame to write to the parquet file.
|
||||
src_path: Path to the source file (used for size estimation).
|
||||
idx: Dictionary containing current 'chunk' and 'file' indices.
|
||||
max_mb: Maximum allowed file size in MB before rotation.
|
||||
chunk_size: Maximum number of files per chunk before incrementing chunk index.
|
||||
default_path: Format string for generating file paths.
|
||||
contains_images: Whether the data contains images requiring special handling.
|
||||
aggr_root: Root path for the aggregated dataset.
|
||||
hf_features: Optional HuggingFace Features schema for proper image typing.
|
||||
concatenate: When False, always rotate to a new file instead of appending to the current one.
|
||||
one_row_group_per_episode: True for DATA parquet (emit one row group per episode); False for
|
||||
the episodes-metadata parquet (already one row per episode).
|
||||
|
||||
Returns:
|
||||
tuple: (updated_idx, (dst_chunk, dst_file)) where updated_idx is the index dict
|
||||
@@ -838,8 +802,8 @@ def finalize_aggregation(
|
||||
aggregated statistics from all source datasets.
|
||||
|
||||
Args:
|
||||
aggr_meta (`LeRobotDatasetMetadata`): Aggregated dataset metadata.
|
||||
all_metadata (`list`): List of all source dataset metadata objects.
|
||||
aggr_meta: Aggregated dataset metadata.
|
||||
all_metadata: List of all source dataset metadata objects.
|
||||
"""
|
||||
logger.info("write tasks")
|
||||
write_tasks(aggr_meta.tasks, aggr_meta.root)
|
||||
|
||||
@@ -28,22 +28,16 @@ DEFAULT_QUANTILES = [0.01, 0.10, 0.50, 0.90, 0.99]
|
||||
|
||||
|
||||
class RunningQuantileStats:
|
||||
"""Maintains running statistics for batches of vectors.
|
||||
"""
|
||||
Maintains running statistics for batches of vectors, including mean,
|
||||
standard deviation, min, max, and approximate quantiles.
|
||||
|
||||
Includes mean, standard deviation, min, max, and approximate quantiles. Statistics are computed per
|
||||
feature dimension and updated incrementally
|
||||
Statistics are computed per feature dimension and updated incrementally
|
||||
as new batches are observed. Quantiles are estimated using histograms,
|
||||
which adapt dynamically if the observed data range expands.
|
||||
"""
|
||||
|
||||
def __init__(self, quantile_list: list[float] | None = None, num_quantile_bins: int = 5000):
|
||||
"""Initialize empty running statistics.
|
||||
|
||||
Args:
|
||||
quantile_list: Quantiles to track (e.g. `0.01` for the 1st percentile). Defaults to
|
||||
`DEFAULT_QUANTILES` (1st, 10th, 50th, 90th, 99th percentiles).
|
||||
num_quantile_bins: Number of histogram bins used to estimate quantiles.
|
||||
"""
|
||||
self._count = 0
|
||||
self._mean = None
|
||||
self._mean_of_squares = None
|
||||
@@ -210,9 +204,8 @@ def estimate_num_samples(
|
||||
dataset_len: int, min_num_samples: int = 100, max_num_samples: int = 10_000, power: float = 0.75
|
||||
) -> int:
|
||||
"""Heuristic to estimate the number of samples based on dataset size.
|
||||
|
||||
The power controls the sample growth relative to dataset size. Lower the power for less number of
|
||||
samples.
|
||||
The power controls the sample growth relative to dataset size.
|
||||
Lower the power for less number of samples.
|
||||
|
||||
For default arguments, we have:
|
||||
- from 1 to ~500, num_samples=100
|
||||
@@ -228,24 +221,11 @@ def estimate_num_samples(
|
||||
|
||||
|
||||
def sample_indices(data_len: int) -> list[int]:
|
||||
"""Return evenly-spaced indices into a sequence of length `data_len`, sized by `estimate_num_samples`."""
|
||||
num_samples = estimate_num_samples(data_len)
|
||||
return np.round(np.linspace(0, data_len - 1, num_samples)).astype(int).tolist()
|
||||
|
||||
|
||||
def auto_downsample_height_width(img: np.ndarray, target_size: int = 150, max_size_threshold: int = 300):
|
||||
"""Downsample a `(C, H, W)` image by integer striding if either dimension exceeds `max_size_threshold`.
|
||||
|
||||
Args:
|
||||
img (`np.ndarray`): Input image in `(C, H, W)` layout to potentially downsample.
|
||||
target_size (`int`, *optional*, defaults to 150): Approximate size, in pixels, that the
|
||||
largest side should be reduced to.
|
||||
max_size_threshold (`int`, *optional*, defaults to 300): Size, in pixels, above which the
|
||||
largest side of `img` triggers downsampling.
|
||||
|
||||
Returns:
|
||||
`img` unchanged, or strided down so its largest side is roughly `target_size`.
|
||||
"""
|
||||
_, height, width = img.shape
|
||||
|
||||
if max(width, height) < max_size_threshold:
|
||||
@@ -257,16 +237,6 @@ def auto_downsample_height_width(img: np.ndarray, target_size: int = 150, max_si
|
||||
|
||||
|
||||
def sample_images(image_paths: list[str]) -> np.ndarray:
|
||||
"""Load and downsample a sampled subset of `image_paths` into a single `uint8` array.
|
||||
|
||||
Args:
|
||||
image_paths (`list[str]`): Paths of all images for the episode/feature, from which a
|
||||
subset is sampled (see `sample_indices`).
|
||||
|
||||
Returns:
|
||||
A `(N, C, H, W)` `uint8` array of the sampled, downsampled images (see
|
||||
`auto_downsample_height_width`), where `N` is chosen by `sample_indices`.
|
||||
"""
|
||||
sampled_indices = sample_indices(len(image_paths))
|
||||
|
||||
images = None
|
||||
@@ -439,7 +409,6 @@ def _compute_basic_stats(
|
||||
Args:
|
||||
array: Reshaped array ready for statistics computation
|
||||
sample_count: Number of samples represented in the data
|
||||
quantile_list: Quantiles to fill with the mean value. Defaults to `DEFAULT_QUANTILES`.
|
||||
|
||||
Returns:
|
||||
Dictionary with basic statistics and quantiles set to mean values
|
||||
@@ -478,14 +447,13 @@ def get_feature_stats(
|
||||
- Global: axis=None computes statistics over entire array
|
||||
|
||||
Args:
|
||||
array (`np.ndarray`): Input data array with a shape appropriate for the specified `axis`.
|
||||
axis (`int | tuple[int, ...] | None`): Axis or axes along which to compute statistics:
|
||||
`(0, 2, 3)` for image data (batch, channels, height, width), `0` or `(0,)` for
|
||||
vector/tabular data (samples, features), `(1,)` to compute across features, or
|
||||
`None` for global statistics over the entire array.
|
||||
keepdims (`bool`): If `True`, reduced axes are kept as dimensions of size 1.
|
||||
quantile_list (`list[float] | None`, *optional*): Quantiles to compute (e.g. `0.01` for
|
||||
the 1st percentile). Defaults to `DEFAULT_QUANTILES` when not provided.
|
||||
array: Input data array with shape appropriate for the specified axis
|
||||
axis: Axis or axes along which to compute statistics
|
||||
- (0, 2, 3): For image data (batch, channels, height, width)
|
||||
- 0 or (0,): For vector/tabular data (samples, features)
|
||||
- (1,): For computing across features
|
||||
- None: For global statistics over entire array
|
||||
keepdims: If True, reduced axes are kept as dimensions with size 1
|
||||
|
||||
Returns:
|
||||
Dictionary containing:
|
||||
@@ -528,13 +496,10 @@ def compute_episode_stats(
|
||||
- Strings: Skipped (no statistics computed)
|
||||
|
||||
Args:
|
||||
episode_data (`dict[str, list[str] | np.ndarray]`): Mapping from feature name to its data
|
||||
for the episode: a list of file paths for `image`/`video` features, or a numpy array
|
||||
for numerical features.
|
||||
features (`dict`): Dataset feature metadata, keyed by feature name, describing each
|
||||
feature's `dtype` and shape.
|
||||
quantile_list (`list[float] | None`, *optional*): Quantiles to compute (e.g. `0.01` for
|
||||
the 1st percentile). Defaults to `DEFAULT_QUANTILES` when not provided.
|
||||
episode_data: Dictionary mapping feature names to data
|
||||
- For images/videos: list of file paths
|
||||
- For numerical data: numpy arrays
|
||||
features: Dictionary describing each feature's dtype and shape
|
||||
|
||||
Returns:
|
||||
Dictionary mapping feature names to their statistics dictionaries.
|
||||
@@ -665,6 +630,7 @@ def aggregate_stats(stats_list: list[dict[str, dict]]) -> dict[str, dict[str, np
|
||||
- new_mean = (mean of all data, weighted by counts)
|
||||
- new_std = (std of all data)
|
||||
"""
|
||||
|
||||
_assert_type_and_shape(stats_list)
|
||||
|
||||
data_keys = {key for stats in stats_list for key in stats}
|
||||
@@ -724,16 +690,16 @@ def compute_relative_action_stats(
|
||||
statistics suitable for normalization.
|
||||
|
||||
Args:
|
||||
hf_dataset (`datasets.Dataset`): The underlying HuggingFace dataset, must expose
|
||||
`"action"`, `"observation.state"`, and `"episode_index"` columns.
|
||||
features (`dict`): Dataset feature metadata; must contain `"action"` with a `"shape"`
|
||||
entry and optionally `"names"`.
|
||||
chunk_size (`int`): Number of consecutive frames per action chunk.
|
||||
exclude_joints (`list[str] | None`, *optional*): Joint names whose dimensions should
|
||||
remain absolute instead of being converted to relative actions.
|
||||
num_workers (`int`, *optional*, defaults to 0): Number of parallel threads used for
|
||||
computation. Values `<= 1` run single-threaded; NumPy releases the GIL so threads
|
||||
give real parallelism here.
|
||||
hf_dataset: The underlying HuggingFace dataset with "action",
|
||||
"observation.state", and "episode_index" columns.
|
||||
features: Dataset feature metadata (must contain "action" with "shape"
|
||||
and optionally "names").
|
||||
chunk_size: Number of consecutive frames per action chunk.
|
||||
exclude_joints: Joint names whose dimensions should remain absolute
|
||||
(not converted to relative actions).
|
||||
num_workers: Number of parallel threads for computation. Values ≤1
|
||||
mean single-threaded. Numpy releases the GIL so threads give
|
||||
real parallelism here.
|
||||
|
||||
Returns:
|
||||
Statistics dict with keys "mean", "std", "min", "max", "q01", …, "q99".
|
||||
|
||||
@@ -529,9 +529,9 @@ class LeRobotDatasetMetadata:
|
||||
return self.info.video_files_size_in_mb
|
||||
|
||||
def get_task_index(self, task: str) -> int | None:
|
||||
"""Given a task in natural language, returns its task_index if the task already exists in the dataset.
|
||||
|
||||
Otherwise return None.
|
||||
"""
|
||||
Given a task in natural language, returns its task_index if the task already exists in the dataset,
|
||||
otherwise return None.
|
||||
"""
|
||||
if task in self.tasks.index:
|
||||
return int(self.tasks.loc[task].task_index)
|
||||
@@ -774,7 +774,6 @@ class LeRobotDatasetMetadata:
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
"""A short summary: repo ID, total episode/frame counts, and feature keys."""
|
||||
feature_keys = list(self.features)
|
||||
return (
|
||||
f"{self.__class__.__name__}({{\n"
|
||||
|
||||
@@ -293,9 +293,7 @@ class DatasetReader:
|
||||
return result
|
||||
|
||||
def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]:
|
||||
"""Decode the requested per-camera frame timestamps from `ep_idx`'s videos.
|
||||
|
||||
Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
|
||||
"""Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
|
||||
in the main process (e.g. by using a second Dataloader with num_workers=0). It will result in a
|
||||
Segmentation Fault.
|
||||
"""
|
||||
|
||||
@@ -118,11 +118,10 @@ def delete_episodes(
|
||||
consistent with its own metadata.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobotDataset.
|
||||
episode_indices (`list`): List of episode indices to delete.
|
||||
output_dir (`str | pathlib.Path | None`, *optional*): Root directory where the edited dataset
|
||||
will be stored. If not specified, defaults to `$HF_LEROBOT_HOME/repo_id`.
|
||||
repo_id (`str | None`, *optional*): Identifier for the edited dataset.
|
||||
dataset: The source LeRobotDataset.
|
||||
episode_indices: List of episode indices to delete.
|
||||
output_dir: Root directory where the edited dataset will be stored. If not specified, defaults to $HF_LEROBOT_HOME/repo_id. Equivalent to new_root in EditDatasetConfig.
|
||||
repo_id: Edited dataset identifier. Equivalent to new_repo_id in EditDatasetConfig.
|
||||
"""
|
||||
if not episode_indices:
|
||||
raise ValueError("No episodes to delete")
|
||||
@@ -186,11 +185,10 @@ def split_dataset(
|
||||
output split stays consistent with its own metadata.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobotDataset to split.
|
||||
splits (`dict`): Either a dict mapping split names to episode indices, or a dict mapping
|
||||
split names to fractions (must sum to <= 1.0).
|
||||
output_dir (`str | pathlib.Path | None`, *optional*): Root directory where the split
|
||||
datasets will be stored. If not specified, defaults to `$HF_LEROBOT_HOME/repo_id`.
|
||||
dataset: The source LeRobotDataset to split.
|
||||
splits: Either a dict mapping split names to episode indices, or a dict mapping
|
||||
split names to fractions (must sum to <= 1.0).
|
||||
output_dir: Root directory where the split datasets will be stored. If not specified, defaults to $HF_LEROBOT_HOME/repo_id.
|
||||
|
||||
Examples:
|
||||
Split by specific episodes
|
||||
@@ -282,14 +280,11 @@ def merge_datasets(
|
||||
This is a wrapper around the aggregate_datasets functionality with a cleaner API.
|
||||
|
||||
Args:
|
||||
datasets (`list`): List of LeRobotDatasets to merge.
|
||||
output_repo_id (`str`): Identifier for the merged dataset.
|
||||
output_dir (`str | pathlib.Path | None`, *optional*): Root directory where the merged dataset
|
||||
will be stored. If not specified, defaults to `$HF_LEROBOT_HOME/output_repo_id`.
|
||||
concatenate_videos (`bool`, *optional*, defaults to `True`): When `False`, keep one mp4 per
|
||||
source file instead of packing them into shards.
|
||||
concatenate_data (`bool`, *optional*, defaults to `True`): When `False`, keep one parquet file
|
||||
per source file instead of packing them into shards.
|
||||
datasets: List of LeRobotDatasets to merge.
|
||||
output_repo_id: Merged dataset identifier.
|
||||
output_dir: Root directory where the merged dataset will be stored. If not specified, defaults to $HF_LEROBOT_HOME/output_repo_id.
|
||||
concatenate_videos: When False, keep one mp4 per source file instead of packing into shards.
|
||||
concatenate_data: When False, keep one parquet per source file instead of packing into shards.
|
||||
"""
|
||||
if not datasets:
|
||||
raise ValueError("No datasets to merge")
|
||||
@@ -332,14 +327,11 @@ def modify_features(
|
||||
regardless of how many features are being added or removed.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobotDataset.
|
||||
add_features (`dict[str, tuple[numpy.ndarray | torch.Tensor | collections.abc.Callable, dict]] | None`, *optional*):
|
||||
Dict mapping feature names to `(feature_values, feature_info)` tuples.
|
||||
remove_features (`str | list[str] | None`, *optional*): Feature name(s) to remove. Can be a
|
||||
single string or a list.
|
||||
output_dir (`str | pathlib.Path | None`, *optional*): Root directory where the edited dataset
|
||||
will be stored. If not specified, defaults to `$HF_LEROBOT_HOME/repo_id`.
|
||||
repo_id (`str | None`, *optional*): Identifier for the edited dataset.
|
||||
dataset: The source LeRobotDataset.
|
||||
add_features: Optional dict mapping feature names to (feature_values, feature_info) tuples.
|
||||
remove_features: Optional feature name(s) to remove. Can be a single string or list.
|
||||
output_dir: Root directory where the edited dataset will be stored. If not specified, defaults to $HF_LEROBOT_HOME/repo_id. Equivalent to new_root in EditDatasetConfig.
|
||||
repo_id: Edited dataset identifier. Equivalent to new_repo_id in EditDatasetConfig.
|
||||
|
||||
Returns:
|
||||
New dataset with features modified.
|
||||
@@ -438,11 +430,10 @@ def add_features(
|
||||
copies the dataset once regardless of how many features are being added.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobotDataset.
|
||||
features (`dict`): Dictionary mapping feature names to `(feature_values, feature_info)` tuples.
|
||||
output_dir (`str | pathlib.Path | None`, *optional*): Root directory where the edited dataset
|
||||
will be stored. If not specified, defaults to `$HF_LEROBOT_HOME/repo_id`.
|
||||
repo_id (`str | None`, *optional*): Identifier for the edited dataset.
|
||||
dataset: The source LeRobotDataset.
|
||||
features: Dictionary mapping feature names to (feature_values, feature_info) tuples.
|
||||
output_dir: Root directory where the edited dataset will be stored. If not specified, defaults to $HF_LEROBOT_HOME/repo_id. Equivalent to new_root in EditDatasetConfig.
|
||||
repo_id: Edited dataset identifier. Equivalent to new_repo_id in EditDatasetConfig.
|
||||
|
||||
Returns:
|
||||
New dataset with all features added.
|
||||
@@ -476,12 +467,10 @@ def remove_feature(
|
||||
"""Remove features from a LeRobotDataset.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobotDataset.
|
||||
feature_names (`str | list[str]`): Name(s) of features to remove. Can be a single string or
|
||||
a list.
|
||||
output_dir (`str | pathlib.Path | None`, *optional*): Root directory where the edited dataset
|
||||
will be stored. If not specified, defaults to `$HF_LEROBOT_HOME/repo_id`.
|
||||
repo_id (`str | None`, *optional*): Identifier for the edited dataset.
|
||||
dataset: The source LeRobotDataset.
|
||||
feature_names: Name(s) of features to remove. Can be a single string or list.
|
||||
output_dir: Root directory where the edited dataset will be stored. If not specified, defaults to $HF_LEROBOT_HOME/repo_id. Equivalent to new_root in EditDatasetConfig.
|
||||
repo_id: Edited dataset identifier. Equivalent to new_repo_id in EditDatasetConfig.
|
||||
|
||||
Returns:
|
||||
New dataset with features removed.
|
||||
@@ -960,7 +949,7 @@ def _copy_and_reindex_episodes_metadata(
|
||||
|
||||
|
||||
def _write_parquet(df: pd.DataFrame, path: Path, meta: LeRobotDatasetMetadata) -> None:
|
||||
"""Write DataFrame to parquet.
|
||||
"""Write DataFrame to parquet
|
||||
|
||||
This ensures images are properly embedded and the file can be loaded correctly by HF datasets.
|
||||
"""
|
||||
@@ -1468,14 +1457,13 @@ def modify_tasks(
|
||||
- meta/info.json (total_tasks)
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobotDataset to modify.
|
||||
new_task (`str | None`, *optional*): Default task applied to any episode not covered by
|
||||
`episode_tasks` or a matching `task_replacements` entry.
|
||||
episode_tasks (`dict[int, str] | None`, *optional*): Dict mapping episode indices to task
|
||||
strings. Takes precedence over both `task_replacements` and `new_task`.
|
||||
task_replacements (`dict[str, str] | None`, *optional*): Dict mapping existing task strings to
|
||||
new ones. Applied to episodes whose current task matches a key. Every key must be an
|
||||
existing task.
|
||||
dataset: The source LeRobotDataset to modify.
|
||||
new_task: Default task applied to any episode not covered by `episode_tasks` or a
|
||||
matching `task_replacements` entry.
|
||||
episode_tasks: Optional dict mapping episode indices to task strings. Takes precedence
|
||||
over both `task_replacements` and `new_task`.
|
||||
task_replacements: Optional dict mapping existing task strings to new ones. Applied to
|
||||
episodes whose current task matches a key. Every key must be an existing task.
|
||||
|
||||
At least one of `new_task`, `episode_tasks`, or `task_replacements` must be provided.
|
||||
|
||||
@@ -1606,19 +1594,19 @@ def recompute_stats(
|
||||
"""Recompute stats.json from scratch by iterating all episodes.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The LeRobotDataset to recompute stats for.
|
||||
skip_image_video (`bool`, *optional*, defaults to `True`): If `True`, only recompute stats for
|
||||
numeric features (action, state, etc.) and keep existing image/video stats unchanged.
|
||||
relative_action (`bool`, *optional*, defaults to `False`): If `True`, compute action stats in
|
||||
relative space by iterating all valid action chunks and subtracting the current state.
|
||||
This matches the normalization distribution the model sees during training with
|
||||
`use_relative_actions=True`.
|
||||
relative_exclude_joints (`list[str] | None`, *optional*): Joint names to exclude from relative
|
||||
conversion when `relative_action=True`. These dims keep absolute stats.
|
||||
chunk_size (`int`, *optional*, defaults to 50): Action chunk size used for relative stats
|
||||
computation. Should match `policy.chunk_size`. Only used when `relative_action=True`.
|
||||
num_workers (`int`, *optional*, defaults to 0): Number of parallel threads for relative action
|
||||
stats computation. Values <=1 mean single-threaded. Only used when `relative_action=True`.
|
||||
dataset: The LeRobotDataset to recompute stats for.
|
||||
skip_image_video: If True (default), only recompute stats for numeric features
|
||||
(action, state, etc.) and keep existing image/video stats unchanged.
|
||||
relative_action: If True, compute action stats in relative space by
|
||||
iterating all valid action chunks and subtracting the current state.
|
||||
This matches the normalization distribution the model sees during
|
||||
training with ``use_relative_actions=True``.
|
||||
relative_exclude_joints: Joint names to exclude from relative conversion when
|
||||
relative_action=True. These dims keep absolute stats.
|
||||
chunk_size: Action chunk size used for relative stats computation. Should match
|
||||
``policy.chunk_size``. Only used when ``relative_action=True``.
|
||||
num_workers: Number of parallel threads for relative action stats computation.
|
||||
Values ≤1 mean single-threaded. Only used when ``relative_action=True``.
|
||||
|
||||
Returns:
|
||||
The same dataset with updated stats.
|
||||
@@ -1721,22 +1709,24 @@ def convert_image_to_video_dataset(
|
||||
LeRobot dataset structure with videos stored in chunked MP4 files.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The source LeRobot dataset with images.
|
||||
output_dir (`pathlib.Path | None`, *optional*): Root directory where the converted dataset will
|
||||
be stored. When `None`, defaults to `$HF_LEROBOT_HOME/repo_id`.
|
||||
repo_id (`str | None`, *optional*): Identifier for the converted dataset.
|
||||
rgb_encoder (`lerobot.configs.video.RGBEncoderConfig | None`, *optional*): Video encoder settings
|
||||
applied to RGB cameras. When `None`, `rgb_encoder_defaults` is used.
|
||||
depth_encoder (`lerobot.configs.video.DepthEncoderConfig | None`, *optional*): Video encoder
|
||||
settings applied to depth-map cameras, including the quantization parameters persisted to
|
||||
the dataset metadata. When `None`, `depth_encoder_defaults` is used.
|
||||
episode_indices (`list[int] | None`, *optional*): Episode indices to convert. When `None`, all
|
||||
episodes are converted.
|
||||
num_workers (`int`, *optional*, defaults to 4): Number of threads for parallel processing.
|
||||
max_episodes_per_batch (`int | None`, *optional*): Maximum episodes per video batch, to bound
|
||||
memory use. `None` means no limit.
|
||||
max_frames_per_batch (`int | None`, *optional*): Maximum frames per video batch, to bound memory
|
||||
use. `None` means no limit.
|
||||
dataset: The source LeRobot dataset with images.
|
||||
output_dir: Root directory where the converted dataset will be stored. When
|
||||
``None``, defaults to ``$HF_LEROBOT_HOME/repo_id``. Equivalent to
|
||||
``new_root`` in ``EditDatasetConfig``.
|
||||
repo_id: Converted dataset identifier. Equivalent to ``new_repo_id`` in
|
||||
``EditDatasetConfig``.
|
||||
rgb_encoder: Video encoder settings applied to RGB cameras. When ``None``,
|
||||
:func:`~lerobot.configs.video.rgb_encoder_defaults` is used.
|
||||
depth_encoder: Video encoder settings applied to depth-map cameras, including
|
||||
the quantization parameters persisted to the dataset metadata. When
|
||||
``None``, :func:`~lerobot.configs.video.depth_encoder_defaults` is used.
|
||||
episode_indices: Episode indices to convert. When ``None``, all episodes are
|
||||
converted.
|
||||
num_workers: Number of threads for parallel processing.
|
||||
max_episodes_per_batch: Maximum episodes per video batch, to bound memory use.
|
||||
``None`` means no limit.
|
||||
max_frames_per_batch: Maximum frames per video batch, to bound memory use.
|
||||
``None`` means no limit.
|
||||
|
||||
Returns:
|
||||
A new :class:`LeRobotDataset` with images encoded as videos.
|
||||
@@ -1976,17 +1966,18 @@ def reencode_dataset(
|
||||
Videos are re-encoded in-place and the video information in ``info.json`` is refreshed.
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): An existing :class:`LeRobotDataset` whose videos will be re-encoded.
|
||||
rgb_encoder (`lerobot.configs.video.RGBEncoderConfig | None`, *optional*): Target encoder
|
||||
configuration applied to every RGB video file. If `None`, re-encoding is skipped for RGB
|
||||
videos.
|
||||
depth_encoder (`lerobot.configs.video.DepthEncoderConfig | None`, *optional*): Target encoder
|
||||
configuration applied to every depth video file. If `None`, re-encoding is skipped for depth
|
||||
videos. Quantization parameters will not override the ones in the current dataset.
|
||||
encoder_threads (`int | None`, *optional*): Per-encoder thread count forwarded to
|
||||
`reencode_video`. `None` lets the codec decide.
|
||||
num_workers (`int | None`, *optional*): Number of parallel processes. `None` or `0` means
|
||||
sequential (no multiprocessing); `1+` spawns a `ProcessPoolExecutor`.
|
||||
dataset: An existing :class:`LeRobotDataset` whose videos will be
|
||||
re-encoded.
|
||||
rgb_encoder: Target encoder configuration applied to every RGB video
|
||||
file. If ``None``, re-encoding is skipped for RGB videos.
|
||||
depth_encoder: Target encoder configuration applied to every depth video
|
||||
file. If ``None``, re-encoding is skipped for depth videos.
|
||||
Quantization parameters will not override the ones in the current dataset.
|
||||
encoder_threads: Per-encoder thread count forwarded to
|
||||
:func:`reencode_video`. ``None`` lets the codec decide.
|
||||
num_workers: Number of parallel processes. ``None`` or ``0`` means
|
||||
sequential (no multiprocessing); ``1+`` spawns a
|
||||
:class:`~concurrent.futures.ProcessPoolExecutor`.
|
||||
|
||||
Returns:
|
||||
The same :class:`LeRobotDataset` instance with its metadata updated
|
||||
|
||||
@@ -200,7 +200,8 @@ class DatasetWriter:
|
||||
self.image_writer.save_image(image=image, fpath=fpath, compress_level=compress_level)
|
||||
|
||||
def add_frame(self, frame: dict) -> None:
|
||||
"""Add a single frame to the current episode buffer.
|
||||
"""
|
||||
Add a single frame to the current episode buffer.
|
||||
|
||||
Apart from images written to a temporary directory, nothing is written to disk
|
||||
until ``save_episode()`` is called.
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Depth encoding/decoding helpers for :class:`DepthEncoderConfig`."""
|
||||
"""
|
||||
Depth encoding/decoding helpers for :class:`DepthEncoderConfig`.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Literal
|
||||
@@ -90,24 +92,13 @@ def quantize_depth(
|
||||
``depth_min``, ``depth_max``, and ``shift`` are always in **metres**.
|
||||
|
||||
Args:
|
||||
depth (`numpy.ndarray[tuple[int, ...], numpy.dtype[numpy.uint16]] | numpy.ndarray[tuple[int, ...], numpy.dtype[numpy.float32]] | torch.Tensor`): Depth
|
||||
map to quantize. A `torch.Tensor` is moved to CPU before conversion.
|
||||
depth_min (`float`, *optional*, defaults to 0.01): Depth, in metres, mapped to quantum
|
||||
`0`.
|
||||
depth_max (`float`, *optional*, defaults to 10.0): Depth, in metres, mapped to quantum
|
||||
`DEPTH_QMAX`.
|
||||
shift (`float`, *optional*, defaults to 3.5): Depth shift, in metres, used in log mode.
|
||||
Must satisfy `depth_min + shift > 0`.
|
||||
use_log (`bool`, *optional*, defaults to `True`): If `True`, quantize in log space, which
|
||||
allocates more quanta to near-range depth.
|
||||
pix_fmt (`str`, *optional*, defaults to `"gray12le"`): Pixel format used to build the
|
||||
`av.VideoFrame` when `video_backend="pyav"`.
|
||||
video_backend (`str | None`, *optional*, defaults to `"pyav"`): Video backend used for
|
||||
encoding. When `"pyav"`, returns an `av.VideoFrame`; otherwise returns the raw
|
||||
`uint16` array.
|
||||
input_unit (`Literal`, *optional*, defaults to `"auto"`): Input unit policy: `"auto"`
|
||||
infers the unit from `depth`'s dtype, while `"mm"` or `"m"` force millimetres or
|
||||
metres respectively.
|
||||
depth: Depth map; ``torch.Tensor`` is moved to CPU for conversion.
|
||||
depth_min: Depth (metres) at quantum ``0``.
|
||||
depth_max: Depth (metres) at quantum :data:`DEPTH_QMAX`.
|
||||
shift: Depth shift (metres); used in log mode. Must satisfy ``depth_min + shift > 0``.
|
||||
use_log: If ``True`` (default), quantize in log space.
|
||||
video_backend: Video backend to use for encoding. Defaults to "pyav".
|
||||
input_unit: Input unit policy (``"auto"``, ``"mm"``, ``"m"``).
|
||||
|
||||
Returns:
|
||||
``numpy.ndarray``, ``dtype=uint16``, same shape as ``depth``, values in
|
||||
@@ -183,28 +174,15 @@ def dequantize_depth(
|
||||
Output layout is determined by ``output_channel_last``.
|
||||
|
||||
Args:
|
||||
quantized (`numpy.ndarray[tuple[int, ...], numpy.dtype[numpy.uint16]] | av.video.frame.VideoFrame | torch.Tensor`): 12-bit
|
||||
codes in `[0, DEPTH_QMAX]`, as a numpy array, `av.VideoFrame`, or `torch.Tensor`
|
||||
(any integer or float dtype).
|
||||
depth_min (`float`, *optional*, defaults to 0.01): Depth, in metres, mapped to quantum
|
||||
`0`. Must match the value passed to `quantize_depth`.
|
||||
depth_max (`float`, *optional*, defaults to 10.0): Depth, in metres, mapped to quantum
|
||||
`DEPTH_QMAX`. Must match the value passed to `quantize_depth`.
|
||||
shift (`float`, *optional*, defaults to 3.5): Depth shift, in metres, used in log mode.
|
||||
Must match the value passed to `quantize_depth`.
|
||||
use_log (`bool`, *optional*, defaults to `True`): If `True`, invert the log-space mapping
|
||||
used by `quantize_depth`. Must match the encoding call.
|
||||
pix_fmt (`str`, *optional*, defaults to `"gray12le"`): Pixel format used to extract the
|
||||
plane data when `quantized` is an `av.VideoFrame`.
|
||||
output_unit (`Literal`, *optional*, defaults to `"mm"`): `"mm"` returns `uint16`
|
||||
millimetres, clipped to `[0, 65535]`, when returning a numpy array, or `float32`
|
||||
millimetres when `output_tensor=True`. `"m"` returns `float32` metres in
|
||||
`[depth_min, depth_max]`.
|
||||
output_tensor (`bool`, *optional*, defaults to `True`): If `True`, return a
|
||||
`torch.Tensor` instead of a numpy array.
|
||||
output_channel_last (`bool`, *optional*, defaults to `False`): If `True`, add the
|
||||
restored singleton channel dimension as the last axis instead of the third-to-last
|
||||
axis.
|
||||
quantized: 12-bit codes in ``[0, DEPTH_QMAX]``. ``np.ndarray``,
|
||||
``av.VideoFrame``, or ``torch.Tensor`` (any integer or float dtype).
|
||||
depth_min, depth_max, shift, use_log: Same as :func:`quantize_depth` (metres).
|
||||
pix_fmt: Pixel format used to extract the plane from an ``av.VideoFrame``.
|
||||
output_unit: ``"mm"`` returns ``uint16`` millimetres (rint, clip
|
||||
``[0, 65535]``) when returning a numpy array, or ``float32`` mm when
|
||||
``output_tensor=True``. ``"m"`` returns ``float32`` metres in
|
||||
``[depth_min, depth_max]``.
|
||||
output_tensor: If True, return a ``torch.Tensor`` instead of a numpy array.
|
||||
|
||||
Returns:
|
||||
Depth map in the requested unit and dtype.
|
||||
|
||||
@@ -101,10 +101,10 @@ def create_empty_dataset_info(
|
||||
fps (int): The frames per second of the data.
|
||||
features (dict): The LeRobot features dictionary for the dataset.
|
||||
use_videos (bool): Whether the dataset will store videos.
|
||||
robot_type (str | None, *optional*): The type of robot used, if any.
|
||||
chunks_size (int | None, *optional*): Max files per chunk directory. Defaults to ``DEFAULT_CHUNK_SIZE``.
|
||||
data_files_size_in_mb (int | None, *optional*): Max parquet file size in MB. Defaults to ``DEFAULT_DATA_FILE_SIZE_IN_MB``.
|
||||
video_files_size_in_mb (int | None, *optional*): Max video file size in MB. Defaults to ``DEFAULT_VIDEO_FILE_SIZE_IN_MB``.
|
||||
robot_type (str | None): The type of robot used, if any.
|
||||
chunks_size (int | None): Max files per chunk directory. Defaults to ``DEFAULT_CHUNK_SIZE``.
|
||||
data_files_size_in_mb (int | None): Max parquet file size in MB. Defaults to ``DEFAULT_DATA_FILE_SIZE_IN_MB``.
|
||||
video_files_size_in_mb (int | None): Max video file size in MB. Defaults to ``DEFAULT_VIDEO_FILE_SIZE_IN_MB``.
|
||||
|
||||
Returns:
|
||||
DatasetInfo: A typed dataset information object with initial metadata.
|
||||
@@ -170,7 +170,7 @@ def check_delta_timestamps(
|
||||
deltas in seconds.
|
||||
fps (int): The frames per second of the dataset.
|
||||
tolerance_s (float): The allowed tolerance in seconds.
|
||||
raise_value_error (bool, *optional*, defaults to `True`): If True, raises an error on failure.
|
||||
raise_value_error (bool): If True, raises an error on failure.
|
||||
|
||||
Returns:
|
||||
bool: True if all deltas are valid, False otherwise.
|
||||
@@ -219,18 +219,6 @@ def get_delta_indices(delta_timestamps: dict[str, list[float]], fps: int) -> dic
|
||||
|
||||
|
||||
def validate_frame(frame: dict, features: dict) -> None:
|
||||
"""Check that `frame` has a `"task"` key and matches `features` (minus auto-populated defaults).
|
||||
|
||||
Args:
|
||||
frame (`dict`): The frame to validate, mapping feature names to their values, as passed by the
|
||||
caller to `add_frame`.
|
||||
features (`dict`): The dataset's feature specification, mapping feature names to their dtype and
|
||||
shape metadata.
|
||||
|
||||
Raises:
|
||||
ValueError: If `frame` is missing `"task"`, or has missing/extra features, or a feature's dtype
|
||||
or shape doesn't match its definition in `features`.
|
||||
"""
|
||||
# DEFAULT_FEATURES (timestamp, frame_index, episode_index, index, task_index) are
|
||||
# auto-populated by the recording pipeline (add_frame / save_episode) and must not
|
||||
# be supplied by the caller. Excluding them here means any frame dict that contains
|
||||
@@ -287,7 +275,7 @@ def validate_feature_dtype_and_shape(
|
||||
Args:
|
||||
name (str): The name of the feature.
|
||||
feature (dict): The feature specification from the LeRobot features dictionary.
|
||||
value (`numpy.ndarray | PIL.Image.Image | str`): The value of the feature to validate.
|
||||
value: The value of the feature to validate.
|
||||
|
||||
Returns:
|
||||
str: An error message if validation fails, otherwise an empty string.
|
||||
@@ -349,7 +337,7 @@ def validate_feature_image_or_video(
|
||||
Args:
|
||||
name (str): The name of the feature.
|
||||
expected_shape (list[str]): The expected shape, e.g. (C, H, W) or (H, W, C).
|
||||
value (`numpy.ndarray | PIL.Image.Image`): The image or video frame data to validate.
|
||||
value: The image data to validate.
|
||||
|
||||
Returns:
|
||||
str: An error message if validation fails, otherwise an empty string.
|
||||
@@ -395,8 +383,7 @@ def validate_feature_language(name: str, value) -> str:
|
||||
|
||||
Args:
|
||||
name (str): The name of the feature.
|
||||
value (`Any`): The value supplied for the language feature. Only checked for being `None`; any
|
||||
other value is dropped with a warning.
|
||||
value: The value to validate.
|
||||
|
||||
Returns:
|
||||
str: Always an empty string — language values are non-fatal.
|
||||
|
||||
@@ -27,10 +27,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_stop_image_writer(func):
|
||||
"""Decorator: on an exception from `func`, stop the `dataset` kwarg's image writer before re-raising."""
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
"""Call `func`; on any exception, stop `kwargs["dataset"].writer.image_writer` before re-raising."""
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except BaseException:
|
||||
@@ -129,7 +126,8 @@ def save_kwargs_for_path(fpath: Path, compress_level: int) -> dict:
|
||||
|
||||
|
||||
def write_image(image: np.ndarray | PIL.Image.Image, fpath: Path, compress_level: int = 1):
|
||||
"""Saves a NumPy array or PIL Image to a file.
|
||||
"""
|
||||
Saves a NumPy array or PIL Image to a file.
|
||||
|
||||
This function handles both NumPy arrays and PIL Image objects, converting
|
||||
the former to a PIL Image before saving. It includes error handling for
|
||||
@@ -140,7 +138,7 @@ def write_image(image: np.ndarray | PIL.Image.Image, fpath: Path, compress_level
|
||||
Args:
|
||||
image (np.ndarray | PIL.Image.Image): The image data to save.
|
||||
fpath (Path): The destination file path for the image.
|
||||
compress_level (int, optional, *optional*, defaults to 1): The compression level for the saved
|
||||
compress_level (int, optional): The compression level for the saved
|
||||
image, as used by PIL.Image.save(). Defaults to 1.
|
||||
Refer to: https://github.com/huggingface/lerobot/pull/2135
|
||||
for more details on the default value rationale.
|
||||
@@ -165,7 +163,6 @@ def write_image(image: np.ndarray | PIL.Image.Image, fpath: Path, compress_level
|
||||
|
||||
|
||||
def worker_thread_loop(queue: queue.Queue):
|
||||
"""Pop `(image_array, fpath, compress_level)` items from `queue` and write each until a `None` sentinel."""
|
||||
while True:
|
||||
item = queue.get()
|
||||
if item is None:
|
||||
@@ -177,7 +174,6 @@ def worker_thread_loop(queue: queue.Queue):
|
||||
|
||||
|
||||
def worker_process(queue: queue.Queue, num_threads: int):
|
||||
"""Run `num_threads` `worker_thread_loop` threads against `queue` and block until they all exit."""
|
||||
threads = []
|
||||
for _ in range(num_threads):
|
||||
t = threading.Thread(target=worker_thread_loop, args=(queue,))
|
||||
@@ -189,9 +185,9 @@ def worker_process(queue: queue.Queue, num_threads: int):
|
||||
|
||||
|
||||
class AsyncImageWriter:
|
||||
"""This class abstracts away the initialisation of processes or/and threads.
|
||||
|
||||
It saves images on disk asynchronously, which is critical to control a robot and record data
|
||||
"""
|
||||
This class abstract away the initialisation of processes or/and threads to
|
||||
save images on disk asynchronously, which is critical to control a robot and record data
|
||||
at a high frame rate.
|
||||
|
||||
When `num_processes=0`, it creates a threads pool of size `num_threads`.
|
||||
@@ -204,15 +200,6 @@ class AsyncImageWriter:
|
||||
"""
|
||||
|
||||
def __init__(self, num_processes: int = 0, num_threads: int = 1):
|
||||
"""Start the thread or process pool.
|
||||
|
||||
Args:
|
||||
num_processes: Number of worker subprocesses. `0` uses threads only (in this process).
|
||||
num_threads: Number of writer threads per process (or in this process, if `num_processes=0`).
|
||||
|
||||
Raises:
|
||||
ValueError: If both `num_threads` and `num_processes` are non-positive.
|
||||
"""
|
||||
self.num_processes = num_processes
|
||||
self.num_threads = num_threads
|
||||
self.queue = None
|
||||
@@ -243,18 +230,15 @@ class AsyncImageWriter:
|
||||
def save_image(
|
||||
self, image: torch.Tensor | np.ndarray | PIL.Image.Image, fpath: Path, compress_level: int = 1
|
||||
):
|
||||
"""Enqueue `image` to be written to `fpath` asynchronously; returns immediately."""
|
||||
if isinstance(image, torch.Tensor):
|
||||
# Convert tensor to numpy array to minimize main process time
|
||||
image = image.cpu().numpy()
|
||||
self.queue.put((image, fpath, compress_level))
|
||||
|
||||
def wait_until_done(self):
|
||||
"""Block until every enqueued image has been written to disk."""
|
||||
self.queue.join()
|
||||
|
||||
def stop(self):
|
||||
"""Signal all worker threads/processes to exit and wait for them to join. No-op if already stopped."""
|
||||
if self._stopped:
|
||||
return
|
||||
|
||||
|
||||
@@ -46,7 +46,6 @@ from .utils import (
|
||||
|
||||
|
||||
def get_parquet_file_size_in_mb(parquet_path: str | Path) -> float:
|
||||
"""Return the uncompressed size, in megabytes, of a parquet file's column data (from its metadata)."""
|
||||
metadata = pq.read_metadata(parquet_path)
|
||||
total_uncompressed_size = 0
|
||||
for row_group in range(metadata.num_row_groups):
|
||||
@@ -58,24 +57,20 @@ def get_parquet_file_size_in_mb(parquet_path: str | Path) -> float:
|
||||
|
||||
|
||||
def get_hf_dataset_size_in_mb(hf_ds: Dataset) -> int:
|
||||
"""Return the in-memory (Arrow buffer) size of a Hugging Face `Dataset`, in megabytes."""
|
||||
return hf_ds.data.nbytes // (1024**2)
|
||||
|
||||
|
||||
def load_nested_dataset(
|
||||
pq_dir: Path, features: datasets.Features | None = None, episodes: list[int] | None = None
|
||||
) -> Dataset:
|
||||
"""Find parquet files in provided directory {pq_dir}/chunk-xxx/file-xxx.parquet.
|
||||
|
||||
Convert parquet files to pyarrow memory mapped in a cache folder for efficient RAM usage, then
|
||||
concatenate all pyarrow references to return HF Dataset format.
|
||||
"""Find parquet files in provided directory {pq_dir}/chunk-xxx/file-xxx.parquet
|
||||
Convert parquet files to pyarrow memory mapped in a cache folder for efficient RAM usage
|
||||
Concatenate all pyarrow references to return HF Dataset format
|
||||
|
||||
Args:
|
||||
pq_dir (`Path`): Directory containing parquet files.
|
||||
features (`datasets.features.features.Features | None`, *optional*): Features schema used to ensure
|
||||
consistent loading of complex types like images.
|
||||
episodes (`list[int] | None`, *optional*): List of episode indices to filter. Uses PyArrow
|
||||
predicate pushdown for efficiency.
|
||||
pq_dir: Directory containing parquet files
|
||||
features: Optional features schema to ensure consistent loading of complex types like images
|
||||
episodes: Optional list of episode indices to filter. Uses PyArrow predicate pushdown for efficiency.
|
||||
"""
|
||||
paths = sorted(pq_dir.glob("*/*.parquet"))
|
||||
if len(paths) == 0:
|
||||
@@ -88,7 +83,6 @@ def load_nested_dataset(
|
||||
|
||||
|
||||
def get_parquet_num_frames(parquet_path: str | Path) -> int:
|
||||
"""Return the number of rows in a parquet file, read from its metadata (no data is loaded)."""
|
||||
metadata = pq.read_metadata(parquet_path)
|
||||
return metadata.num_rows
|
||||
|
||||
@@ -124,7 +118,6 @@ def embed_images(dataset: datasets.Dataset) -> datasets.Dataset:
|
||||
|
||||
|
||||
def write_info(info: DatasetInfo, local_dir: Path) -> None:
|
||||
"""Write dataset info metadata to its standard file path (the inverse of `load_info`)."""
|
||||
write_json(info.to_dict(), local_dir / INFO_PATH)
|
||||
|
||||
|
||||
@@ -183,14 +176,12 @@ def load_stats(local_dir: Path) -> dict[str, dict[str, np.ndarray]] | None:
|
||||
|
||||
|
||||
def write_tasks(tasks: pandas.DataFrame, local_dir: Path) -> None:
|
||||
"""Write the task-prompt table to its standard parquet file path (the inverse of `load_tasks`)."""
|
||||
path = local_dir / DEFAULT_TASKS_PATH
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tasks.to_parquet(path)
|
||||
|
||||
|
||||
def load_tasks(local_dir: Path) -> pandas.DataFrame:
|
||||
"""Load the task-prompt table from its standard file path, indexed by task string."""
|
||||
tasks = pd.read_parquet(local_dir / DEFAULT_TASKS_PATH)
|
||||
tasks.index.name = "task"
|
||||
return tasks
|
||||
@@ -198,13 +189,12 @@ def load_tasks(local_dir: Path) -> pandas.DataFrame:
|
||||
|
||||
def write_episodes(episodes: Dataset, local_dir: Path) -> None:
|
||||
"""Write episode metadata to a parquet file in the LeRobot v3.0 format.
|
||||
|
||||
This function writes episode-level metadata to a single parquet file.
|
||||
Used primarily during dataset conversion (v2.1 → v3.0) and in test fixtures.
|
||||
|
||||
Args:
|
||||
episodes (`Dataset`): Hugging Face `Dataset` containing the episode metadata.
|
||||
local_dir (`Path`): Root directory where the dataset is stored.
|
||||
episodes: HuggingFace Dataset containing episode metadata
|
||||
local_dir: Root directory where the dataset will be stored
|
||||
"""
|
||||
episode_size_mb = get_hf_dataset_size_in_mb(episodes)
|
||||
if episode_size_mb > DEFAULT_DATA_FILE_SIZE_IN_MB:
|
||||
@@ -220,7 +210,6 @@ def write_episodes(episodes: Dataset, local_dir: Path) -> None:
|
||||
|
||||
|
||||
def load_episodes(local_dir: Path) -> datasets.Dataset:
|
||||
"""Load episode metadata, excluding per-episode `stats/*` columns (for faster access to the rest)."""
|
||||
episodes = load_nested_dataset(local_dir / EPISODES_DIR)
|
||||
# Select episode features/columns containing references to episode data and videos
|
||||
# (e.g. tasks, dataset_from_index, dataset_to_index, data/chunk_index, data/file_index, etc.)
|
||||
@@ -236,9 +225,9 @@ def load_image_as_numpy(
|
||||
|
||||
Args:
|
||||
fpath (str | Path): Path to the image file.
|
||||
dtype (np.dtype, *optional*, defaults to `float32`): The desired data type of the output array. If floating,
|
||||
dtype (np.dtype): The desired data type of the output array. If floating,
|
||||
pixels are scaled to [0, 1]. Only used for RGB images.
|
||||
channel_first (bool, *optional*, defaults to `True`): If True, converts the image to (C, H, W) format.
|
||||
channel_first (bool): If True, converts the image to (C, H, W) format.
|
||||
Otherwise, it remains in (H, W, C) format.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -44,14 +44,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LeRobotDataset(torch.utils.data.Dataset):
|
||||
"""A PyTorch `Dataset` over episodic robot data: per-frame state/action tensors, optional videos.
|
||||
|
||||
Backed by parquet files (`data/`) for tabular observation/action/reward data, optional video files
|
||||
(`videos/`) for image observations, and a `meta/` directory holding `info.json` (shapes, keys, fps),
|
||||
`stats.json` (normalization statistics), and per-episode metadata. See `__init__`'s docstring for the
|
||||
on-disk layout, and `create()` for building a new (empty) dataset from scratch.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo_id: str,
|
||||
@@ -76,7 +68,8 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
||||
*,
|
||||
token: str | bool | None = None,
|
||||
):
|
||||
"""2 modes are available for instantiating this class, depending on 2 different use cases.
|
||||
"""
|
||||
2 modes are available for instantiating this class, depending on 2 different use cases:
|
||||
|
||||
1. Your dataset already exists:
|
||||
- On your local disk in the 'root' folder. This is typically the case when you recorded your
|
||||
@@ -175,9 +168,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
||||
conversion. This works for both image-backed and video-backed observations and can later be
|
||||
updated with `set_image_transforms()` or cleared with `clear_image_transforms()`.
|
||||
Defaults to None.
|
||||
delta_timestamps (dict[list[float]] | None, optional): Per-feature timestamp offsets (in
|
||||
seconds, relative to a frame's own timestamp) of additional frames to return alongside it.
|
||||
Defaults to None.
|
||||
delta_timestamps (dict[list[float]] | None, optional): _description_. Defaults to None.
|
||||
tolerance_s (float, optional): Tolerance in seconds used to ensure data timestamps are actually in
|
||||
sync with the fps value. It is used at the init of the dataset to make sure that each
|
||||
timestamps is separated to the next by 1/fps +/- tolerance_s. This also applies to frames
|
||||
@@ -194,11 +185,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
||||
True.
|
||||
video_backend (str | None, optional): Video backend to use for decoding videos. Defaults to torchcodec when available int the platform; otherwise, defaults to 'pyav'.
|
||||
You can also use the 'pyav' decoder used by Torchvision, which used to be the default option, or 'video_reader' which is another decoder of Torchvision.
|
||||
return_uint8 (bool, optional): For RGB videos, whether to return raw uint8 frames instead of
|
||||
the default float32 frames normalized to [0, 1]. Defaults to False.
|
||||
depth_output_unit (str, optional): Physical unit depth maps are dequantized to at load time:
|
||||
"mm" (millimeters) or "m" (metres). Has no effect on datasets without depth cameras.
|
||||
Defaults to "mm".
|
||||
batch_encoding_size (int, optional): Number of episodes to accumulate before batch encoding videos.
|
||||
Set to 1 for immediate encoding (default), or higher for batched encoding. Defaults to 1.
|
||||
rgb_encoder (RGBEncoderConfig | None, optional): Video encoder settings for cameras
|
||||
@@ -409,7 +395,7 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
||||
|
||||
@property
|
||||
def hf_dataset(self) -> datasets.Dataset:
|
||||
"""The underlying Hugging Face Dataset object."""
|
||||
"""The underlying Hugging Face Dataset object"""
|
||||
self.reader = self._ensure_reader()
|
||||
if self.reader.hf_dataset is None:
|
||||
self.reader.load_and_activate()
|
||||
@@ -554,7 +540,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
||||
return self.hf_dataset[idx]
|
||||
|
||||
def __repr__(self):
|
||||
"""A short summary: repo ID, selected episode/sample counts, and feature keys."""
|
||||
feature_keys = list(self.features)
|
||||
return (
|
||||
f"{self.__class__.__name__}({{\n"
|
||||
@@ -753,10 +738,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
||||
during capture instead of writing images first.
|
||||
encoder_queue_maxsize: Max buffered frames per camera when using
|
||||
streaming encoding.
|
||||
video_files_size_in_mb: Max video file size in MB. Defaults to
|
||||
``DEFAULT_VIDEO_FILE_SIZE_IN_MB``.
|
||||
data_files_size_in_mb: Max parquet file size in MB. Defaults to
|
||||
``DEFAULT_DATA_FILE_SIZE_IN_MB``.
|
||||
|
||||
Returns:
|
||||
A new :class:`LeRobotDataset` in write mode.
|
||||
|
||||
@@ -51,20 +51,6 @@ class MultiLeRobotDataset(torch.utils.data.Dataset):
|
||||
*,
|
||||
token: str | bool | None = None,
|
||||
):
|
||||
"""Construct a `LeRobotDataset` for each `repo_id` and concatenate them.
|
||||
|
||||
Args:
|
||||
repo_ids: The Hub repo IDs (or local dataset names, if `root` is set) to load.
|
||||
root: Root directory containing the underlying datasets. Defaults to `$HF_LEROBOT_HOME`.
|
||||
episodes: Optional mapping from `repo_id` to the episode indices to load from it.
|
||||
image_transforms: Transform applied to visual observations in each underlying dataset.
|
||||
delta_timestamps: Passed through to each underlying `LeRobotDataset`.
|
||||
tolerances_s: Optional mapping from `repo_id` to its timestamp tolerance, in seconds. Defaults
|
||||
to `1e-4` for every dataset.
|
||||
download_videos: Whether to download video files for each underlying dataset.
|
||||
video_backend: The video decoding backend to use.
|
||||
token: Hugging Face Hub authentication token.
|
||||
"""
|
||||
super().__init__()
|
||||
self.repo_ids = repo_ids
|
||||
self.root = Path(root) if root else HF_LEROBOT_HOME
|
||||
@@ -154,7 +140,6 @@ class MultiLeRobotDataset(torch.utils.data.Dataset):
|
||||
|
||||
@property
|
||||
def features(self) -> datasets.Features:
|
||||
"""The union of all underlying datasets' features (minus `disabled_features`)."""
|
||||
features = {}
|
||||
for dataset in self._datasets:
|
||||
features.update(
|
||||
@@ -201,26 +186,17 @@ class MultiLeRobotDataset(torch.utils.data.Dataset):
|
||||
|
||||
@property
|
||||
def tolerance_s(self) -> float:
|
||||
"""Tolerance in seconds used to discard loaded frames when their timestamps aren't close enough.
|
||||
|
||||
Only used when `delta_timestamps` is provided or when loading video frames from mp4 files.
|
||||
"""Tolerance in seconds used to discard loaded frames when their timestamps
|
||||
are not close enough from the requested frames. It is only used when `delta_timestamps`
|
||||
is provided or when loading video frames from mp4 files.
|
||||
"""
|
||||
# 1e-4 to account for possible numerical error
|
||||
return 1 / self.fps - 1e-4
|
||||
|
||||
def __len__(self):
|
||||
"""The total number of frames across all underlying datasets."""
|
||||
return self.num_frames
|
||||
|
||||
def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:
|
||||
"""Return the frame at `idx`, resolved to the underlying dataset it falls in.
|
||||
|
||||
Adds a `"dataset_index"` key identifying which underlying dataset the frame came from, and drops
|
||||
any `disabled_features` keys.
|
||||
|
||||
Raises:
|
||||
IndexError: If `idx` is out of bounds.
|
||||
"""
|
||||
if idx >= len(self):
|
||||
raise IndexError(f"Index {idx} out of bounds.")
|
||||
# Determine which dataset to get an item from based on the index.
|
||||
@@ -243,7 +219,6 @@ class MultiLeRobotDataset(torch.utils.data.Dataset):
|
||||
return item
|
||||
|
||||
def __repr__(self):
|
||||
"""A summary: repo IDs, sample/episode counts, media type, fps, camera keys, and transforms."""
|
||||
return (
|
||||
f"{self.__class__.__name__}(\n"
|
||||
f" Repository IDs: '{self.repo_ids}',\n"
|
||||
|
||||
@@ -26,13 +26,12 @@ from lerobot.utils.feature_utils import hw_to_dataset_features
|
||||
def create_initial_features(
|
||||
action: RobotAction | None = None, observation: RobotObservation | None = None
|
||||
) -> dict[PipelineFeatureType, dict[str, Any]]:
|
||||
"""Creates the initial features dict for the dataset from action and observation specs.
|
||||
"""
|
||||
Creates the initial features dict for the dataset from action and observation specs.
|
||||
|
||||
Args:
|
||||
action (`dict[str, typing.Any] | None`, *optional*): A dictionary of action feature names to their
|
||||
types/shapes.
|
||||
observation (`dict[str, typing.Any] | None`, *optional*): A dictionary of observation feature names
|
||||
to their types/shapes.
|
||||
action: A dictionary of action feature names to their types/shapes.
|
||||
observation: A dictionary of observation feature names to their types/shapes.
|
||||
|
||||
Returns:
|
||||
The initial features dictionary structured by PipelineFeatureType.
|
||||
@@ -47,14 +46,12 @@ def create_initial_features(
|
||||
|
||||
# Helper to filter state/action keys based on compiled regex patterns.
|
||||
def should_keep(key: str, patterns: tuple[re.Pattern] | None) -> bool:
|
||||
"""Return `True` if `patterns` is `None` or any pattern in it matches `key`."""
|
||||
if patterns is None:
|
||||
return True
|
||||
return any(pat.search(key) for pat in patterns)
|
||||
|
||||
|
||||
def strip_prefix(key: str, prefixes_to_strip: tuple[str]) -> str:
|
||||
"""Remove the first prefix in `prefixes_to_strip` that `key` starts with, if any."""
|
||||
for prefix in prefixes_to_strip:
|
||||
if key.startswith(prefix):
|
||||
return key[len(prefix) :]
|
||||
@@ -76,22 +73,20 @@ def aggregate_pipeline_dataset_features(
|
||||
exclude_images: bool = False,
|
||||
patterns: Sequence[str] | None = None,
|
||||
) -> dict[str, dict]:
|
||||
"""Aggregates and filters pipeline features to create a dataset-ready features dictionary.
|
||||
"""
|
||||
Aggregates and filters pipeline features to create a dataset-ready features dictionary.
|
||||
|
||||
This function transforms initial features using the pipeline, categorizes them as action or observations
|
||||
(image or state), filters them based on `exclude_images` and `patterns`, and finally
|
||||
formats them for use with a Hugging Face LeRobot Dataset.
|
||||
|
||||
Args:
|
||||
pipeline (`DataProcessorPipeline`): The processor pipeline to apply to `initial_features`.
|
||||
initial_features (`dict`): A dictionary of raw feature specs for actions and observations, keyed by
|
||||
`PipelineFeatureType`.
|
||||
use_videos (`bool`, *optional*, defaults to `True`): Controls the storage dtype for image features.
|
||||
If `True`, images are stored as `"video"`; if `False`, they are stored as `"image"`.
|
||||
exclude_images (`bool`, *optional*, defaults to `False`): If `True`, image features are dropped
|
||||
entirely from the output.
|
||||
patterns (`collections.abc.Sequence[str] | None`, *optional*): A sequence of regex patterns used to
|
||||
filter action and state features.
|
||||
pipeline: The DataProcessorPipeline to apply.
|
||||
initial_features: A dictionary of raw feature specs for actions and observations.
|
||||
use_videos: Controls the storage dtype for image features. If True, images are stored as "video"; if False, they are stored as "image".
|
||||
exclude_images: If True, image features are dropped entirely from the output.
|
||||
patterns: A sequence of regex patterns to filter action and state features.
|
||||
Image features are not affected by this filter.
|
||||
|
||||
Returns:
|
||||
A dictionary of features formatted for a Hugging Face LeRobot Dataset.
|
||||
|
||||
@@ -41,11 +41,10 @@ def write_u16_plane(plane: av.video.plane.VideoPlane, src: np.ndarray, fill_valu
|
||||
leave the padding untouched.
|
||||
|
||||
Args:
|
||||
plane (`VideoPlane`): Destination 16-bit plane to copy into.
|
||||
src (`ndarray`): Source image, shape `(height, width)`, dtype `uint16`.
|
||||
fill_value (`int | None`, *optional*): If given, every pixel of the plane
|
||||
(including the row padding) is set to this value first, so the padding
|
||||
holds clean data instead of garbage.
|
||||
plane: Destination 16-bit plane.
|
||||
src: Source image, shape ``(height, width)``, dtype ``uint16``.
|
||||
fill_value: If given, every pixel (padding included) is set to this first, so the
|
||||
padding holds clean data instead of garbage.
|
||||
"""
|
||||
height, width = src.shape
|
||||
stride_u16 = plane.line_size // np.dtype(np.uint16).itemsize
|
||||
|
||||
@@ -55,8 +55,7 @@ class EpisodeAwareSampler:
|
||||
seed: int = 0,
|
||||
absolute_to_relative_idx: dict[int, int] | None = None,
|
||||
):
|
||||
"""Build the sampler from per-episode `[from, to)` frame-index boundaries.
|
||||
|
||||
"""
|
||||
Args:
|
||||
dataset_from_indices: Start index of each episode in the dataset.
|
||||
dataset_to_indices: End index of each episode in the dataset.
|
||||
@@ -65,13 +64,6 @@ class EpisodeAwareSampler:
|
||||
drop_n_last_frames: Frames to drop from the end of each episode.
|
||||
shuffle: Whether to shuffle the indices.
|
||||
seed: Seed the permutation is derived from (together with the epoch).
|
||||
absolute_to_relative_idx: Optional mapping from absolute dataset frame index to the relative
|
||||
index actually yielded (e.g. when the sampler is used over a filtered subset).
|
||||
|
||||
Raises:
|
||||
ValueError: If `drop_n_first_frames`/`drop_n_last_frames` is negative, if
|
||||
`dataset_from_indices`/`dataset_to_indices` have different lengths, or if no episode has
|
||||
any frames remaining after dropping.
|
||||
"""
|
||||
if drop_n_first_frames < 0:
|
||||
raise ValueError(f"drop_n_first_frames must be >= 0, got {drop_n_first_frames}")
|
||||
@@ -124,15 +116,12 @@ class EpisodeAwareSampler:
|
||||
return [self._frame_index(k) for k in range(self._num_frames)]
|
||||
|
||||
def set_epoch(self, epoch: int) -> None:
|
||||
"""Set the epoch the next `__iter__` call will use, without consuming an auto-advance."""
|
||||
self._epoch = epoch
|
||||
|
||||
def state_dict(self) -> dict:
|
||||
"""Return `{"epoch": ..., "start_index": ...}`, enough to resume mid-epoch sample-exactly."""
|
||||
return {"epoch": self._epoch, "start_index": self._start_index}
|
||||
|
||||
def load_state_dict(self, state: dict) -> None:
|
||||
"""Restore the epoch and within-epoch offset from a `state_dict()`-produced dict."""
|
||||
self._epoch = state["epoch"]
|
||||
self._start_index = state["start_index"]
|
||||
|
||||
@@ -151,10 +140,6 @@ class EpisodeAwareSampler:
|
||||
return absolute_idx
|
||||
|
||||
def __iter__(self) -> Iterator[int]:
|
||||
"""Yield frame indices for the current epoch (from `set_epoch`/`load_state_dict`), then advance it.
|
||||
|
||||
Shuffled if `self.shuffle`, using a permutation seeded from `(seed, epoch)`.
|
||||
"""
|
||||
# Advance epoch state eagerly, not on first consumption of the generator.
|
||||
epoch, start = self._epoch, self._start_index
|
||||
self._epoch += 1
|
||||
@@ -171,7 +156,6 @@ class EpisodeAwareSampler:
|
||||
yield self._frame_index(k)
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""The total number of frames across the sampled episodes (full length, even mid-resume)."""
|
||||
return self._num_frames
|
||||
|
||||
|
||||
|
||||
@@ -44,13 +44,17 @@ from .video_utils import (
|
||||
|
||||
|
||||
class LookBackError(Exception):
|
||||
"""Exception raised when trying to look back in the history of a Backtrackable object."""
|
||||
"""
|
||||
Exception raised when trying to look back in the history of a Backtrackable object.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class LookAheadError(Exception):
|
||||
"""Exception raised when trying to look ahead in the future of a Backtrackable object."""
|
||||
"""
|
||||
Exception raised when trying to look ahead in the future of a Backtrackable object.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -60,10 +64,11 @@ class _ShardExhaustedError(Exception):
|
||||
|
||||
|
||||
class Backtrackable[T]:
|
||||
"""Wrap any iterator/iterable so you can step back up to `history` items and look ahead.
|
||||
"""
|
||||
Wrap any iterator/iterable so you can step back up to `history` items
|
||||
and look ahead up to `lookahead` items.
|
||||
|
||||
Looking ahead is bounded by `lookahead` items. This is useful for streaming datasets where you need
|
||||
to access previous and future items
|
||||
This is useful for streaming datasets where you need to access previous and future items
|
||||
but can't load the entire dataset into memory.
|
||||
|
||||
Example:
|
||||
@@ -93,16 +98,6 @@ class Backtrackable[T]:
|
||||
__slots__ = ("_source", "_back_buf", "_ahead_buf", "_cursor", "_history", "_lookahead")
|
||||
|
||||
def __init__(self, iterable: Iterable[T], *, history: int = 1, lookahead: int = 0):
|
||||
"""Wrap `iterable`, buffering up to `history` past items and `lookahead` future items.
|
||||
|
||||
Args:
|
||||
iterable: The iterable to wrap.
|
||||
history: How many past items `prev()`/`peek_back()` can reach. Must be `>= 1`.
|
||||
lookahead: How many future items `peek_ahead()` can reach. Must be `> 0`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `history < 1` or `lookahead <= 0`.
|
||||
"""
|
||||
if history < 1:
|
||||
raise ValueError("history must be >= 1")
|
||||
if lookahead <= 0:
|
||||
@@ -116,11 +111,9 @@ class Backtrackable[T]:
|
||||
self._lookahead = lookahead
|
||||
|
||||
def __iter__(self) -> "Backtrackable[T]":
|
||||
"""Return `self`; `Backtrackable` is its own iterator."""
|
||||
return self
|
||||
|
||||
def __next__(self) -> T:
|
||||
"""Return the next item, consuming from the back buffer first if `prev()` stepped back."""
|
||||
# If we've stepped back, consume from back buffer first
|
||||
if self._cursor < 0: # -1 means "last item", etc.
|
||||
self._cursor += 1
|
||||
@@ -135,9 +128,9 @@ class Backtrackable[T]:
|
||||
return item
|
||||
|
||||
def prev(self) -> T:
|
||||
"""Step one item back in history and return it.
|
||||
|
||||
Raises `LookBackError` if already at the oldest buffered item.
|
||||
"""
|
||||
Step one item back in history and return it.
|
||||
Raises IndexError if already at the oldest buffered item.
|
||||
"""
|
||||
if len(self._back_buf) + self._cursor <= 1:
|
||||
raise LookBackError("At start of history")
|
||||
@@ -146,15 +139,17 @@ class Backtrackable[T]:
|
||||
return self._back_buf[self._cursor]
|
||||
|
||||
def peek_back(self, n: int = 1) -> T:
|
||||
"""Look `n` items back (n=1 == previous item) without moving the cursor."""
|
||||
"""
|
||||
Look `n` items back (n=1 == previous item) without moving the cursor.
|
||||
"""
|
||||
if n < 0 or n + 1 > len(self._back_buf) + self._cursor:
|
||||
raise LookBackError("peek_back distance out of range")
|
||||
|
||||
return self._back_buf[self._cursor - (n + 1)]
|
||||
|
||||
def peek_ahead(self, n: int = 1) -> T:
|
||||
"""Look `n` items ahead (n=1 == next item) without moving the cursor.
|
||||
|
||||
"""
|
||||
Look `n` items ahead (n=1 == next item) without moving the cursor.
|
||||
Fills the ahead buffer if necessary.
|
||||
"""
|
||||
if n < 1:
|
||||
@@ -174,9 +169,9 @@ class Backtrackable[T]:
|
||||
return self._ahead_buf[n - 1]
|
||||
|
||||
def history(self) -> list[T]:
|
||||
"""Return a copy of the buffered history (most recent last).
|
||||
|
||||
The list length is at most the `history` argument passed at construction.
|
||||
"""
|
||||
Return a copy of the buffered history (most recent last).
|
||||
The list length ≤ `history` argument passed at construction.
|
||||
"""
|
||||
if self._cursor == 0:
|
||||
return list(self._back_buf)
|
||||
@@ -185,12 +180,14 @@ class Backtrackable[T]:
|
||||
return list(self._back_buf)[: self._cursor or None]
|
||||
|
||||
def can_peek_back(self, steps: int = 1) -> bool:
|
||||
"""Check if we can go back `steps` items without raising a `LookBackError`."""
|
||||
"""
|
||||
Check if we can go back `steps` items without raising an IndexError.
|
||||
"""
|
||||
return steps < len(self._back_buf) + self._cursor
|
||||
|
||||
def can_peek_ahead(self, steps: int = 1) -> bool:
|
||||
"""Check if we can peek ahead `steps` items.
|
||||
|
||||
"""
|
||||
Check if we can peek ahead `steps` items.
|
||||
This may involve trying to fill the ahead buffer.
|
||||
"""
|
||||
if self._lookahead > 0 and steps > self._lookahead:
|
||||
@@ -278,8 +275,6 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
episodes (list[int] | None, optional): If specified, this will only load episodes specified by
|
||||
their episode_index in this list.
|
||||
image_transforms (Callable | None, optional): Transform to apply to image data.
|
||||
delta_timestamps (dict[list[float]] | None, optional): Per-feature timestamp offsets (in
|
||||
seconds, relative to a frame's own timestamp) of additional frames to return alongside it.
|
||||
tolerance_s (float, optional): Tolerance in seconds for timestamp matching.
|
||||
revision (str, optional): Git revision id (branch name, tag, or commit hash).
|
||||
force_cache_sync (bool, optional): Flag to sync and refresh local files first.
|
||||
@@ -289,8 +284,6 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
seed (int, optional): Reproducibility random seed.
|
||||
rng (np.random.Generator | None, optional): Random number generator.
|
||||
shuffle (bool, optional): Whether to shuffle the dataset across exhaustions. Defaults to True.
|
||||
return_uint8 (bool, optional): For RGB videos, whether to return raw uint8 frames instead of
|
||||
the default float32 frames normalized to [0, 1].
|
||||
depth_output_unit (str, optional): Physical unit depth maps are dequantized to ("m" or "mm").
|
||||
Defaults to "mm".
|
||||
repo_type: "dataset" (default) or "bucket" to stream from an HF Storage Bucket
|
||||
@@ -390,17 +383,14 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
|
||||
@property
|
||||
def num_frames(self):
|
||||
"""The total number of frames in the dataset."""
|
||||
return self.meta.total_frames
|
||||
|
||||
@property
|
||||
def num_episodes(self):
|
||||
"""The total number of episodes in the dataset."""
|
||||
return self.meta.total_episodes
|
||||
|
||||
@property
|
||||
def fps(self):
|
||||
"""The dataset's recording frame rate."""
|
||||
return self.meta.fps
|
||||
|
||||
@property
|
||||
@@ -425,11 +415,6 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
# could be used with a ThreadPoolExecutor to run `make_frame` (especially video decoding)
|
||||
# in parallel, feeding a queue from which this iterator will yield processed items.
|
||||
def __iter__(self) -> Iterator[dict[str, torch.Tensor]]:
|
||||
"""Yield frames via reservoir-buffered random sampling across shards, streaming indefinitely.
|
||||
|
||||
Samples a random shard, then a random frame from a fixed-size buffer refilled from that shard, so
|
||||
no full shuffle or shard is ever fully materialized in memory.
|
||||
"""
|
||||
if self.video_decoder_cache is None:
|
||||
self.video_decoder_cache = VideoDecoderCache()
|
||||
|
||||
@@ -507,7 +492,7 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
return dict.fromkeys(self.meta.video_keys, [start_ts])
|
||||
|
||||
def _make_padding_camera_frame(self, camera_key: str):
|
||||
"""Variable-shape padding frame for the given camera key, shaped (H, W, C)."""
|
||||
"""Variable-shape padding frame for given camera keys, given in (H, W, C)"""
|
||||
return torch.zeros(self.meta.info.features[camera_key]["shape"]).permute(-1, 0, 1)
|
||||
|
||||
def _get_video_frame_padding_mask(
|
||||
@@ -538,7 +523,7 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
return padding_mask
|
||||
|
||||
def make_frame(self, dataset_iterator: Backtrackable) -> Generator:
|
||||
"""Makes a frame starting from a dataset iterator."""
|
||||
"""Makes a frame starting from a dataset iterator"""
|
||||
try:
|
||||
item = next(dataset_iterator)
|
||||
except StopIteration as e:
|
||||
@@ -631,13 +616,12 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
return query_timestamps
|
||||
|
||||
def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict:
|
||||
"""Decode the requested per-camera frame timestamps from `ep_idx`'s videos.
|
||||
|
||||
Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
|
||||
"""Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function
|
||||
in the main process (e.g. by using a second Dataloader with num_workers=0). It will result in a
|
||||
Segmentation Fault. This probably happens because a memory reference to the video loader is created in
|
||||
the main process and a subprocess fails to access it.
|
||||
"""
|
||||
|
||||
item = {}
|
||||
for video_key, query_ts in query_timestamps.items():
|
||||
root = self.meta.url_root if self.streaming and not self.streaming_from_local else self.root
|
||||
@@ -680,9 +664,8 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
"""Get frames with delta offsets using the backtrackable iterator.
|
||||
|
||||
Args:
|
||||
dataset_iterator (Backtrackable): The backtrackable iterator to peek/step through for delta
|
||||
frames.
|
||||
current_item (dict): Current item from the iterator.
|
||||
ep_idx (int): Episode index.
|
||||
|
||||
Returns:
|
||||
tuple: (query_result, padding) - frames at delta offsets and padding info.
|
||||
@@ -787,7 +770,8 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
|
||||
return query_result, padding
|
||||
|
||||
def _validate_delta_timestamp_keys(self, delta_timestamps: dict[list[float]]) -> None:
|
||||
"""Validate that all keys in delta_timestamps correspond to actual features in the dataset.
|
||||
"""
|
||||
Validate that all keys in delta_timestamps correspond to actual features in the dataset.
|
||||
|
||||
Raises:
|
||||
ValueError: If any delta timestamp key doesn't correspond to a dataset feature.
|
||||
|
||||
@@ -63,21 +63,11 @@ hub_api.create_tag("{repo_id}", tag="_version_", repo_type="dataset")
|
||||
"""
|
||||
|
||||
|
||||
class CompatibilityError(Exception):
|
||||
"""Base class for errors raised when a dataset's `codebase_version` doesn't match this install."""
|
||||
|
||||
...
|
||||
class CompatibilityError(Exception): ...
|
||||
|
||||
|
||||
class BackwardCompatibilityError(CompatibilityError):
|
||||
"""Raised when a dataset was saved with an older, unsupported `codebase_version`."""
|
||||
|
||||
def __init__(self, repo_id: str, version: packaging.version.Version):
|
||||
"""Build the error message pointing the user at the v2.1-to-v3.0 conversion script.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If `version` isn't the one supported legacy version (2.1).
|
||||
"""
|
||||
if version.major == 2 and version.minor == 1:
|
||||
message = V30_MESSAGE.format(repo_id=repo_id, version=version)
|
||||
else:
|
||||
@@ -88,10 +78,7 @@ class BackwardCompatibilityError(CompatibilityError):
|
||||
|
||||
|
||||
class ForwardCompatibilityError(CompatibilityError):
|
||||
"""Raised when a dataset was saved with a newer `codebase_version` than this install supports."""
|
||||
|
||||
def __init__(self, repo_id: str, version: packaging.version.Version):
|
||||
"""Build the error message pointing the user at upgrading their `lerobot` install."""
|
||||
message = FUTURE_MESSAGE.format(repo_id=repo_id, version=version)
|
||||
super().__init__(message)
|
||||
|
||||
@@ -202,12 +189,6 @@ class DatasetInfo:
|
||||
tools: list[dict] | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Coerce feature shapes from list to tuple, and validate `fps`/`chunks_size`/file-size fields.
|
||||
|
||||
Raises:
|
||||
ValueError: If `fps`, `chunks_size`, `data_files_size_in_mb`, or `video_files_size_in_mb` isn't
|
||||
positive.
|
||||
"""
|
||||
# Coerce feature shapes from list to tuple — JSON deserialisation
|
||||
# returns lists, but the rest of the codebase expects tuples.
|
||||
for ft in self.features.values():
|
||||
@@ -258,11 +239,6 @@ class DatasetInfo:
|
||||
# Once all callers have been migrated to attribute access, remove these.
|
||||
# ---------------------------------------------------------------------------
|
||||
def __getitem__(self, key: str):
|
||||
"""Deprecated dict-style read; use attribute access instead.
|
||||
|
||||
Raises:
|
||||
KeyError: If `key` isn't a field on this class.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
@@ -277,7 +253,6 @@ class DatasetInfo:
|
||||
raise KeyError(key) from err
|
||||
|
||||
def __setitem__(self, key: str, value) -> None:
|
||||
"""Deprecated dict-style write; use attribute assignment instead."""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
@@ -315,7 +290,6 @@ def has_legacy_hub_download_metadata(root: Path) -> bool:
|
||||
|
||||
|
||||
def update_chunk_file_indices(chunk_idx: int, file_idx: int, chunks_size: int) -> tuple[int, int]:
|
||||
"""Advance to the next `(chunk_idx, file_idx)`, rolling over to a new chunk once `chunks_size` is hit."""
|
||||
if file_idx == chunks_size - 1:
|
||||
file_idx = 0
|
||||
chunk_idx += 1
|
||||
@@ -381,7 +355,7 @@ def check_version_compatibility(
|
||||
repo_id (str): The repository ID for logging purposes.
|
||||
version_to_check (str | packaging.version.Version): The version of the dataset.
|
||||
current_version (str | packaging.version.Version): The current version of the codebase.
|
||||
enforce_breaking_major (bool, *optional*, defaults to `True`): If True, raise an error on major version mismatch.
|
||||
enforce_breaking_major (bool): If True, raise an error on major version mismatch.
|
||||
|
||||
Raises:
|
||||
BackwardCompatibilityError: If the dataset version is from a newer, incompatible
|
||||
@@ -408,9 +382,9 @@ def get_repo_versions(repo_id: str, *, token: str | bool | None = None) -> list[
|
||||
|
||||
Args:
|
||||
repo_id (str): The repository ID on the Hugging Face Hub.
|
||||
token (`str | bool | None`, *optional*): Authentication token used for Hub requests. Pass a string
|
||||
token, `True` to require the locally stored token, `False` to disable authentication, or `None`
|
||||
to use the Hugging Face Hub default.
|
||||
token: Authentication token used for Hub requests. Pass a string token,
|
||||
``True`` to require the locally stored token, ``False`` to disable
|
||||
authentication, or ``None`` to use the Hugging Face Hub default.
|
||||
|
||||
Returns:
|
||||
list[packaging.version.Version]: A list of valid versions found.
|
||||
@@ -440,7 +414,7 @@ def get_safe_version(
|
||||
Args:
|
||||
repo_id (str): The repository ID on the Hugging Face Hub.
|
||||
version (str | packaging.version.Version): The target version.
|
||||
token (`str | bool | None`, *optional*): Authentication token forwarded to the Hub version lookup.
|
||||
token: Authentication token forwarded to the Hub version lookup.
|
||||
|
||||
Returns:
|
||||
str: The safe version string (e.g., "v1.2.3") to use as a revision.
|
||||
@@ -487,7 +461,7 @@ def create_branch(repo_id: str, *, branch: str, repo_type: str | None = None) ->
|
||||
Args:
|
||||
repo_id (str): The ID of the repository.
|
||||
branch (str): The name of the branch to create.
|
||||
repo_type (str | None, *optional*): The type of the repository (e.g., "dataset").
|
||||
repo_type (str | None): The type of the repository (e.g., "dataset").
|
||||
"""
|
||||
api = HfApi()
|
||||
|
||||
@@ -512,12 +486,10 @@ def create_lerobot_dataset_card(
|
||||
https://huggingface.co/docs/hub/repositories-licenses.
|
||||
|
||||
Args:
|
||||
tags (list | None, *optional*): A list of tags to add to the dataset card.
|
||||
dataset_info (DatasetInfo | None, *optional*): The dataset's info object, which will
|
||||
tags (list | None): A list of tags to add to the dataset card.
|
||||
dataset_info (DatasetInfo | None): The dataset's info object, which will
|
||||
be displayed on the card.
|
||||
kwargs (`Any`, *optional*): Values used to replace placeholders in the card template, e.g. `license`, which
|
||||
must be a valid license identifier from
|
||||
https://huggingface.co/docs/hub/repositories-licenses.
|
||||
**kwargs: Additional keyword arguments to populate the card template.
|
||||
|
||||
Returns:
|
||||
DatasetCard: The generated dataset card object.
|
||||
@@ -552,12 +524,10 @@ def create_lerobot_dataset_card(
|
||||
|
||||
|
||||
def is_float_in_list(target, float_list, threshold=1e-6):
|
||||
"""Return `True` if `float_list` contains a value within `threshold` of `target`."""
|
||||
return any(abs(target - x) <= threshold for x in float_list)
|
||||
|
||||
|
||||
def find_float_index(target, float_list, threshold=1e-6):
|
||||
"""Return the index of the first value in `float_list` within `threshold` of `target`, or -1."""
|
||||
for i, x in enumerate(float_list):
|
||||
if abs(target - x) <= threshold:
|
||||
return i
|
||||
@@ -565,7 +535,9 @@ def find_float_index(target, float_list, threshold=1e-6):
|
||||
|
||||
|
||||
def safe_shard(dataset: datasets.IterableDataset, index: int, num_shards: int) -> datasets.Dataset:
|
||||
"""Safe shards the dataset."""
|
||||
"""
|
||||
Safe shards the dataset.
|
||||
"""
|
||||
shard_idx = min(dataset.num_shards, index + 1) - 1
|
||||
|
||||
return dataset.shard(num_shards, index=shard_idx)
|
||||
|
||||
@@ -61,18 +61,19 @@ def decode_video_frames(
|
||||
return_uint8: bool = False,
|
||||
is_depth: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Decodes video frames using the specified backend.
|
||||
"""
|
||||
Decodes video frames using the specified backend.
|
||||
|
||||
Args:
|
||||
video_path (Path): Path to the video file.
|
||||
timestamps (list[float]): List of timestamps to extract frames.
|
||||
tolerance_s (float): Allowed deviation in seconds for frame retrieval.
|
||||
backend (str, optional, *optional*): Backend to use for decoding. Defaults to "torchcodec" when available
|
||||
backend (str, optional): Backend to use for decoding. Defaults to "torchcodec" when available
|
||||
in the platform; otherwise, defaults to "pyav". The legacy value "video_reader" is
|
||||
accepted for one release as an alias for "pyav" and will be removed in a future version.
|
||||
return_uint8 (bool, *optional*, defaults to `False`): For RGB videos, if True return raw uint8 frames without float32 normalization.
|
||||
return_uint8 (bool): For RGB videos, if True return raw uint8 frames without float32 normalization.
|
||||
This reduces memory for DataLoader IPC; normalization can be done on GPU afterward.
|
||||
is_depth (bool, *optional*, defaults to `False`): Set to True if the video is a depth map (1 channel, uint12).
|
||||
is_depth (bool): Set to True if the video is a depth map (1 channel, uint12).
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Decoded frames (RGB: float32 in [0,1] by default, or uint8 if return_uint8=True, Depth: uint12).
|
||||
@@ -123,16 +124,14 @@ def decode_video_frames_pyav(
|
||||
video can be adjusted at encoding time to trade off decoding speed against file size.
|
||||
|
||||
Args:
|
||||
video_path (`pathlib.Path | str`): Path to the video file.
|
||||
timestamps (`list`): List of timestamps, in seconds, to extract frames for.
|
||||
tolerance_s (`float`): Allowed deviation in seconds between a queried timestamp
|
||||
and the closest decoded frame.
|
||||
log_loaded_timestamps (`bool`, *optional*, defaults to `False`): Whether to log
|
||||
every decoded frame's timestamp at INFO level.
|
||||
return_uint8 (`bool`, *optional*, defaults to `False`): For RGB videos, whether to
|
||||
return raw uint8 frames instead of the default float32 frames normalized to [0, 1].
|
||||
is_depth (`bool`, *optional*, defaults to `False`): Whether the video is a depth map
|
||||
(1 channel, uint12).
|
||||
video_path: Path to the video file.
|
||||
timestamps: List of timestamps (in seconds) to extract frames for.
|
||||
tolerance_s: Allowed deviation in seconds between a queried timestamp and the closest
|
||||
decoded frame.
|
||||
log_loaded_timestamps: When True, log every decoded frame's timestamp at INFO level.
|
||||
return_uint8: For RGB videos, if True return raw uint8 frames (C, H, W).
|
||||
Otherwise, return float32 in [0, 1] range.
|
||||
is_depth: Set to True if the video is a depth map (1 channel, uint12).
|
||||
|
||||
Returns:
|
||||
torch.Tensor of shape (len(timestamps), C, H, W).
|
||||
@@ -266,31 +265,15 @@ class VideoDecoderCache:
|
||||
ever opened until the process exits).
|
||||
|
||||
Args:
|
||||
max_size (`int | None | object`, *optional*, defaults to `<unset>`): Maximum
|
||||
number of decoders to retain. `None` disables eviction and restores legacy unbounded
|
||||
behaviour. The sentinel default defers to the value of `LEROBOT_VIDEO_DECODER_CACHE_SIZE`
|
||||
if set, otherwise `DEFAULT_DECODER_CACHE_SIZE`.
|
||||
max_size: Maximum number of decoders to retain. ``None`` disables
|
||||
eviction and restores legacy unbounded behaviour. Defaults to the
|
||||
value of ``LEROBOT_VIDEO_DECODER_CACHE_SIZE`` if set, otherwise
|
||||
:data:`DEFAULT_DECODER_CACHE_SIZE`.
|
||||
"""
|
||||
|
||||
class _UnsetSentinel:
|
||||
"""Singleton marker distinguishing "not passed" from an explicit `None` `max_size`.
|
||||
|
||||
Has a fixed `__repr__` (unlike a bare `object()`) so it renders identically across
|
||||
processes, which keeps the class docstring's `defaults to` clause stable.
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Return `"<unset>"`, a stable placeholder for docstrings/logging."""
|
||||
return "<unset>"
|
||||
|
||||
_SENTINEL: ClassVar[object] = _UnsetSentinel()
|
||||
_SENTINEL: ClassVar[object] = object()
|
||||
|
||||
def __init__(self, max_size: int | None | object = _SENTINEL):
|
||||
"""Create the cache. See the class docstring for `max_size`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `max_size` is neither `None` nor a positive integer.
|
||||
"""
|
||||
if max_size is VideoDecoderCache._SENTINEL:
|
||||
max_size = _default_max_cache_size()
|
||||
if max_size is not None and max_size <= 0:
|
||||
@@ -300,7 +283,6 @@ class VideoDecoderCache:
|
||||
self._lock = Lock()
|
||||
|
||||
def __contains__(self, video_path: object) -> bool:
|
||||
"""Return `True` if `video_path` (as `str`) has a cached decoder."""
|
||||
with self._lock:
|
||||
return str(video_path) in self._cache
|
||||
|
||||
@@ -356,7 +338,7 @@ class VideoDecoderCache:
|
||||
|
||||
|
||||
class FrameTimestampError(ValueError):
|
||||
"""Helper error to indicate the retrieved timestamps exceed the queried ones."""
|
||||
"""Helper error to indicate the retrieved timestamps exceed the queried ones"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -375,16 +357,11 @@ def decode_video_frames_torchcodec(
|
||||
"""Loads frames associated with the requested timestamps of a video using torchcodec.
|
||||
|
||||
Args:
|
||||
video_path (`pathlib.Path | str`): Path to the video file.
|
||||
timestamps (`list`): List of timestamps, in seconds, to extract frames for.
|
||||
tolerance_s (`float`): Allowed deviation in seconds between a queried timestamp
|
||||
and the closest decoded frame.
|
||||
log_loaded_timestamps (`bool`, *optional*, defaults to `False`): Whether to log
|
||||
every decoded frame's timestamp at INFO level.
|
||||
decoder_cache (`lerobot.datasets.video_utils.VideoDecoderCache | None`, *optional*): Decoder
|
||||
cache to fetch the `VideoDecoder` from. Uses the module-level default cache if `None`.
|
||||
return_uint8 (`bool`, *optional*, defaults to `False`): For RGB videos, whether to
|
||||
return raw uint8 frames instead of the default float32 frames normalized to [0, 1].
|
||||
video_path: Path to the video file.
|
||||
timestamps: List of timestamps to extract frames.
|
||||
tolerance_s: Allowed deviation in seconds for frame retrieval.
|
||||
log_loaded_timestamps: Whether to log loaded timestamps.
|
||||
decoder_cache: Optional decoder cache instance. Uses default if None.
|
||||
|
||||
Note: Setting device="cuda" outside the main process, e.g. in data loader workers, will lead to CUDA initialization errors.
|
||||
|
||||
@@ -474,19 +451,19 @@ def encode_video_frames(
|
||||
RGB frames are encoded directly.
|
||||
|
||||
Args:
|
||||
imgs_dir (`pathlib.Path | str`): Directory containing the frames to encode, named
|
||||
`frame-000000` onwards (`.png` for RGB, `.tiff` for depth).
|
||||
video_path (`pathlib.Path | str`): Output path for the encoded `.mp4` file.
|
||||
fps (`int`): Frame rate of the output video.
|
||||
video_encoder (`lerobot.configs.video.VideoEncoderConfig | None`, *optional*): Encoder settings
|
||||
(codec, pixel format, quality, ...). When `None`, `rgb_encoder_defaults` is used. Pass a
|
||||
`DepthEncoderConfig` to encode depth frames.
|
||||
encoder_threads (`int | None`, *optional*): Per-encoder thread count forwarded to the codec.
|
||||
`None` lets the codec decide.
|
||||
log_level (`int | None`, *optional*, defaults to 24): libav log level to set while encoding,
|
||||
or `None` to leave the current logging configuration unchanged.
|
||||
overwrite (`bool`, *optional*, defaults to `False`): When `False` and `video_path` already
|
||||
exists, skip encoding and log a warning. When `True`, re-encode and replace the existing file.
|
||||
imgs_dir: Directory containing the frames to encode, named ``frame-000000``
|
||||
onwards (``.png`` for RGB, ``.tiff`` for depth).
|
||||
video_path: Output path for the encoded ``.mp4`` file.
|
||||
fps: Frame rate of the output video.
|
||||
video_encoder: Encoder settings (codec, pixel format, quality, ...). When
|
||||
``None``, :func:`rgb_encoder_defaults` is used. Pass a
|
||||
:class:`~lerobot.configs.video.DepthEncoderConfig` to encode depth frames.
|
||||
encoder_threads: Per-encoder thread count forwarded to the codec. ``None``
|
||||
lets the codec decide.
|
||||
log_level: libav log level to set while encoding, or ``None`` to leave the
|
||||
current logging configuration unchanged.
|
||||
overwrite: When ``False`` and ``video_path`` already exists, skip encoding and
|
||||
log a warning. When ``True``, re-encode and replace the existing file.
|
||||
"""
|
||||
if video_encoder is None:
|
||||
video_encoder = rgb_encoder_defaults()
|
||||
@@ -575,21 +552,16 @@ def reencode_video(
|
||||
"""Re-encode a video file, optionally trimming it to ``[start_time_s, end_time_s)``.
|
||||
|
||||
Args:
|
||||
input_video_path (`pathlib.Path | str`): Existing video file to read.
|
||||
output_video_path (`pathlib.Path | str`): Path for the re-encoded file.
|
||||
video_encoder (`lerobot.configs.video.VideoEncoderConfig | None`, *optional*): Encoder
|
||||
configuration. Defaults to `rgb_encoder_defaults`.
|
||||
encoder_threads (`int | None`, *optional*): Optional thread count forwarded to
|
||||
`VideoEncoderConfig.get_codec_options`.
|
||||
log_level (`int | None`, *optional*, defaults to 24): libav log level while encoding,
|
||||
or `None` to leave logging unchanged.
|
||||
overwrite (`bool`, *optional*, defaults to `False`): When `False` and `output_video_path`
|
||||
already exists, skip and log a warning.
|
||||
start_time_s (`float | None`, *optional*): When set, trim the output to start at this
|
||||
timestamp, in seconds.
|
||||
end_time_s (`float | None`, *optional*): When set, trim the output to end at this
|
||||
timestamp, in seconds, exclusive.
|
||||
input_video_path: Existing video file to read.
|
||||
output_video_path: Path for the re-encoded file.
|
||||
video_encoder: Encoder configuration. Defaults to :func:`rgb_encoder_defaults`.
|
||||
encoder_threads: Optional thread count forwarded to :meth:`VideoEncoderConfig.get_codec_options`.
|
||||
log_level: libav log level while encoding, or ``None`` to leave logging unchanged. Defaults to WARNING.
|
||||
overwrite: When ``False`` and ``output_video_path`` already exists, skip and log a warning.
|
||||
start_time_s: When set, trim the output to start at this timestamp (seconds).
|
||||
end_time_s: When set, trim the output to end at this timestamp (seconds, exclusive).
|
||||
"""
|
||||
|
||||
video_encoder = video_encoder or rgb_encoder_defaults()
|
||||
|
||||
if (start_time_s is not None and start_time_s < 0) or (end_time_s is not None and end_time_s < 0):
|
||||
@@ -679,26 +651,25 @@ def concatenate_video_files(
|
||||
overwrite: bool = True,
|
||||
compatibility_check: bool = False,
|
||||
):
|
||||
"""Concatenate multiple video files into a single video file using pyav.
|
||||
"""
|
||||
Concatenate multiple video files into a single video file using pyav.
|
||||
|
||||
This function takes a list of video input file paths and concatenates them into a single
|
||||
output video file. It uses ffmpeg's concat demuxer with stream copy mode for fast
|
||||
concatenation without re-encoding.
|
||||
|
||||
Args:
|
||||
input_video_paths (`list`): Ordered list of input video file paths to concatenate.
|
||||
output_video_path (`Path`): Path to the output video file.
|
||||
overwrite (`bool`, *optional*, defaults to `True`): Whether to overwrite the output
|
||||
video file if it already exists.
|
||||
compatibility_check (`bool`, *optional*, defaults to `False`): Whether to check that
|
||||
the input videos share the same height, width, fps, codec, and pixel format
|
||||
before concatenating.
|
||||
input_video_paths: Ordered list of input video file paths to concatenate.
|
||||
output_video_path: Path to the output video file.
|
||||
overwrite: Whether to overwrite the output video file if it already exists. Default is True.
|
||||
compatibility_check: Whether to check if the input videos are compatible. Default is False.
|
||||
|
||||
Note:
|
||||
- Creates a temporary directory for intermediate files that is cleaned up after use.
|
||||
- Uses ffmpeg's concat demuxer which requires all input videos to have the same
|
||||
codec, resolution, and frame rate for proper concatenation.
|
||||
"""
|
||||
|
||||
output_video_path = Path(output_video_path)
|
||||
|
||||
if output_video_path.exists() and not overwrite:
|
||||
@@ -796,17 +767,6 @@ class _CameraEncoderThread(threading.Thread):
|
||||
stop_event: threading.Event,
|
||||
encoder_threads: int | None = None,
|
||||
):
|
||||
"""Set up the thread; frames are only consumed once `start()` is called.
|
||||
|
||||
Args:
|
||||
video_path: Output MP4 path.
|
||||
fps: Output frame rate.
|
||||
video_encoder: Codec/quality settings; `DepthEncoderConfig` selects depth-map encoding.
|
||||
frame_queue: Queue this thread reads `(frame, ...)` items from.
|
||||
result_queue: Queue the final stats are pushed to once encoding finishes.
|
||||
stop_event: Set by the caller to signal this thread to stop early.
|
||||
encoder_threads: Number of threads passed to the codec, if it supports one.
|
||||
"""
|
||||
super().__init__(daemon=True)
|
||||
self.video_path = video_path
|
||||
self.fps = fps
|
||||
@@ -818,10 +778,6 @@ class _CameraEncoderThread(threading.Thread):
|
||||
self.encoder_threads = encoder_threads
|
||||
|
||||
def run(self) -> None:
|
||||
"""Encode frames from `frame_queue` to `video_path` until a stop sentinel or `stop_event`.
|
||||
|
||||
Pushes the accumulated `RunningQuantileStats` to `result_queue` once encoding finishes.
|
||||
"""
|
||||
from .compute_stats import RunningQuantileStats, auto_downsample_height_width
|
||||
|
||||
container = None
|
||||
@@ -942,8 +898,7 @@ class StreamingVideoEncoder:
|
||||
queue_maxsize: int = 30,
|
||||
encoder_threads: int | None = None,
|
||||
):
|
||||
"""Create the manager; per-camera encoder threads are started lazily on first frame.
|
||||
|
||||
"""
|
||||
Args:
|
||||
fps: Frames per second for the output videos.
|
||||
rgb_encoder: Video encoder settings applied to all RGB cameras.
|
||||
@@ -1150,9 +1105,11 @@ class StreamingVideoEncoder:
|
||||
@dataclass
|
||||
class VideoFrame:
|
||||
# TODO(rcadene, lhoestq): move to Hugging Face `datasets` repo
|
||||
"""Provides a type for a dataset containing video frames.
|
||||
"""
|
||||
Provides a type for a dataset containing video frames.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
data_dict = [{"image": {"path": "videos/episode_0.mp4", "timestamp": 0.3}}]
|
||||
features = {"image": VideoFrame()}
|
||||
@@ -1164,7 +1121,6 @@ class VideoFrame:
|
||||
_type: str = field(default="VideoFrame", init=False, repr=False)
|
||||
|
||||
def __call__(self):
|
||||
"""Return the pyarrow struct type backing this feature, as required by `datasets.Features`."""
|
||||
return self.pa_type
|
||||
|
||||
|
||||
@@ -1179,11 +1135,6 @@ with warnings.catch_warnings():
|
||||
|
||||
|
||||
def get_audio_info(video_path: Path | str) -> dict:
|
||||
"""Read audio-stream metadata (channels, codec, bit rate, sample rate, etc.) from a video file.
|
||||
|
||||
Returns:
|
||||
A dict of `"audio.*"` keys, or `{"has_audio": False}` if `video_path` has no audio stream.
|
||||
"""
|
||||
# Set logging level
|
||||
logging.getLogger("libav").setLevel(av.logging.WARNING)
|
||||
|
||||
@@ -1222,13 +1173,13 @@ def get_video_info(
|
||||
"""Build the ``video.*`` / ``audio.*`` info dict persisted in ``info.json``.
|
||||
|
||||
Args:
|
||||
video_path (`pathlib.Path | str`): Path to the encoded video file to probe.
|
||||
video_encoder (`lerobot.configs.video.VideoEncoderConfig | None`, *optional*): If provided,
|
||||
record the exact encoder settings used to encode this video. Stream-derived values take
|
||||
precedence — encoder fields are only written for keys not already populated from the
|
||||
video file itself. When a `DepthEncoderConfig` is passed, the depth quantization
|
||||
parameters (`depth_min` / `depth_max` / `shift` / `use_log`) are recorded so frames can
|
||||
be dequantized on read.
|
||||
video_path: Path to the encoded video file to probe.
|
||||
video_encoder: If provided, record the exact encoder settings used to encode this
|
||||
video. Stream-derived values take precedence — encoder fields are only written for keys
|
||||
not already populated from the video file itself. When a
|
||||
:class:`~lerobot.configs.video.DepthEncoderConfig` is passed, the depth
|
||||
quantization parameters (``depth_min`` / ``depth_max`` / ``shift`` /
|
||||
``use_log``) are recorded so frames can be dequantized on read.
|
||||
|
||||
Returns:
|
||||
The ``video.*`` / ``audio.*`` info dict, including ``is_depth_map`` which is
|
||||
@@ -1276,10 +1227,11 @@ def get_video_info(
|
||||
|
||||
|
||||
def get_video_duration_in_s(video_path: Path | str) -> float:
|
||||
"""Get the duration of a video file in seconds using PyAV.
|
||||
"""
|
||||
Get the duration of a video file in seconds using PyAV.
|
||||
|
||||
Args:
|
||||
video_path (`pathlib.Path | str`): Path to the video file.
|
||||
video_path: Path to the video file.
|
||||
|
||||
Returns:
|
||||
Duration of the video in seconds.
|
||||
@@ -1297,7 +1249,8 @@ def get_video_duration_in_s(video_path: Path | str) -> float:
|
||||
|
||||
|
||||
class VideoEncodingManager:
|
||||
"""Context manager that ensures proper video encoding and data cleanup even if exceptions occur.
|
||||
"""
|
||||
Context manager that ensures proper video encoding and data cleanup even if exceptions occur.
|
||||
|
||||
This manager handles:
|
||||
- Batch encoding for any remaining episodes when recording interrupted
|
||||
@@ -1305,23 +1258,16 @@ class VideoEncodingManager:
|
||||
- Removing empty image directories
|
||||
|
||||
Args:
|
||||
dataset (`LeRobotDataset`): The LeRobotDataset instance.
|
||||
dataset: The LeRobotDataset instance
|
||||
"""
|
||||
|
||||
def __init__(self, dataset):
|
||||
"""Store the `LeRobotDataset` this manager will finalize/clean up on exit."""
|
||||
self.dataset = dataset
|
||||
|
||||
def __enter__(self):
|
||||
"""Return `self`; no setup is needed on entry."""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Finalize the dataset, cancelling pending videos and cleaning up interrupted-episode files.
|
||||
|
||||
Runs unconditionally (even if `exc_type` is set), so partial/interrupted recordings still leave
|
||||
a consistent dataset on disk.
|
||||
"""
|
||||
writer = self.dataset.writer
|
||||
if writer is not None:
|
||||
if exc_type is not None and writer._streaming_encoder is not None:
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Distributed-training runtime for LeRobot.
|
||||
|
||||
This package owns everything that turns the declarative topology in
|
||||
:class:`lerobot.configs.parallelism.ParallelismConfig` into a running engine:
|
||||
mesh math (:class:`~lerobot.distributed.parallel_dims.ParallelDims`), the
|
||||
`Accelerator` factory (:func:`~lerobot.distributed.factory.make_accelerator`),
|
||||
sharding-aware checkpoint helpers, and small rank utilities.
|
||||
|
||||
Setup-order contract (normative):
|
||||
CP dispatch install -> activation checkpointing -> torch.compile ->
|
||||
``fully_shard``/DDP (via ``accelerator.prepare``) -> optimizer rebind.
|
||||
Only the last two steps are active today; CP/AC/compile are configured
|
||||
placeholders wired in later rounds.
|
||||
"""
|
||||
|
||||
from .factory import guard_against_env_interference, make_accelerator, set_fsdp_wrap_modules
|
||||
from .parallel_dims import ParallelDims
|
||||
from .utils import finalize_sharded_policy, is_main_process, strip_accelerate_cp_hooks
|
||||
|
||||
__all__ = [
|
||||
"ParallelDims",
|
||||
"finalize_sharded_policy",
|
||||
"guard_against_env_interference",
|
||||
"is_main_process",
|
||||
"make_accelerator",
|
||||
"set_fsdp_wrap_modules",
|
||||
"strip_accelerate_cp_hooks",
|
||||
]
|
||||
@@ -1,195 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Sharding-aware checkpoint primitives.
|
||||
|
||||
Two artifact channels with distinct owners:
|
||||
|
||||
- the **distributable** ``model.safetensors``: produced by ``PreTrainedPolicy.save_pretrained``
|
||||
through :func:`full_model_state_dict` — a collective full gather when the model is sharded;
|
||||
- the **resume** channel (sharded runs): torch DCP directories written/read through accelerate's
|
||||
``save/load_fsdp_model`` and ``save/load_fsdp_optimizer`` (``pytorch_model_fsdp_0/`` and
|
||||
``optimizer_0/``, names imported from accelerate constants), which reshard on load across
|
||||
topology changes.
|
||||
|
||||
Every function that touches sharded state is a collective and must run on ALL ranks.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from accelerate import Accelerator
|
||||
|
||||
|
||||
def is_sharded_module(module: nn.Module) -> bool:
|
||||
"""True when `fully_shard` owns this module's parameters (FSDP2's in-place class swap).
|
||||
|
||||
Args:
|
||||
module (nn.Module): The module to inspect (a torch.compile wrapper is looked through
|
||||
via `_orig_mod`).
|
||||
|
||||
Returns:
|
||||
bool: True when the module (or its compiled `_orig_mod`) is an `FSDPModule`.
|
||||
"""
|
||||
from torch.distributed.fsdp import FSDPModule
|
||||
|
||||
if isinstance(module, FSDPModule):
|
||||
return True
|
||||
# torch.compile wraps the sharded module; mirror accelerate's `_orig_mod` check.
|
||||
orig_mod = getattr(module, "_orig_mod", None)
|
||||
return orig_mod is not None and isinstance(orig_mod, FSDPModule)
|
||||
|
||||
|
||||
def full_model_state_dict(module: nn.Module) -> dict[str, torch.Tensor]:
|
||||
"""The module's full (unsharded) state dict, however its parameters are laid out.
|
||||
|
||||
Sharded modules gather through torch's DCP state-dict API: a COLLECTIVE that must run on
|
||||
every rank; with ``cpu_offload=True`` the full dict materializes on the main rank only and
|
||||
every other rank receives a literal ``{}`` (runtime-verified — a
|
||||
rank-0-gated call deadlocks). Plain modules return ``module.state_dict()`` on every rank.
|
||||
|
||||
Args:
|
||||
module (nn.Module): The (possibly sharded) module to read the state dict from.
|
||||
|
||||
Returns:
|
||||
dict[str, torch.Tensor]: The full state dict — on the main rank only (``{}``
|
||||
elsewhere) when the module is sharded, on every rank otherwise.
|
||||
"""
|
||||
if not is_sharded_module(module):
|
||||
return module.state_dict()
|
||||
|
||||
from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict
|
||||
|
||||
return get_model_state_dict(module, options=StateDictOptions(full_state_dict=True, cpu_offload=True))
|
||||
|
||||
|
||||
def _fsdp_plugin(accelerator: "Accelerator") -> object:
|
||||
"""The accelerator's FSDP plugin, required by every DCP save/load helper below.
|
||||
|
||||
Args:
|
||||
accelerator (Accelerator): The accelerator that prepared the sharded model.
|
||||
|
||||
Returns:
|
||||
object: The FSDP plugin held by `accelerator.state`.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the accelerator was not configured with an FSDP plugin.
|
||||
"""
|
||||
plugin = getattr(accelerator.state, "fsdp_plugin", None)
|
||||
if plugin is None:
|
||||
raise RuntimeError("Sharded checkpointing requires an FSDP-prepared Accelerator.")
|
||||
return plugin
|
||||
|
||||
|
||||
def save_sharded_model(accelerator: "Accelerator", model: nn.Module, output_dir: Path) -> None:
|
||||
"""Write the DCP model shards (`pytorch_model_fsdp_0/`). Collective: call on all ranks.
|
||||
|
||||
Args:
|
||||
accelerator (Accelerator): The accelerator that prepared the sharded model.
|
||||
model (nn.Module): The prepared (sharded) model to save.
|
||||
output_dir (Path): The directory the shard subdirectory is created in.
|
||||
"""
|
||||
from accelerate.utils import save_fsdp_model
|
||||
|
||||
# accelerate 1.14's DCP helpers do string containment checks on the path:
|
||||
# always hand them str, never Path.
|
||||
save_fsdp_model(_fsdp_plugin(accelerator), accelerator, model, str(output_dir))
|
||||
|
||||
|
||||
def load_sharded_model(accelerator: "Accelerator", model: nn.Module, input_dir: Path) -> None:
|
||||
"""Load DCP model shards into the prepared (sharded) model. Collective: call on all ranks.
|
||||
|
||||
Args:
|
||||
accelerator (Accelerator): The accelerator that prepared the sharded model.
|
||||
model (nn.Module): The prepared (sharded) model to load into.
|
||||
input_dir (Path): The directory containing the `pytorch_model_fsdp_0/` shard
|
||||
subdirectory.
|
||||
"""
|
||||
from accelerate.utils import load_fsdp_model
|
||||
from accelerate.utils.constants import FSDP_MODEL_NAME
|
||||
|
||||
# Pass the exact shard directory: accelerate's load resolves it with a substring check
|
||||
# ("pytorch_model_fsdp" in the path -> use as-is), which misfires on run paths that happen
|
||||
# to contain the marker; the exact dir makes the check deterministic.
|
||||
load_fsdp_model(_fsdp_plugin(accelerator), accelerator, model, str(input_dir / f"{FSDP_MODEL_NAME}_0"))
|
||||
|
||||
|
||||
def save_sharded_optimizer(
|
||||
accelerator: "Accelerator", optimizer: torch.optim.Optimizer, model: nn.Module, output_dir: Path
|
||||
) -> None:
|
||||
"""Write the DCP optimizer shards (`optimizer_0/`). Collective: call on all ranks.
|
||||
|
||||
Args:
|
||||
accelerator (Accelerator): The accelerator that prepared the model and optimizer.
|
||||
optimizer (torch.optim.Optimizer): The prepared optimizer to save the state from.
|
||||
model (nn.Module): The prepared (sharded) model the optimizer state is keyed by.
|
||||
output_dir (Path): The directory the shard subdirectory is created in.
|
||||
"""
|
||||
from accelerate.utils import save_fsdp_optimizer
|
||||
|
||||
save_fsdp_optimizer(_fsdp_plugin(accelerator), accelerator, optimizer, model, str(output_dir))
|
||||
|
||||
|
||||
def load_sharded_optimizer(
|
||||
accelerator: "Accelerator", optimizer: torch.optim.Optimizer, model: nn.Module, input_dir: Path
|
||||
) -> None:
|
||||
"""Load DCP optimizer shards into the prepared optimizer. Collective: call on all ranks.
|
||||
|
||||
Must run AFTER ``accelerator.prepare()``: FSDP2's prepare rebinds the optimizer's param
|
||||
groups to sharded DTensors but never migrates ``optimizer.state`` — the resharding load is
|
||||
the only correct way to restore it.
|
||||
|
||||
Args:
|
||||
accelerator (Accelerator): The accelerator that prepared the model and optimizer.
|
||||
optimizer (torch.optim.Optimizer): The prepared optimizer to restore the state into.
|
||||
model (nn.Module): The prepared (sharded) model the optimizer state is keyed by.
|
||||
input_dir (Path): The directory containing the `optimizer_0/` shard subdirectory.
|
||||
"""
|
||||
from accelerate.utils import load_fsdp_optimizer
|
||||
from accelerate.utils.constants import OPTIMIZER_NAME
|
||||
|
||||
# Exact shard directory for the same reason as load_sharded_model: accelerate's substring
|
||||
# check ("optimizer" in the path) would misread e.g. --job_name=optimizer_sweep run paths.
|
||||
load_fsdp_optimizer(
|
||||
_fsdp_plugin(accelerator), accelerator, optimizer, model, str(input_dir / f"{OPTIMIZER_NAME}_0")
|
||||
)
|
||||
|
||||
|
||||
def dcp_to_safetensors(dcp_dir: Path, output_dir: Path, *, delete_dcp: bool = False) -> Path:
|
||||
"""Merge a DCP shard directory into a single `model.safetensors` (offline, single process).
|
||||
|
||||
Thin wrapper over `accelerate.utils.merge_fsdp_weights`, which loads the shards without a
|
||||
process group, writes safetensors directly, and — when asked — removes the merged shard
|
||||
directory itself, only on the main process and only once the merge has succeeded.
|
||||
|
||||
Args:
|
||||
dcp_dir (Path): The DCP shard directory to merge (e.g. `.../pytorch_model_fsdp_0`).
|
||||
output_dir (Path): The directory the merged `model.safetensors` is written into.
|
||||
delete_dcp (bool): Whether to remove the shard directory once it has been merged.
|
||||
Defaults to False.
|
||||
|
||||
Returns:
|
||||
Path: The written `model.safetensors` file's path.
|
||||
"""
|
||||
from accelerate.utils import merge_fsdp_weights
|
||||
|
||||
merge_fsdp_weights(
|
||||
str(dcp_dir), str(output_dir), safe_serialization=True, remove_checkpoint_dir=delete_dcp
|
||||
)
|
||||
return output_dir / "model.safetensors"
|
||||
@@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""The `Accelerator` factory — the only place accelerate gets configured.
|
||||
|
||||
`torchrun` is the launcher; every accelerate parameter comes from `TrainPipelineConfig`
|
||||
(`cfg.parallelism` + `cfg.accelerator`) so a run is reproducible from its `train_config.json`
|
||||
alone. `accelerate launch` without a `--config_file` remains equivalent (it only sets rendezvous
|
||||
env vars in that mode); the yaml flow is superseded.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from lerobot.configs.parallelism import world_size_from_env
|
||||
from lerobot.configs.train import TrainPipelineConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from accelerate import Accelerator
|
||||
|
||||
from lerobot.policies.pretrained import PreTrainedPolicy
|
||||
|
||||
# Env vars through which `accelerate launch --config_file` (or a stray shell) would configure
|
||||
# accelerate behind the config system's back. Plugin `__post_init__`s read these silently as
|
||||
# field fallbacks (ACCELERATE_DYNAMO_* enables torch.compile through the default
|
||||
# TorchDynamoPlugin; ACCELERATE_GRADIENT_ACCUMULATION_STEPS overrides the explicitly passed
|
||||
# value inside Accelerator.__init__), which would make train_config.json lie about what ran.
|
||||
_ACCELERATE_ENV_PREFIXES = ("FSDP_", "PARALLELISM_CONFIG_", "ACCELERATE_DYNAMO_")
|
||||
_ACCELERATE_ENV_VARS = (
|
||||
"ACCELERATE_USE_FSDP",
|
||||
"ACCELERATE_USE_PARALLELISM_CONFIG",
|
||||
"ACCELERATE_GRADIENT_ACCUMULATION_STEPS",
|
||||
)
|
||||
_ENV_OVERRIDE = "LEROBOT_ALLOW_ACCELERATE_ENV"
|
||||
|
||||
|
||||
def guard_against_env_interference() -> None:
|
||||
"""Hard-error when accelerate-configuring env vars are set.
|
||||
|
||||
A silently env-overridden "reproducible" config is worse than a stop: users migrating from
|
||||
the old `accelerate launch --config_file fsdp.yaml` flow get a precise error instead of a
|
||||
config that lies. Set LEROBOT_ALLOW_ACCELERATE_ENV=1 to acknowledge and proceed.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If any accelerate-configuring environment variable is set and the
|
||||
LEROBOT_ALLOW_ACCELERATE_ENV override is not.
|
||||
"""
|
||||
if os.environ.get(_ENV_OVERRIDE):
|
||||
return
|
||||
offending = sorted(
|
||||
name
|
||||
for name in os.environ
|
||||
if name in _ACCELERATE_ENV_VARS or name.startswith(_ACCELERATE_ENV_PREFIXES)
|
||||
)
|
||||
if offending:
|
||||
raise RuntimeError(
|
||||
f"Accelerate-configuring environment variables are set: {', '.join(offending)}. "
|
||||
"LeRobot manages accelerate exclusively through TrainPipelineConfig "
|
||||
"(--parallelism.* / --accelerator.*); launch with plain torchrun and remove these "
|
||||
"variables (the `accelerate launch --config_file` flow is superseded), or set "
|
||||
f"{_ENV_OVERRIDE}=1 to acknowledge that they may override your config."
|
||||
)
|
||||
|
||||
|
||||
def make_accelerator(cfg: TrainPipelineConfig) -> "Accelerator":
|
||||
"""Resolve the topology against the launched world and build the `Accelerator`.
|
||||
|
||||
Must run once per process, before any other component needs the device or the process
|
||||
group (`Accelerator.__init__` initializes both and builds the device mesh).
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The full training config; `cfg.parallelism` is resolved in
|
||||
place against the launched world size and `cfg.accelerator` builds the result.
|
||||
|
||||
Returns:
|
||||
Accelerator: The configured accelerator, with device and process group initialized.
|
||||
|
||||
Raises:
|
||||
ValueError: If `cfg.checkpoint_format` requires DCP but the topology resolved to a
|
||||
non-sharded run.
|
||||
"""
|
||||
guard_against_env_interference()
|
||||
cfg.parallelism.resolve(world_size_from_env())
|
||||
# The parse-time format check ran against the declared degrees, where the dp_shard=-1
|
||||
# sentinel counts as sharded; it may resolve to an unsharded run (e.g. -1 at world size 1).
|
||||
# Re-check against the concrete degrees so the recorded format never lies about the
|
||||
# artifacts a checkpoint will actually contain.
|
||||
if cfg.checkpoint_format.wants_dcp and not cfg.parallelism.is_sharded:
|
||||
raise ValueError(
|
||||
f"checkpoint_format={cfg.checkpoint_format.value} requires a sharded run, but the "
|
||||
f"topology resolved to a non-sharded one (dp_replicate={cfg.parallelism.dp_replicate}, "
|
||||
f"dp_shard={cfg.parallelism.dp_shard}); non-sharded checkpoints are always safetensors."
|
||||
)
|
||||
return cfg.accelerator.build(
|
||||
cfg.parallelism,
|
||||
cpu=cfg.trainable_config.device == "cpu",
|
||||
)
|
||||
|
||||
|
||||
def set_fsdp_wrap_modules(accelerator: "Accelerator", policy: "PreTrainedPolicy") -> None:
|
||||
"""Resolve the FSDP wrap-unit class names onto the plugin before `accelerator.prepare()`.
|
||||
|
||||
Resolution order: user override (`--accelerator.fsdp.wrap_modules`, already on the plugin)
|
||||
-> the policy's `_fsdp_wrap_modules` declaration -> hard error. Root-only wrapping — the
|
||||
silent default when no wrap source exists — is never accepted: it quietly forfeits all
|
||||
sharding memory savings.
|
||||
|
||||
No-op for the size-based policy (`--accelerator.fsdp.min_num_params`), which needs no class
|
||||
names, and for non-sharded runs (no fsdp plugin).
|
||||
|
||||
Args:
|
||||
accelerator (Accelerator): The accelerator whose FSDP plugin receives the wrap-unit
|
||||
class names.
|
||||
policy (PreTrainedPolicy): The trainable whose class may declare `_fsdp_wrap_modules`.
|
||||
|
||||
Raises:
|
||||
ValueError: If sharded class-based wrapping is configured but neither a user override
|
||||
nor a policy declaration supplies wrap-unit class names.
|
||||
"""
|
||||
plugin = getattr(accelerator.state, "fsdp_plugin", None)
|
||||
if plugin is None or plugin.min_num_params:
|
||||
return
|
||||
if plugin.transformer_cls_names_to_wrap: # user override, set at build time
|
||||
return
|
||||
# getattr, not attribute access: non-policy trainables (no `_fsdp_wrap_modules` attribute)
|
||||
# must reach the actionable error below, not an AttributeError.
|
||||
declared = getattr(type(policy), "_fsdp_wrap_modules", None)
|
||||
if not declared:
|
||||
raise ValueError(
|
||||
f"Policy '{type(policy).__name__}' declares no FSDP wrap units. Sharded training "
|
||||
"requires wrap-unit class names: set --accelerator.fsdp.wrap_modules='[\"MyBlock\"]' "
|
||||
"(or --accelerator.fsdp.min_num_params for a size-based policy), or declare "
|
||||
"`_fsdp_wrap_modules` on the policy class."
|
||||
)
|
||||
plugin.transformer_cls_names_to_wrap = list(declared)
|
||||
@@ -1,112 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Runtime mesh math derived from the declarative :class:`ParallelismConfig`.
|
||||
|
||||
`ParallelDims` is the training script's single source of truth for topology-derived numbers
|
||||
(data-parallel world size and rank, sample accounting inputs) and — once the CP engine lands —
|
||||
the owner of LeRobot's private ``(dp_replicate, dp_shard, ring, ulysses)`` mesh. It is a runtime
|
||||
object and is never serialized (the config it derives from is what lands in
|
||||
``train_config.json``).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch.distributed as dist
|
||||
|
||||
from lerobot.configs.parallelism import ParallelismConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParallelDims:
|
||||
"""Concrete parallelism degrees bound to a world size (canonical row-major rank layout)."""
|
||||
|
||||
dp_replicate: int
|
||||
dp_shard: int
|
||||
ring: int
|
||||
ulysses: int
|
||||
world_size: int
|
||||
device_type: str
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, cfg: ParallelismConfig, world_size: int, device_type: str) -> "ParallelDims":
|
||||
"""Bind a *resolved* config to the actual runtime world size (cross-checked here).
|
||||
|
||||
Args:
|
||||
cfg (ParallelismConfig): The declarative topology, already resolved via
|
||||
`ParallelismConfig.resolve(world_size)`.
|
||||
world_size (int): The launched world size the declared degrees must multiply to.
|
||||
device_type (str): The accelerator device type backing the mesh (e.g. "cuda").
|
||||
|
||||
Returns:
|
||||
ParallelDims: The concrete parallelism degrees bound to this world.
|
||||
|
||||
Raises:
|
||||
ValueError: If the config is unresolved (`dp_shard == -1`) or its degrees do not
|
||||
multiply to `world_size`.
|
||||
"""
|
||||
total = cfg.dp_replicate * cfg.dp_shard * cfg.cp_size
|
||||
if cfg.dp_shard == -1 or total != world_size:
|
||||
raise ValueError(
|
||||
f"ParallelismConfig is not resolved against this world: dp_replicate="
|
||||
f"{cfg.dp_replicate} * dp_shard={cfg.dp_shard} * cp={cfg.cp_size} != "
|
||||
f"world_size={world_size}. Call ParallelismConfig.resolve(world_size) first "
|
||||
"(make_accelerator does this)."
|
||||
)
|
||||
return cls(
|
||||
dp_replicate=cfg.dp_replicate,
|
||||
dp_shard=cfg.dp_shard,
|
||||
ring=cfg.context_parallel.ring_degree,
|
||||
ulysses=cfg.context_parallel.ulysses_degree,
|
||||
world_size=world_size,
|
||||
device_type=device_type,
|
||||
)
|
||||
|
||||
@property
|
||||
def cp_size(self) -> int:
|
||||
"""Total context-parallel degree (`ring * ulysses`)."""
|
||||
return self.ring * self.ulysses
|
||||
|
||||
@property
|
||||
def is_sharded(self) -> bool:
|
||||
"""Whether parameters are sharded (`dp_shard > 1` or any context parallelism)."""
|
||||
return self.dp_shard > 1 or self.cp_size > 1
|
||||
|
||||
@property
|
||||
def dp_world_size(self) -> int:
|
||||
"""Number of distinct data-parallel workers — the divisor for all sample accounting."""
|
||||
return self.dp_replicate * self.dp_shard
|
||||
|
||||
@property
|
||||
def dp_rank(self) -> int:
|
||||
"""This process's data-parallel coordinate (CP peers share one dp_rank).
|
||||
|
||||
With the canonical row-major layout and (ring, ulysses) innermost, CP peers are
|
||||
contiguous global ranks, so the dp coordinate is the integer quotient by cp_size —
|
||||
the same arithmetic accelerate's mesh-aware dataloader applies.
|
||||
"""
|
||||
global_rank = dist.get_rank() if dist.is_initialized() else 0
|
||||
return global_rank // self.cp_size
|
||||
|
||||
def cp_mesh(self) -> None:
|
||||
"""Private (ring, ulysses) mesh for the CP engine — reserved for the CP round.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: Always — context parallelism is not implemented yet.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Context parallelism is not implemented yet; ParallelDims.cp_mesh is reserved for "
|
||||
"the CP engine round (a private mesh aligned with accelerate's cp block)."
|
||||
)
|
||||
@@ -1,94 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Rank utilities and post-`prepare()` sharding finalization."""
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch.distributed as dist
|
||||
from torch import nn
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lerobot.distributed.parallel_dims import ParallelDims
|
||||
|
||||
|
||||
def is_main_process() -> bool:
|
||||
"""True on the process that owns rank-0-only side effects (file writes, uploads, logging).
|
||||
|
||||
Torch-native on purpose: persistence code must not depend on an `Accelerator` handle —
|
||||
`_save_pretrained` and the hub publishers run in contexts that have none. Outside
|
||||
distributed runs every process is the main process.
|
||||
|
||||
Returns:
|
||||
bool: True when this process is rank 0 or no process group is initialized.
|
||||
"""
|
||||
return not dist.is_initialized() or dist.get_rank() == 0
|
||||
|
||||
|
||||
def strip_accelerate_cp_hooks(model: nn.Module) -> int:
|
||||
"""Remove accelerate's context-parallel forward-pre-hooks from every module.
|
||||
|
||||
When `cp_size > 1` is declared, `accelerator.prepare()` unconditionally attaches hooks that
|
||||
silently replace any `attention_mask` kwarg of `*self_attn` modules with `is_causal=True`
|
||||
(`accelerate.big_modeling._attach_context_parallel_hooks`) — mask corruption for policies
|
||||
with non-causal attention. LeRobot implements CP itself and never enters accelerate's CP
|
||||
context, so these hooks are pure hazard. Deterministically identified by their defining
|
||||
module; a version canary pins that identity.
|
||||
|
||||
Args:
|
||||
model (nn.Module): The prepared model to strip the hooks from (all submodules are
|
||||
visited).
|
||||
|
||||
Returns:
|
||||
int: The number of hooks removed.
|
||||
"""
|
||||
removed = 0
|
||||
for module in model.modules():
|
||||
for hook_id, hook in list(module._forward_pre_hooks.items()):
|
||||
if getattr(hook, "__module__", None) == "accelerate.big_modeling":
|
||||
del module._forward_pre_hooks[hook_id]
|
||||
module._forward_pre_hooks_with_kwargs.pop(hook_id, None)
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def finalize_sharded_policy(policy: nn.Module, parallel_dims: "ParallelDims") -> None:
|
||||
"""Sharding correctness protocol, applied once, immediately after `accelerator.prepare()`.
|
||||
|
||||
1. Strip accelerate's CP mask hooks (only attached when cp > 1 was declared).
|
||||
2. Register the policy's non-`forward` entry points (`_fsdp_forward_methods`) so FSDP2
|
||||
unshards parameters around `select_action` & co. — without this, any inference-style
|
||||
call on a sharded policy crashes on mixed Tensor/DTensor.
|
||||
|
||||
No-op for DDP/single-process runs.
|
||||
|
||||
Args:
|
||||
policy (nn.Module): The policy as returned by `accelerator.prepare()`.
|
||||
parallel_dims (ParallelDims): The run's resolved topology; decides whether the protocol
|
||||
applies.
|
||||
"""
|
||||
if not parallel_dims.is_sharded:
|
||||
return
|
||||
if parallel_dims.cp_size > 1:
|
||||
removed = strip_accelerate_cp_hooks(policy)
|
||||
logging.info("Stripped %d accelerate context-parallel attention-mask hooks.", removed)
|
||||
|
||||
from torch.distributed.fsdp import FSDPModule, register_fsdp_forward_method
|
||||
|
||||
if isinstance(policy, FSDPModule):
|
||||
for method_name in getattr(type(policy), "_fsdp_forward_methods", ()):
|
||||
if callable(getattr(policy, method_name, None)):
|
||||
register_fsdp_forward_method(policy, method_name)
|
||||
@@ -432,7 +432,7 @@ def submit_to_hf(cfg: TrainPipelineConfig) -> None:
|
||||
|
||||
# Finish as soon as the model is pushed, rather than waiting out the platform's
|
||||
# post-run finalization before the job stage flips to COMPLETED. This matches the
|
||||
# exact log line emitted by lerobot.common.train_utils.publish_trained_model — the two must stay
|
||||
# exact log line emitted by PreTrainedPolicy.push_model_to_hub — the two must stay
|
||||
# in sync. If it ever stops matching we just fall back to stage-based completion
|
||||
# (~30s slower), so the contract is an optimization, not a correctness requirement.
|
||||
success_marker = f"Model pushed to https://huggingface.co/{repo_id}"
|
||||
|
||||
@@ -314,16 +314,11 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
To find the port, you can run our utility script:
|
||||
```bash
|
||||
lerobot-find-port.py
|
||||
```
|
||||
|
||||
which prints:
|
||||
|
||||
```
|
||||
Finding all available ports for the MotorsBus.
|
||||
["/dev/tty.usbmodem575E0032081", "/dev/tty.usbmodem575E0031751"]
|
||||
Remove the usb cable from your MotorsBus and press Enter when done.
|
||||
The port of this MotorsBus is /dev/tty.usbmodem575E0031751.
|
||||
Reconnect the usb cable.
|
||||
>>> Finding all available ports for the MotorsBus.
|
||||
>>> ["/dev/tty.usbmodem575E0032081", "/dev/tty.usbmodem575E0031751"]
|
||||
>>> Remove the usb cable from your MotorsBus and press Enter when done.
|
||||
>>> The port of this MotorsBus is /dev/tty.usbmodem575E0031751.
|
||||
>>> Reconnect the usb cable.
|
||||
```
|
||||
|
||||
Example of usage for 1 Feetech sts3215 motor connected to the bus:
|
||||
@@ -600,7 +595,7 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
ID, and finally programs the bus' default baud-rate.
|
||||
|
||||
Args:
|
||||
motor (str): Key of the motor in `motors`.
|
||||
motor (str): Key of the motor in :pyattr:`motors`.
|
||||
initial_baudrate (int | None, optional): Current baud-rate (skips scanning when provided).
|
||||
Defaults to None.
|
||||
initial_id (int | None, optional): Current ID (skips scanning when provided). Defaults to None.
|
||||
@@ -671,7 +666,7 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
"""Enable torque on selected motors.
|
||||
|
||||
Args:
|
||||
motors (int | str | list[str] | None, optional): Same semantics as [`~motors.motors_bus.MotorsBus.disable_torque`].
|
||||
motors (int | str | list[str] | None, optional): Same semantics as :pymeth:`disable_torque`.
|
||||
Defaults to `None`.
|
||||
num_retry (int, optional): Number of additional retry attempts on communication failure.
|
||||
Defaults to 0.
|
||||
@@ -684,12 +679,10 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
|
||||
This helper is useful to temporarily disable torque when configuring motors.
|
||||
|
||||
Example:
|
||||
```python
|
||||
>>> with bus.torque_disabled(): # doctest: +SKIP
|
||||
Examples:
|
||||
>>> with bus.torque_disabled():
|
||||
... # Safe operations here
|
||||
... pass
|
||||
```
|
||||
"""
|
||||
self.disable_torque(motors)
|
||||
try:
|
||||
@@ -702,7 +695,7 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
|
||||
Args:
|
||||
timeout_ms (int | None, optional): Timeout in *milliseconds*. If `None` (default) the method falls
|
||||
back to `default_timeout`.
|
||||
back to :pyattr:`default_timeout`.
|
||||
"""
|
||||
timeout_ms = timeout_ms if timeout_ms is not None else self.default_timeout
|
||||
self.port_handler.setPacketTimeoutMillis(timeout_ms)
|
||||
@@ -753,8 +746,8 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
|
||||
Args:
|
||||
calibration_dict (dict[str, MotorCalibration]): Calibration obtained from
|
||||
[`~motors.motors_bus.MotorsBus.read_calibration`] or crafted by the user.
|
||||
cache (bool, optional): Save the calibration to `calibration`. Defaults to True.
|
||||
:pymeth:`read_calibration` or crafted by the user.
|
||||
cache (bool, optional): Save the calibration to :pyattr:`calibration`. Defaults to True.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -762,7 +755,7 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
"""Restore factory calibration for the selected motors.
|
||||
|
||||
Homing offset is set to ``0`` and min/max position limits are set to the full usable range.
|
||||
The in-memory `calibration` is cleared.
|
||||
The in-memory :pyattr:`calibration` is cleared.
|
||||
|
||||
Args:
|
||||
motors (NameOrID | Sequence[NameOrID] | None, optional): Selection of motors. `None` (default)
|
||||
@@ -1076,9 +1069,9 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
) -> None:
|
||||
"""Write a value to a single motor's register.
|
||||
|
||||
Contrary to [`~motors.motors_bus.MotorsBus.sync_write`], this expects a response status packet emitted by the motor, which
|
||||
Contrary to :pymeth:`sync_write`, this expects a response status packet emitted by the motor, which
|
||||
provides a guarantee that the value was written to the register successfully. In consequence, it is
|
||||
slower than [`~motors.motors_bus.MotorsBus.sync_write`] but it is more reliable. It should typically be used when configuring
|
||||
slower than :pymeth:`sync_write` but it is more reliable. It should typically be used when configuring
|
||||
motors.
|
||||
|
||||
Args:
|
||||
@@ -1235,8 +1228,8 @@ class SerialMotorsBus(MotorsBusBase):
|
||||
) -> None:
|
||||
"""Write the same register on multiple motors.
|
||||
|
||||
Contrary to [`~motors.motors_bus.MotorsBus.write`], this *does not* expects a response status packet emitted by the motor, which
|
||||
can allow for lost packets. It is faster than [`~motors.motors_bus.MotorsBus.write`] and should typically be used when
|
||||
Contrary to :pymeth:`write`, this *does not* expects a response status packet emitted by the motor, which
|
||||
can allow for lost packets. It is faster than :pymeth:`write` and should typically be used when
|
||||
frequency matters and losing some packets is acceptable (e.g. teleoperation loops).
|
||||
|
||||
Args:
|
||||
|
||||
@@ -20,6 +20,7 @@ from .optimizers import (
|
||||
SGDConfig as SGDConfig,
|
||||
XVLAAdamWConfig as XVLAAdamWConfig,
|
||||
load_optimizer_state,
|
||||
load_optimizer_state_dict,
|
||||
save_optimizer_state,
|
||||
)
|
||||
from .schedulers import (
|
||||
@@ -50,6 +51,7 @@ __all__ = [
|
||||
"VQBeTSchedulerConfig",
|
||||
# State management
|
||||
"load_optimizer_state",
|
||||
"load_optimizer_state_dict",
|
||||
"load_scheduler_state",
|
||||
"save_optimizer_state",
|
||||
"save_scheduler_state",
|
||||
|
||||
@@ -27,7 +27,7 @@ from lerobot.utils.constants import (
|
||||
OPTIMIZER_PARAM_GROUPS,
|
||||
OPTIMIZER_STATE,
|
||||
)
|
||||
from lerobot.utils.io_utils import deserialize_json_into_object, write_json
|
||||
from lerobot.utils.io_utils import deserialize_json_into_object, load_json, write_json
|
||||
from lerobot.utils.utils import flatten_dict, unflatten_dict
|
||||
|
||||
# Type alias for parameters accepted by optimizer build() methods.
|
||||
@@ -52,11 +52,6 @@ class OptimizerConfig(draccus.ChoiceRegistry, abc.ABC):
|
||||
def type(self) -> str:
|
||||
return self.get_choice_name(self.__class__)
|
||||
|
||||
@property
|
||||
def builds_multiple_optimizers(self) -> bool:
|
||||
"""True when build() returns a dict of optimizers (unsupported under sharded training)."""
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def default_choice_name(cls) -> str | None:
|
||||
return "adam"
|
||||
@@ -250,10 +245,6 @@ class MultiAdamConfig(OptimizerConfig):
|
||||
grad_clip_norm: float = 10.0
|
||||
optimizer_groups: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def builds_multiple_optimizers(self) -> bool:
|
||||
return True
|
||||
|
||||
def build(self, params: OptimizerParams) -> dict[str, torch.optim.Optimizer]:
|
||||
"""Build multiple Adam optimizers.
|
||||
|
||||
@@ -292,27 +283,35 @@ class MultiAdamConfig(OptimizerConfig):
|
||||
def save_optimizer_state(
|
||||
optimizer: torch.optim.Optimizer | dict[str, torch.optim.Optimizer],
|
||||
save_dir: Path,
|
||||
optim_state_dict: dict | None = None,
|
||||
) -> None:
|
||||
"""Save optimizer state to disk (non-sharded runs; sharded runs use the DCP channel).
|
||||
"""Save optimizer state to disk.
|
||||
|
||||
Args:
|
||||
optimizer: Either a single optimizer or a dictionary of optimizers.
|
||||
save_dir: Directory to save the optimizer state.
|
||||
optim_state_dict: Pre-gathered optimizer state dict (for FSDP, where the sharded state must
|
||||
be gathered across ranks first). If provided, it is saved directly instead of calling
|
||||
``optimizer.state_dict()``. Only supported for a single optimizer. Defaults to None.
|
||||
"""
|
||||
if isinstance(optimizer, dict):
|
||||
# Handle dictionary of optimizers
|
||||
if optim_state_dict is not None:
|
||||
raise ValueError("optim_state_dict is not supported for a dict of optimizers")
|
||||
for name, opt in optimizer.items():
|
||||
optimizer_dir = save_dir / name
|
||||
optimizer_dir.mkdir(exist_ok=True, parents=True)
|
||||
_save_single_optimizer_state(opt, optimizer_dir)
|
||||
else:
|
||||
# Handle single optimizer
|
||||
_save_single_optimizer_state(optimizer, save_dir)
|
||||
_save_single_optimizer_state(optimizer, save_dir, optim_state_dict=optim_state_dict)
|
||||
|
||||
|
||||
def _save_single_optimizer_state(optimizer: torch.optim.Optimizer, save_dir: Path) -> None:
|
||||
def _save_single_optimizer_state(
|
||||
optimizer: torch.optim.Optimizer, save_dir: Path, optim_state_dict: dict | None = None
|
||||
) -> None:
|
||||
"""Save a single optimizer's state to disk."""
|
||||
state = optimizer.state_dict()
|
||||
state = dict(optim_state_dict) if optim_state_dict is not None else optimizer.state_dict()
|
||||
param_groups = state.pop("param_groups")
|
||||
flat_state = flatten_dict(state)
|
||||
save_file(flat_state, save_dir / OPTIMIZER_STATE)
|
||||
@@ -366,3 +365,19 @@ def _load_single_optimizer_state(optimizer: torch.optim.Optimizer, save_dir: Pat
|
||||
|
||||
optimizer.load_state_dict(loaded_state_dict)
|
||||
return optimizer
|
||||
|
||||
|
||||
def load_optimizer_state_dict(save_dir: Path) -> dict:
|
||||
"""Read a saved optimizer state dict (safetensors + json) back into a plain dict.
|
||||
|
||||
Unlike `load_optimizer_state`, this does not load into an optimizer and preserves the original
|
||||
``state`` keys verbatim (e.g. FSDP parameter FQNs, which are not integer-castable). It is used by
|
||||
the FSDP resume path, where the full state must be resharded via `FSDP.optim_state_dict_to_load`
|
||||
before being loaded into the (sharded) optimizer.
|
||||
"""
|
||||
flat_state = load_file(save_dir / OPTIMIZER_STATE)
|
||||
state = unflatten_dict(flat_state)
|
||||
return {
|
||||
"state": state.get("state", {}),
|
||||
"param_groups": load_json(save_dir / OPTIMIZER_PARAM_GROUPS),
|
||||
}
|
||||
|
||||
@@ -47,8 +47,6 @@ class ACTPolicy(PreTrainedPolicy):
|
||||
|
||||
config_class = ACTConfig
|
||||
name = "act"
|
||||
# FSDP2 wrap units: one unit per transformer layer of both stacks.
|
||||
_fsdp_wrap_modules = ["ACTEncoderLayer", "ACTDecoderLayer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -68,6 +68,8 @@ def euler_integrate(
|
||||
inference_delay: int | None = None,
|
||||
prev_chunk_left_over: Tensor | None = None,
|
||||
execution_horizon: int | None = None,
|
||||
hard_prefix: Tensor | None = None,
|
||||
hard_prefix_mask: Tensor | None = None,
|
||||
) -> Tensor:
|
||||
"""Forward-Euler integration of a velocity field from t=1 (noise) to t=0 (actions).
|
||||
|
||||
@@ -89,6 +91,8 @@ def euler_integrate(
|
||||
inference_delay: RTC guidance parameter, forwarded verbatim.
|
||||
prev_chunk_left_over: RTC guidance parameter, forwarded verbatim.
|
||||
execution_horizon: RTC guidance parameter, forwarded verbatim.
|
||||
hard_prefix: Optional clean action prefix to clamp throughout denoising.
|
||||
hard_prefix_mask: Boolean mask selecting the values clamped from ``hard_prefix``.
|
||||
"""
|
||||
bsize = noise.shape[0]
|
||||
device = noise.device
|
||||
@@ -99,6 +103,13 @@ def euler_integrate(
|
||||
time = 1.0 + step * dt
|
||||
time_tensor = torch.tensor(time, dtype=torch.float32, device=device).expand(bsize)
|
||||
|
||||
if hard_prefix is not None:
|
||||
if hard_prefix_mask is None:
|
||||
raise ValueError("hard_prefix_mask is required when hard_prefix is provided")
|
||||
x_t = torch.where(hard_prefix_mask, hard_prefix, x_t)
|
||||
time_tensor = time_tensor[:, None].expand(bsize, x_t.shape[1]).clone()
|
||||
time_tensor[hard_prefix_mask[..., 0]] = 0.0
|
||||
|
||||
def denoise_step_partial_call(input_x_t, current_timestep=time_tensor):
|
||||
return denoise_fn(input_x_t, current_timestep)
|
||||
|
||||
@@ -116,6 +127,9 @@ def euler_integrate(
|
||||
|
||||
x_t = x_t + dt * v_t
|
||||
|
||||
if hard_prefix is not None:
|
||||
x_t = torch.where(hard_prefix_mask, hard_prefix, x_t)
|
||||
|
||||
if rtc_processor is not None and rtc_processor.is_debug_enabled():
|
||||
rtc_processor.track(time=time, x_t=x_t, v_t=v_t)
|
||||
|
||||
|
||||
@@ -41,21 +41,20 @@ else:
|
||||
def create_sinusoidal_pos_embedding( # see openpi `create_sinusoidal_pos_embedding` (exact copy)
|
||||
time: torch.Tensor, dimension: int, min_period: float, max_period: float, device="cpu"
|
||||
) -> Tensor:
|
||||
"""Computes sine-cosine positional embedding vectors for scalar positions."""
|
||||
"""Compute sine-cosine embeddings for scalar or per-action positions."""
|
||||
if dimension % 2 != 0:
|
||||
raise ValueError(f"dimension ({dimension}) must be divisible by 2")
|
||||
|
||||
if time.ndim != 1:
|
||||
raise ValueError("The time tensor is expected to be of shape `(batch_size, )`.")
|
||||
if time.ndim not in (1, 2):
|
||||
raise ValueError("The time tensor must have shape (batch_size,) or (batch_size, action_horizon).")
|
||||
|
||||
dtype = get_safe_dtype(torch.float64, device.type)
|
||||
fraction = torch.linspace(0.0, 1.0, dimension // 2, dtype=dtype, device=device)
|
||||
period = min_period * (max_period / min_period) ** fraction
|
||||
|
||||
# Compute the outer product
|
||||
scaling_factor = 1.0 / period * 2 * math.pi
|
||||
sin_input = scaling_factor[None, :] * time[:, None]
|
||||
return torch.cat([torch.sin(sin_input), torch.cos(sin_input)], dim=1)
|
||||
sin_input = time[..., None] * scaling_factor
|
||||
return torch.cat([torch.sin(sin_input), torch.cos(sin_input)], dim=-1)
|
||||
|
||||
|
||||
def make_att_2d_masks(pad_masks: Tensor, att_masks: Tensor) -> Tensor: # see openpi (exact copy)
|
||||
|
||||
@@ -131,16 +131,12 @@ class ProcessorConfigKwargs(TypedDict, total=False):
|
||||
This provides type hints for the optional arguments passed to `make_pre_post_processors`,
|
||||
improving code clarity and enabling static analysis.
|
||||
|
||||
**Attributes**:
|
||||
- **preprocessor_config_filename** (`str | None`) -- The filename for the preprocessor configuration.
|
||||
- **postprocessor_config_filename** (`str | None`) -- The filename for the postprocessor
|
||||
configuration.
|
||||
- **preprocessor_overrides** (`dict[str, Any] | None`) -- A dictionary of overrides for the
|
||||
preprocessor configuration.
|
||||
- **postprocessor_overrides** (`dict[str, Any] | None`) -- A dictionary of overrides for the
|
||||
postprocessor configuration.
|
||||
- **dataset_stats** (`dict[str, dict[str, torch.Tensor]] | None`) -- Dataset statistics for
|
||||
normalization.
|
||||
Attributes:
|
||||
preprocessor_config_filename: The filename for the preprocessor configuration.
|
||||
postprocessor_config_filename: The filename for the postprocessor configuration.
|
||||
preprocessor_overrides: A dictionary of overrides for the preprocessor configuration.
|
||||
postprocessor_overrides: A dictionary of overrides for the postprocessor configuration.
|
||||
dataset_stats: Dataset statistics for normalization.
|
||||
"""
|
||||
|
||||
preprocessor_config_filename: str | None
|
||||
@@ -246,7 +242,6 @@ def make_policy(
|
||||
ds_meta: LeRobotDatasetMetadata | None = None,
|
||||
env_cfg: EnvConfig | None = None,
|
||||
rename_map: dict[str, str] | None = None,
|
||||
defer_weight_load: bool = False,
|
||||
) -> PreTrainedPolicy:
|
||||
"""
|
||||
Instantiate a policy model.
|
||||
@@ -257,27 +252,22 @@ def make_policy(
|
||||
can either initialize a new policy from scratch or load a pretrained one.
|
||||
|
||||
Args:
|
||||
cfg (PreTrainedConfig): The configuration for the policy to be created. If
|
||||
`cfg.pretrained_path` is set, the policy will be loaded with weights from that path.
|
||||
ds_meta (LeRobotDatasetMetadata | None): Dataset metadata used to infer feature shapes and
|
||||
types. Also provides statistics for normalization layers.
|
||||
env_cfg (EnvConfig | None): Environment configuration used to infer feature shapes and
|
||||
types. One of `ds_meta` or `env_cfg` must be provided.
|
||||
rename_map (dict[str, str] | None): Optional mapping of dataset or environment feature
|
||||
keys to match expected policy feature names (e.g., `"left"` → `"camera1"`).
|
||||
defer_weight_load (bool): Build the exact policy `from_pretrained` would build — same
|
||||
config resolution, same stats-derived buffers, same device placement and eval mode —
|
||||
but skip the safetensors weight load. Used when resuming from a DCP checkpoint, whose
|
||||
sharded weights stream in after `accelerator.prepare()` (the distributed checkpoint
|
||||
engine overwrites the random init).
|
||||
cfg: The configuration for the policy to be created. If `cfg.pretrained_path` is
|
||||
set, the policy will be loaded with weights from that path.
|
||||
ds_meta: Dataset metadata used to infer feature shapes and types. Also provides
|
||||
statistics for normalization layers.
|
||||
env_cfg: Environment configuration used to infer feature shapes and types.
|
||||
One of `ds_meta` or `env_cfg` must be provided.
|
||||
rename_map: Optional mapping of dataset or environment feature keys to match
|
||||
expected policy feature names (e.g., `"left"` → `"camera1"`).
|
||||
|
||||
Returns:
|
||||
PreTrainedPolicy: An instantiated and device-placed policy model.
|
||||
An instantiated and device-placed policy model.
|
||||
|
||||
Raises:
|
||||
ValueError: If both or neither of `ds_meta` and `env_cfg` are provided.
|
||||
NotImplementedError: If attempting to use an unsupported policy-backend combination
|
||||
(e.g., VQBeT with 'mps').
|
||||
NotImplementedError: If attempting to use an unsupported policy-backend
|
||||
combination (e.g., VQBeT with 'mps').
|
||||
"""
|
||||
if bool(ds_meta) == bool(env_cfg):
|
||||
raise ValueError("Either one of a dataset metadata or a sim env must be provided.")
|
||||
@@ -342,18 +332,11 @@ def make_policy(
|
||||
)
|
||||
|
||||
if cfg.pretrained_path and not cfg.use_peft:
|
||||
if defer_weight_load:
|
||||
# Same construction path as from_pretrained (config already resolved from the
|
||||
# checkpoint by the caller; dataset_stats/dataset_meta kwargs identical), minus the
|
||||
# weight load — parity by construction.
|
||||
policy = policy_cls(**kwargs)
|
||||
policy.eval()
|
||||
else:
|
||||
# Load a pretrained policy and override the config if needed (for example, if there
|
||||
# are inference-time hyperparameters that we want to vary).
|
||||
kwargs["pretrained_name_or_path"] = cfg.pretrained_path
|
||||
kwargs["revision"] = cfg.pretrained_revision
|
||||
policy = policy_cls.from_pretrained(**kwargs)
|
||||
# Load a pretrained policy and override the config if needed (for example, if there are inference-time
|
||||
# hyperparameters that we want to vary).
|
||||
kwargs["pretrained_name_or_path"] = cfg.pretrained_path
|
||||
kwargs["revision"] = cfg.pretrained_revision
|
||||
policy = policy_cls.from_pretrained(**kwargs)
|
||||
elif cfg.pretrained_path and cfg.use_peft:
|
||||
# Load a pretrained PEFT model on top of the policy. The pretrained path points to the folder/repo
|
||||
# of the adapter and the adapter's config contains the path to the base policy. So we need the
|
||||
|
||||
@@ -54,9 +54,6 @@ class FastWAMPolicy(PreTrainedPolicy):
|
||||
|
||||
config_class = FastWAMConfig
|
||||
name = "fastwam"
|
||||
# FSDP2 wrap units: MoTLayer is the single FSDP owner of each layer's expert blocks
|
||||
# (the blocks are re-parented onto it precisely so sharding has one boundary to hook).
|
||||
_fsdp_wrap_modules = ["MoTLayer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -58,6 +58,8 @@ class PI05Config(PreTrainedConfig):
|
||||
|
||||
# Real-Time Chunking (RTC) configuration
|
||||
rtc_config: RTCConfig | None = None
|
||||
# Maximum clean action-prefix length sampled during training. Zero disables trained RTC.
|
||||
rtc_training_max_delay: int = 0
|
||||
|
||||
image_resolution: tuple[int, int] = (
|
||||
DEFAULT_IMAGE_SIZE,
|
||||
@@ -101,8 +103,6 @@ class PI05Config(PreTrainedConfig):
|
||||
scheduler_decay_steps: int = 30_000
|
||||
scheduler_decay_lr: float = 2.5e-6
|
||||
|
||||
tokenizer_max_length: int = 200 # see openpi `__post_init__`
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
|
||||
@@ -111,6 +111,11 @@ class PI05Config(PreTrainedConfig):
|
||||
raise ValueError(
|
||||
f"n_action_steps ({self.n_action_steps}) cannot be greater than chunk_size ({self.chunk_size})"
|
||||
)
|
||||
if not 0 <= self.rtc_training_max_delay < self.chunk_size:
|
||||
raise ValueError(
|
||||
"rtc_training_max_delay must satisfy "
|
||||
f"0 <= delay < chunk_size ({self.chunk_size}), got {self.rtc_training_max_delay}"
|
||||
)
|
||||
|
||||
if self.paligemma_variant not in ["gemma_300m", "gemma_2b"]:
|
||||
raise ValueError(f"Invalid paligemma_variant: {self.paligemma_variant}")
|
||||
|
||||
@@ -71,6 +71,107 @@ class ActionSelectKwargs(TypedDict, total=False):
|
||||
execution_horizon: int | None
|
||||
|
||||
|
||||
def _prepare_trained_rtc_prefix(
|
||||
x_t: Tensor,
|
||||
prev_chunk_left_over: Tensor | None,
|
||||
inference_delay: int,
|
||||
training_max_delay: int,
|
||||
) -> tuple[Tensor | None, Tensor | None]:
|
||||
"""Pad and validate a hard prefix for training-time RTC inference."""
|
||||
if prev_chunk_left_over is None or inference_delay <= 0:
|
||||
return None, None
|
||||
if training_max_delay <= 0:
|
||||
raise ValueError(
|
||||
"RTC mode='trained' requires a checkpoint trained with policy.rtc_training_max_delay > 0."
|
||||
)
|
||||
if inference_delay > training_max_delay:
|
||||
raise ValueError(
|
||||
f"Measured RTC inference delay ({inference_delay}) exceeds the checkpoint's "
|
||||
f"rtc_training_max_delay ({training_max_delay})."
|
||||
)
|
||||
if inference_delay >= x_t.shape[1]:
|
||||
raise ValueError(
|
||||
f"RTC inference delay ({inference_delay}) must be smaller than chunk_size ({x_t.shape[1]})."
|
||||
)
|
||||
|
||||
previous = prev_chunk_left_over.to(device=x_t.device, dtype=x_t.dtype)
|
||||
if not torch.isfinite(previous).all():
|
||||
raise ValueError("RTC prefix contains NaN or Inf values.")
|
||||
if previous.ndim == 2:
|
||||
previous = previous.unsqueeze(0)
|
||||
if previous.ndim != 3:
|
||||
raise ValueError(f"Expected RTC prefix shape (B, T, A), got {tuple(previous.shape)}")
|
||||
if previous.shape[0] == 1 and x_t.shape[0] > 1:
|
||||
previous = previous.expand(x_t.shape[0], -1, -1)
|
||||
if previous.shape[0] != x_t.shape[0]:
|
||||
raise ValueError(
|
||||
f"RTC prefix batch size ({previous.shape[0]}) does not match policy batch ({x_t.shape[0]})."
|
||||
)
|
||||
if previous.shape[1] < inference_delay:
|
||||
raise ValueError(f"RTC prefix has {previous.shape[1]} steps, but inference_delay={inference_delay}.")
|
||||
if previous.shape[2] > x_t.shape[2]:
|
||||
raise ValueError(
|
||||
f"RTC prefix action dimension ({previous.shape[2]}) exceeds model dimension ({x_t.shape[2]})."
|
||||
)
|
||||
|
||||
padded_prefix = torch.zeros_like(x_t)
|
||||
padded_prefix[:, :inference_delay, : previous.shape[2]] = previous[:, :inference_delay]
|
||||
prefix_mask = torch.arange(x_t.shape[1], device=x_t.device) < inference_delay
|
||||
prefix_mask = prefix_mask[None, :, None].expand(x_t.shape[0], -1, x_t.shape[2])
|
||||
return padded_prefix, prefix_mask
|
||||
|
||||
|
||||
def _sample_training_rtc_prefix_mask(
|
||||
batch_size: int,
|
||||
action_horizon: int,
|
||||
max_delay: int,
|
||||
device: torch.device,
|
||||
) -> Tensor | None:
|
||||
"""Sample a clean action-prefix length independently for each training example."""
|
||||
if max_delay <= 0:
|
||||
return None
|
||||
delays = torch.randint(0, max_delay + 1, (batch_size,), device=device)
|
||||
positions = torch.arange(action_horizon, device=device)
|
||||
return positions.unsqueeze(0) < delays.unsqueeze(1)
|
||||
|
||||
|
||||
def _build_flow_matching_inputs(
|
||||
actions: Tensor,
|
||||
noise: Tensor,
|
||||
time: Tensor,
|
||||
prefix_mask: Tensor | None,
|
||||
) -> tuple[Tensor, Tensor]:
|
||||
"""Keep the sampled RTC prefix clean while noising the remaining action chunk."""
|
||||
if prefix_mask is None:
|
||||
model_time = time
|
||||
expanded_time = time[:, None, None]
|
||||
else:
|
||||
model_time = time[:, None].expand_as(prefix_mask)
|
||||
model_time = torch.where(prefix_mask, torch.zeros_like(model_time), model_time)
|
||||
expanded_time = model_time.unsqueeze(-1)
|
||||
x_t = expanded_time * noise + (1 - expanded_time) * actions
|
||||
return x_t, model_time
|
||||
|
||||
|
||||
def _reduce_training_rtc_loss(
|
||||
losses: Tensor,
|
||||
prefix_mask: Tensor | None,
|
||||
reduction: str,
|
||||
) -> Tensor:
|
||||
"""Average flow loss over predicted postfix actions, excluding the clean RTC prefix."""
|
||||
if reduction not in {"mean", "none"}:
|
||||
raise ValueError(f"Unsupported loss reduction: {reduction!r}")
|
||||
if prefix_mask is None:
|
||||
return losses.mean() if reduction == "mean" else losses.mean(dim=(1, 2))
|
||||
|
||||
postfix_mask = (~prefix_mask).unsqueeze(-1).expand_as(losses)
|
||||
if reduction == "none":
|
||||
numerator = (losses * postfix_mask).sum(dim=(1, 2))
|
||||
denominator = postfix_mask.sum(dim=(1, 2))
|
||||
return numerator / denominator.clamp(min=1)
|
||||
return (losses * postfix_mask).sum() / postfix_mask.sum().clamp(min=1)
|
||||
|
||||
|
||||
# Define the complete layer computation function for gradient checkpointing
|
||||
def compute_layer_complete(inputs_embeds, attention_mask, position_ids, adarms_cond, layers, rotary_emb):
|
||||
query_states = []
|
||||
@@ -561,14 +662,23 @@ class PI05Pytorch(nn.Module): # see openpi `PI0Pytorch`
|
||||
|
||||
return action_emb, pad_masks, att_masks, adarms_cond
|
||||
|
||||
def forward(self, images, img_masks, tokens, masks, actions, noise, time) -> Tensor:
|
||||
def forward(
|
||||
self,
|
||||
images,
|
||||
img_masks,
|
||||
tokens,
|
||||
masks,
|
||||
actions,
|
||||
noise,
|
||||
time,
|
||||
prefix_mask: Tensor | None = None,
|
||||
) -> Tensor:
|
||||
"""Do a full training forward pass and compute the loss."""
|
||||
time_expanded = time[:, None, None]
|
||||
x_t = time_expanded * noise + (1 - time_expanded) * actions
|
||||
x_t, model_time = _build_flow_matching_inputs(actions, noise, time, prefix_mask)
|
||||
u_t = noise - actions
|
||||
|
||||
prefix_embs, prefix_pad_masks, prefix_att_masks = self.embed_prefix(images, img_masks, tokens, masks)
|
||||
suffix_embs, suffix_pad_masks, suffix_att_masks, adarms_cond = self.embed_suffix(x_t, time)
|
||||
suffix_embs, suffix_pad_masks, suffix_att_masks, adarms_cond = self.embed_suffix(x_t, model_time)
|
||||
|
||||
if (
|
||||
self.paligemma_with_expert.paligemma.model.language_model.layers[0].self_attn.q_proj.weight.dtype
|
||||
@@ -652,6 +762,24 @@ class PI05Pytorch(nn.Module): # see openpi `PI0Pytorch`
|
||||
use_cache=True,
|
||||
)
|
||||
|
||||
rtc_mode = "guided"
|
||||
trained_prefix = trained_prefix_mask = None
|
||||
if self._rtc_enabled():
|
||||
rtc_mode = self.rtc_processor.rtc_config.mode
|
||||
if rtc_mode == "trained":
|
||||
training_max_delay = int(getattr(self.config, "rtc_training_max_delay", 0))
|
||||
if training_max_delay <= 0:
|
||||
raise ValueError(
|
||||
"RTC mode='trained' requires a checkpoint trained with "
|
||||
"policy.rtc_training_max_delay > 0."
|
||||
)
|
||||
trained_prefix, trained_prefix_mask = _prepare_trained_rtc_prefix(
|
||||
noise,
|
||||
kwargs.get("prev_chunk_left_over"),
|
||||
int(kwargs.get("inference_delay") or 0),
|
||||
training_max_delay,
|
||||
)
|
||||
|
||||
return euler_integrate(
|
||||
lambda input_x_t, current_timestep: self.denoise_step(
|
||||
prefix_pad_masks=prefix_pad_masks,
|
||||
@@ -662,10 +790,12 @@ class PI05Pytorch(nn.Module): # see openpi `PI0Pytorch`
|
||||
noise,
|
||||
num_steps,
|
||||
rtc_processor=self.rtc_processor,
|
||||
rtc_enabled=self._rtc_enabled(),
|
||||
rtc_enabled=self._rtc_enabled() and rtc_mode == "guided",
|
||||
inference_delay=kwargs.get("inference_delay"),
|
||||
prev_chunk_left_over=kwargs.get("prev_chunk_left_over"),
|
||||
execution_horizon=kwargs.get("execution_horizon"),
|
||||
hard_prefix=trained_prefix,
|
||||
hard_prefix_mask=trained_prefix_mask,
|
||||
)
|
||||
|
||||
def denoise_step(
|
||||
@@ -940,7 +1070,10 @@ class PI05Policy(PreTrainedPolicy):
|
||||
# Create processor if config provided
|
||||
# If RTC is not enabled - we can still track the denoising data
|
||||
if self.config.rtc_config is not None:
|
||||
self.rtc_processor = RTCProcessor(self.config.rtc_config)
|
||||
self.rtc_processor = RTCProcessor(
|
||||
self.config.rtc_config,
|
||||
trained_mode_supported=int(getattr(self.config, "rtc_training_max_delay", 0)) > 0,
|
||||
)
|
||||
|
||||
model_value = getattr(self, "model", None)
|
||||
if model_value is not None:
|
||||
@@ -1072,28 +1205,35 @@ class PI05Policy(PreTrainedPolicy):
|
||||
|
||||
noise = self.model.sample_noise(actions.shape, actions.device)
|
||||
time = self.model.sample_time(actions.shape[0], actions.device)
|
||||
prefix_mask = _sample_training_rtc_prefix_mask(
|
||||
actions.shape[0],
|
||||
actions.shape[1],
|
||||
self.config.rtc_training_max_delay,
|
||||
actions.device,
|
||||
)
|
||||
|
||||
# Compute loss (no separate state needed for PI05)
|
||||
losses = self.model.forward(images, img_masks, tokens, masks, actions, noise, time)
|
||||
losses = self.model.forward(images, img_masks, tokens, masks, actions, noise, time, prefix_mask)
|
||||
|
||||
# Truncate losses to actual action dimensions
|
||||
original_action_dim = self.config.output_features[ACTION].shape[0]
|
||||
losses = losses[:, :, :original_action_dim]
|
||||
|
||||
loss_dict = {
|
||||
"loss_per_dim": losses.mean(dim=[0, 1]).detach().cpu().numpy().tolist(),
|
||||
}
|
||||
if prefix_mask is None:
|
||||
loss_per_dim = losses.mean(dim=(0, 1))
|
||||
else:
|
||||
postfix_mask = (~prefix_mask).unsqueeze(-1).expand_as(losses)
|
||||
loss_per_dim = (losses * postfix_mask).sum(dim=(0, 1)) / postfix_mask.sum(dim=(0, 1)).clamp(min=1)
|
||||
loss_dict = {"loss_per_dim": loss_per_dim.detach().cpu().numpy().tolist()}
|
||||
|
||||
if reduction == "none":
|
||||
# Return per-sample losses (B,) by averaging over time and action dims
|
||||
per_sample_loss = losses.mean(dim=(1, 2))
|
||||
per_sample_loss = _reduce_training_rtc_loss(losses, prefix_mask, reduction="none")
|
||||
loss_dict["loss"] = per_sample_loss.mean().item()
|
||||
return per_sample_loss, loss_dict
|
||||
else:
|
||||
# Default: return scalar mean loss
|
||||
loss = losses.mean()
|
||||
loss_dict["loss"] = loss.item()
|
||||
return loss, loss_dict
|
||||
|
||||
loss = _reduce_training_rtc_loss(losses, prefix_mask, reduction="mean")
|
||||
loss_dict["loss"] = loss.item()
|
||||
return loss, loss_dict
|
||||
|
||||
def _get_default_peft_targets(self) -> dict[str, any]:
|
||||
"""Return default PEFT target modules for PI0.5 fine-tuning."""
|
||||
|
||||
@@ -87,6 +87,11 @@ class PiGemmaRMSNorm(nn.Module):
|
||||
Adaptive RMSNorm for PI Gemma (AdaRMS).
|
||||
When cond_dim is set, uses cond to modulate scale/shift/gate; otherwise behaves like standard GemmaRMSNorm.
|
||||
forward(x, cond=None) returns (output, gate) for use with _gated_residual.
|
||||
|
||||
``cond`` may be ``(batch_size, cond_dim)`` — one modulation shared by every token — or
|
||||
``(batch_size, seq_len, cond_dim)``, which lets scale/shift/gate differ per token. The
|
||||
per-token form is what training-time RTC needs to mark the clean action prefix with its
|
||||
own flow timestep (arXiv 2512.05964); it adds no parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None):
|
||||
@@ -120,8 +125,16 @@ class PiGemmaRMSNorm(nn.Module):
|
||||
return normed.type_as(x), None
|
||||
if cond.shape[-1] != self.cond_dim:
|
||||
raise ValueError(f"Expected cond dim {self.cond_dim}, got {cond.shape[-1]}")
|
||||
if cond.ndim not in (2, 3):
|
||||
raise ValueError(f"cond must be (B, cond_dim) or (B, T, cond_dim), got {tuple(cond.shape)}")
|
||||
if x.ndim == 3 and cond.ndim == 3 and cond.shape[1] != x.shape[1]:
|
||||
raise ValueError(
|
||||
f"Per-token cond has {cond.shape[1]} tokens but x has {x.shape[1]}; shapes must match."
|
||||
)
|
||||
modulation = self.dense(cond)
|
||||
if len(x.shape) == 3:
|
||||
if x.ndim == 3 and modulation.ndim == 2:
|
||||
# Scalar-per-sample cond: add the token axis so one modulation broadcasts over
|
||||
# all tokens. A per-token cond already carries that axis and must not gain another.
|
||||
modulation = modulation.unsqueeze(1)
|
||||
scale, shift, gate = modulation.chunk(3, dim=-1)
|
||||
normed = normed * (1 + scale.float()) + shift.float()
|
||||
@@ -224,8 +237,9 @@ class PiGemmaModel(GemmaModel): # type: ignore[misc]
|
||||
**kwargs,
|
||||
) -> BaseModelOutputWithPast:
|
||||
"""
|
||||
adarms_cond (`torch.Tensor` of shape `(batch_size, cond_dim)`, *optional*):
|
||||
Condition for ADARMS.
|
||||
adarms_cond (`torch.Tensor` of shape `(batch_size, cond_dim)` or
|
||||
`(batch_size, seq_len, cond_dim)`, *optional*):
|
||||
Condition for ADARMS. The per-token form drives training-time RTC.
|
||||
"""
|
||||
output_attentions = (
|
||||
output_attentions if output_attentions is not None else self.config.output_attentions
|
||||
|
||||
@@ -18,17 +18,20 @@ import builtins
|
||||
import dataclasses
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, TypedDict, TypeVar, Unpack
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, TypedDict, TypeVar, Unpack
|
||||
|
||||
from huggingface_hub import hf_hub_download, save_torch_state_dict
|
||||
from huggingface_hub import HfApi, ModelCard, ModelCardData, hf_hub_download, save_torch_state_dict
|
||||
from huggingface_hub.constants import SAFETENSORS_SINGLE_FILE
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
from safetensors.torch import load_model as load_model_as_safetensor
|
||||
from safetensors.torch import load_model as load_model_as_safetensor, save_model as save_model_as_safetensor
|
||||
from torch import Tensor, nn
|
||||
|
||||
from lerobot.__version__ import __version__
|
||||
from lerobot.configs import PreTrainedConfig
|
||||
from lerobot.configs.train import TrainPipelineConfig
|
||||
from lerobot.utils.device_utils import resolve_safetensors_device
|
||||
from lerobot.utils.hub import HubMixin
|
||||
from lerobot.utils.import_utils import _peft_available, require_package
|
||||
@@ -43,14 +46,56 @@ else:
|
||||
get_peft_model = None
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lerobot.configs.train import TrainPipelineConfig
|
||||
from lerobot.datasets.dataset_metadata import LeRobotDatasetMetadata
|
||||
|
||||
T = TypeVar("T", bound="PreTrainedPolicy")
|
||||
|
||||
# Pinned far above any policy's total size so save_torch_state_dict always emits exactly one
|
||||
# `model.safetensors` (no shards, no index) — a constant, not a computed byte count.
|
||||
_SINGLE_FILE_SHARD_SIZE = "1TB"
|
||||
|
||||
def _build_card_context(
|
||||
cfg: TrainPipelineConfig | None,
|
||||
dataset_meta: LeRobotDatasetMetadata | None,
|
||||
input_features: dict | None,
|
||||
output_features: dict | None,
|
||||
) -> dict:
|
||||
"""Collect optional data for the model-card template.
|
||||
|
||||
Returns plain values only (no Markdown) — the template in
|
||||
``lerobot/templates/lerobot_modelcard_template.md`` decides how and whether to show
|
||||
each one. Everything is best-effort: anything unavailable is left empty/None and the
|
||||
template simply skips that section, so this never breaks a Hub push.
|
||||
"""
|
||||
context = {
|
||||
"training": None,
|
||||
"input_features": input_features or {},
|
||||
"output_features": output_features or {},
|
||||
"dataset": None,
|
||||
"robot_type": None,
|
||||
"cameras": [],
|
||||
}
|
||||
|
||||
if cfg is not None:
|
||||
optimizer = getattr(cfg, "optimizer", None)
|
||||
context["training"] = {
|
||||
"steps": cfg.steps,
|
||||
"batch_size": cfg.batch_size,
|
||||
"seed": cfg.seed,
|
||||
"optimizer": getattr(optimizer, "type", None) if optimizer else None,
|
||||
"lr": getattr(optimizer, "lr", None) if optimizer else None,
|
||||
"lerobot_version": __version__,
|
||||
}
|
||||
|
||||
if dataset_meta is not None:
|
||||
context["dataset"] = {
|
||||
"repo_id": dataset_meta.repo_id,
|
||||
"episodes": dataset_meta.total_episodes,
|
||||
"frames": dataset_meta.total_frames,
|
||||
"fps": dataset_meta.fps,
|
||||
"tasks": [str(task) for task in dataset_meta.tasks.index],
|
||||
}
|
||||
context["robot_type"] = dataset_meta.robot_type
|
||||
context["cameras"] = [key.split(".")[-1] for key in dataset_meta.camera_keys]
|
||||
|
||||
return context
|
||||
|
||||
|
||||
class ActionSelectKwargs(TypedDict, total=False):
|
||||
@@ -65,22 +110,6 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):
|
||||
config_class: None
|
||||
name: None
|
||||
|
||||
# --- declarative parallelism/acceleration surface ----------------------------------------
|
||||
# Module CLASS names forming the FSDP2 wrap units (and, once wired, the activation-
|
||||
# checkpointing units). Resolved onto the accelerate plugin right before
|
||||
# `accelerator.prepare()` by `lerobot.distributed.set_fsdp_wrap_modules`; sharded training
|
||||
# with no wrap source anywhere fails loudly instead of silently wrapping only the root.
|
||||
_fsdp_wrap_modules: ClassVar[list[str] | None] = None
|
||||
# Non-`forward` entry points that must trigger FSDP2 unshard/reshard hooks when called on a
|
||||
# sharded policy (registered post-prepare via `torch.distributed.fsdp
|
||||
# .register_fsdp_forward_method`); calling them unregistered crashes on mixed Tensor/DTensor.
|
||||
_fsdp_forward_methods: ClassVar[tuple[str, ...]] = ("select_action", "predict_action_chunk")
|
||||
# Capability gate for the (future) activation-checkpointing wiring.
|
||||
supports_gradient_checkpointing: ClassVar[bool] = False
|
||||
# Declarative context-parallel plan (diffusers `ContextParallelModelPlan` semantics:
|
||||
# module FQN -> sequence split/gather spec). Reserved for the CP engine round.
|
||||
_cp_plan: ClassVar[dict[str, Any] | None] = None
|
||||
|
||||
def __init__(self, config: PreTrainedConfig, *inputs, **kwargs):
|
||||
super().__init__()
|
||||
if not isinstance(config, PreTrainedConfig):
|
||||
@@ -98,33 +127,43 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):
|
||||
if not getattr(cls, "name", None):
|
||||
raise TypeError(f"Class {cls.__name__} must define 'name'")
|
||||
|
||||
def _save_pretrained(self, save_directory: Path) -> None:
|
||||
"""Serialize this policy's parameters (and config) into `save_directory`.
|
||||
def save_pretrained(
|
||||
self,
|
||||
save_directory: str | Path,
|
||||
*,
|
||||
state_dict: dict[str, Tensor] | None = None,
|
||||
repo_id: str | None = None,
|
||||
push_to_hub: bool = False,
|
||||
card_kwargs: dict | None = None,
|
||||
**push_to_hub_kwargs,
|
||||
) -> str | None:
|
||||
"""Save the policy to a directory (and optionally push to the Hub).
|
||||
|
||||
Sharding is handled internally: under FSDP2 the full state dict is gathered through a
|
||||
COLLECTIVE, so when the policy is sharded this method (via `save_pretrained`) must be
|
||||
called on EVERY rank — a rank-0-gated call deadlocks. File writes happen on the main
|
||||
process only, in all layouts (single, DDP, sharded).
|
||||
|
||||
Args:
|
||||
save_directory (Path): Target directory for the policy config (`config.json`) and the
|
||||
safetensors weight file(s).
|
||||
Overrides `HubMixin.save_pretrained` to add a `state_dict` argument (mirroring
|
||||
`transformers.PreTrainedModel.save_pretrained`). Under FSDP, `self.state_dict()` would
|
||||
return sharded tensors, so the caller gathers the full state dict via a cross-rank
|
||||
collective and passes it here for `_save_pretrained` to write directly.
|
||||
"""
|
||||
# Lazy imports: the persistence layer pulls in lerobot.distributed only when saving.
|
||||
from lerobot.distributed.checkpoint import full_model_state_dict, is_sharded_module
|
||||
from lerobot.distributed.utils import is_main_process
|
||||
save_directory = Path(save_directory)
|
||||
save_directory.mkdir(parents=True, exist_ok=True)
|
||||
self._save_pretrained(save_directory, state_dict=state_dict)
|
||||
if push_to_hub:
|
||||
if repo_id is None:
|
||||
repo_id = save_directory.name
|
||||
return self.push_to_hub(repo_id=repo_id, card_kwargs=card_kwargs, **push_to_hub_kwargs)
|
||||
return None
|
||||
|
||||
model_to_save = self.module if hasattr(self, "module") else self
|
||||
if is_sharded_module(model_to_save):
|
||||
logging.info("Gathering the full state dict from all ranks (sharded policy).")
|
||||
state_dict = full_model_state_dict(model_to_save) # collective when sharded; {} off-main
|
||||
if not state_dict or not is_main_process():
|
||||
# Sharded: the gather materializes on the main rank only (emptiness check).
|
||||
# Non-sharded multi-rank (DDP): every rank holds a full dict — the explicit rank
|
||||
# gate prevents N ranks racing on the same files. Single process: never taken.
|
||||
return
|
||||
def _save_pretrained(self, save_directory: Path, state_dict: dict[str, Tensor] | None = None) -> None:
|
||||
self.config._save_pretrained(save_directory)
|
||||
save_torch_state_dict(state_dict, str(save_directory), max_shard_size=_SINGLE_FILE_SHARD_SIZE)
|
||||
model_to_save = self.module if hasattr(self, "module") else self
|
||||
if state_dict is None:
|
||||
save_model_as_safetensor(model_to_save, str(save_directory / SAFETENSORS_SINGLE_FILE))
|
||||
return
|
||||
# A pre-gathered (e.g. FSDP full) state dict was supplied: write it directly.
|
||||
# `save_torch_state_dict` discards shared-tensor duplicates just like `save_model` does;
|
||||
# pin `max_shard_size` above the total size so the output stays a single `model.safetensors`
|
||||
total_bytes = sum(t.numel() * t.element_size() for t in state_dict.values())
|
||||
save_torch_state_dict(state_dict, str(save_directory), max_shard_size=max(total_bytes, 1))
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
@@ -252,39 +291,92 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):
|
||||
peft_model=None,
|
||||
state_dict: dict[str, Tensor] | None = None,
|
||||
dataset_meta: LeRobotDatasetMetadata | None = None,
|
||||
) -> None:
|
||||
"""Publish this policy to the Hub.
|
||||
):
|
||||
api = HfApi()
|
||||
repo_id = api.create_repo(
|
||||
repo_id=self.config.repo_id, private=self.config.private, exist_ok=True
|
||||
).repo_id
|
||||
|
||||
Deprecated: use :func:`lerobot.common.train_utils.publish_trained_model` instead, which
|
||||
also publishes the pre/post-processors alongside the model.
|
||||
# Push the files to the repo in a single commit
|
||||
with TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
|
||||
saved_path = Path(tmp) / repo_id
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The training config; saved as `train_config.json` and
|
||||
used to render the model card.
|
||||
peft_model: The PEFT wrapper when training adapters, whose weights replace the full
|
||||
model weights in the published repo. Defaults to None.
|
||||
state_dict (dict[str, Tensor] | None): Ignored; weights are now gathered internally
|
||||
when the policy is sharded. Defaults to None.
|
||||
dataset_meta (LeRobotDatasetMetadata | None): Dataset metadata for the model card,
|
||||
if available. Defaults to None.
|
||||
"""
|
||||
from lerobot.common.train_utils import publish_trained_model
|
||||
if peft_model is not None:
|
||||
# Since PEFT just forwards calls to `push_model_to_hub`, `self` is not the PeftModel wrapper
|
||||
# but the actual policy which is why we need the PEFT model passed to us to save the adapter.
|
||||
# That also means that we need to store the policy config ourselves since PEFT can't.
|
||||
peft_model.save_pretrained(saved_path)
|
||||
self.config.save_pretrained(saved_path)
|
||||
else:
|
||||
# Calls _save_pretrained and stores model tensors
|
||||
self.save_pretrained(saved_path, state_dict=state_dict)
|
||||
|
||||
warnings.warn(
|
||||
"PreTrainedPolicy.push_model_to_hub is deprecated and will be removed in a future "
|
||||
"version. Use lerobot.common.train_utils.publish_trained_model(cfg, model, "
|
||||
"preprocessor, postprocessor, dataset_meta) instead.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if state_dict is not None:
|
||||
warnings.warn(
|
||||
"The `state_dict` argument is ignored: sharded weights are gathered internally "
|
||||
"when the policy is saved.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
card = self.generate_model_card(
|
||||
cfg.dataset.repo_id,
|
||||
self.config.type,
|
||||
self.config.license,
|
||||
self.config.tags,
|
||||
cfg=cfg,
|
||||
dataset_meta=dataset_meta,
|
||||
)
|
||||
publish_trained_model(cfg, self, None, None, dataset_meta, peft_model=peft_model)
|
||||
card.save(str(saved_path / "README.md"))
|
||||
|
||||
cfg.save_pretrained(saved_path) # Calls _save_pretrained and stores train config
|
||||
|
||||
commit_info = api.upload_folder(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
folder_path=saved_path,
|
||||
commit_message="Upload policy weights, train config and readme",
|
||||
allow_patterns=["*.safetensors", "*.json", "*.yaml", "*.md"],
|
||||
ignore_patterns=["*.tmp", "*.log"],
|
||||
)
|
||||
|
||||
# Contract: lerobot.jobs.hf.submit_to_hf watches for this exact
|
||||
# "Model pushed to <url>" line to end a remote run early. Keep the wording
|
||||
# and URL format in sync (it falls back to status polling if they drift).
|
||||
logging.info(f"Model pushed to {commit_info.repo_url.url}")
|
||||
|
||||
def generate_model_card(
|
||||
self,
|
||||
dataset_repo_id: str,
|
||||
model_type: str,
|
||||
license: str | None,
|
||||
tags: list[str] | None,
|
||||
cfg: TrainPipelineConfig | None = None,
|
||||
dataset_meta: LeRobotDatasetMetadata | None = None,
|
||||
) -> ModelCard:
|
||||
base_model_mapping = {
|
||||
"smolvla": "lerobot/smolvla_base",
|
||||
"pi0": "lerobot/pi0_base",
|
||||
"pi05": "lerobot/pi05_base",
|
||||
"pi0_fast": "lerobot/pi0fast-base",
|
||||
"xvla": "lerobot/xvla-base",
|
||||
}
|
||||
|
||||
card_data = ModelCardData(
|
||||
license=license or "apache-2.0",
|
||||
library_name="lerobot",
|
||||
pipeline_tag="robotics",
|
||||
tags=list(set(tags or []).union({"robotics", "lerobot", model_type})),
|
||||
model_name=model_type,
|
||||
datasets=dataset_repo_id,
|
||||
base_model=base_model_mapping.get(model_type),
|
||||
)
|
||||
|
||||
context = _build_card_context(
|
||||
cfg, dataset_meta, self.config.input_features, self.config.output_features
|
||||
)
|
||||
# Used by the template to pre-fill commands and the "Fine-tuned from" line.
|
||||
context["policy_repo_id"] = getattr(self.config, "repo_id", None)
|
||||
context["base_model"] = base_model_mapping.get(model_type)
|
||||
|
||||
template_card = (
|
||||
files("lerobot.templates").joinpath("lerobot_modelcard_template.md").read_text(encoding="utf-8")
|
||||
)
|
||||
card = ModelCard.from_template(card_data, template_str=template_card, **context)
|
||||
card.validate()
|
||||
return card
|
||||
|
||||
def wrap_with_peft(
|
||||
self,
|
||||
|
||||
@@ -46,11 +46,10 @@ class ActionQueue:
|
||||
Args:
|
||||
cfg (RTCConfig): Configuration for Real-Time Chunking behavior.
|
||||
|
||||
**Attributes**:
|
||||
- **queue** (`Tensor | None`) -- Processed actions for robot rollout (time_steps, action_dim).
|
||||
- **original_queue** (`Tensor | None`) -- Original actions for RTC computation (time_steps,
|
||||
action_dim).
|
||||
- **last_index** (`int`) -- Current consumption index in the queue.
|
||||
Attributes:
|
||||
queue (Tensor | None): Processed actions for robot rollout (time_steps, action_dim).
|
||||
original_queue (Tensor | None): Original actions for RTC computation (time_steps, action_dim).
|
||||
last_index (int): Current consumption index in the queue.
|
||||
"""
|
||||
|
||||
def __init__(self, cfg: RTCConfig):
|
||||
|
||||
@@ -37,6 +37,10 @@ class RTCConfig:
|
||||
# Infrastructure
|
||||
enabled: bool = True
|
||||
|
||||
# ``guided`` is the original inference-time Jacobian guidance. ``trained``
|
||||
# hard-inpaints a prefix and requires a compatible training-time RTC checkpoint.
|
||||
mode: str = "guided"
|
||||
|
||||
# Core RTC settings
|
||||
# Todo change to exp
|
||||
prefix_attention_schedule: RTCAttentionSchedule = RTCAttentionSchedule.LINEAR
|
||||
@@ -49,6 +53,8 @@ class RTCConfig:
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate RTC configuration parameters."""
|
||||
if self.mode not in {"guided", "trained"}:
|
||||
raise ValueError(f"mode must be 'guided' or 'trained', got {self.mode!r}")
|
||||
if self.max_guidance_weight <= 0:
|
||||
raise ValueError(f"max_guidance_weight must be positive, got {self.max_guidance_weight}")
|
||||
if self.debug_maxlen <= 0:
|
||||
|
||||
@@ -27,19 +27,19 @@ from torch import Tensor
|
||||
class DebugStep:
|
||||
"""Container for debug information from a single denoising step.
|
||||
|
||||
**Attributes**:
|
||||
- **step_idx** (`int`) -- Step index/counter.
|
||||
- **x_t** (`Tensor | None`) -- Current latent/state tensor.
|
||||
- **v_t** (`Tensor | None`) -- Velocity from denoiser.
|
||||
- **x1_t** (`Tensor | None`) -- Denoised prediction (x_t - time * v_t).
|
||||
- **correction** (`Tensor | None`) -- Correction gradient tensor.
|
||||
- **err** (`Tensor | None`) -- Weighted error term.
|
||||
- **weights** (`Tensor | None`) -- Prefix attention weights.
|
||||
- **guidance_weight** (`float | Tensor | None`) -- Applied guidance weight.
|
||||
- **time** (`float | Tensor | None`) -- Time parameter.
|
||||
- **inference_delay** (`int | None`) -- Inference delay parameter.
|
||||
- **execution_horizon** (`int | None`) -- Execution horizon parameter.
|
||||
- **metadata** (`dict[str, Any]`) -- Additional metadata.
|
||||
Attributes:
|
||||
step_idx (int): Step index/counter.
|
||||
x_t (Tensor | None): Current latent/state tensor.
|
||||
v_t (Tensor | None): Velocity from denoiser.
|
||||
x1_t (Tensor | None): Denoised prediction (x_t - time * v_t).
|
||||
correction (Tensor | None): Correction gradient tensor.
|
||||
err (Tensor | None): Weighted error term.
|
||||
weights (Tensor | None): Prefix attention weights.
|
||||
guidance_weight (float | Tensor | None): Applied guidance weight.
|
||||
time (float | Tensor | None): Time parameter.
|
||||
inference_delay (int | None): Inference delay parameter.
|
||||
execution_horizon (int | None): Execution horizon parameter.
|
||||
metadata (dict[str, Any]): Additional metadata.
|
||||
"""
|
||||
|
||||
step_idx: int = 0
|
||||
|
||||
@@ -42,7 +42,12 @@ class RTCProcessor:
|
||||
prefix attention, and adaptive chunk processing.
|
||||
"""
|
||||
|
||||
def __init__(self, rtc_config: RTCConfig):
|
||||
def __init__(self, rtc_config: RTCConfig, *, trained_mode_supported: bool = False):
|
||||
if rtc_config.enabled and rtc_config.mode == "trained" and not trained_mode_supported:
|
||||
raise ValueError(
|
||||
"RTC mode='trained' requires a PI05-compatible checkpoint trained with "
|
||||
"rtc_training_max_delay > 0."
|
||||
)
|
||||
self.rtc_config = rtc_config
|
||||
|
||||
self.tracker = None
|
||||
|
||||
@@ -217,12 +217,10 @@ class AddBatchDimensionProcessorStep(ProcessorStep):
|
||||
This step combines individual processors for actions, observations, and complementary data
|
||||
to create a batched transition (batch size 1) from a single-instance transition.
|
||||
|
||||
**Attributes**:
|
||||
- **to_batch_action_processor** (`AddBatchDimensionActionStep`) -- Processor for the action component.
|
||||
- **to_batch_observation_processor** (`AddBatchDimensionObservationStep`) -- Processor for the
|
||||
observation component.
|
||||
- **to_batch_complementary_data_processor** (`AddBatchDimensionComplementaryDataStep`) -- Processor
|
||||
for the complementary data component.
|
||||
Attributes:
|
||||
to_batch_action_processor: Processor for the action component.
|
||||
to_batch_observation_processor: Processor for the observation component.
|
||||
to_batch_complementary_data_processor: Processor for the complementary data component.
|
||||
"""
|
||||
|
||||
to_batch_action_processor: AddBatchDimensionActionStep = field(
|
||||
|
||||
@@ -32,8 +32,9 @@ class MapTensorToDeltaActionDictStep(ActionProcessorStep):
|
||||
It decomposes the vector into named components for delta movements of the
|
||||
end-effector (x, y, z) and optionally the gripper.
|
||||
|
||||
**Attributes**:
|
||||
- **use_gripper** (`bool`) -- If True, assumes the 4th element of the tensor is the gripper action.
|
||||
Attributes:
|
||||
use_gripper: If True, assumes the 4th element of the tensor is the
|
||||
gripper action.
|
||||
"""
|
||||
|
||||
use_gripper: bool = True
|
||||
@@ -80,10 +81,10 @@ class MapDeltaActionToRobotActionStep(RobotActionProcessorStep):
|
||||
into a target action format that includes an "enabled" flag and target
|
||||
end-effector positions. It also handles scaling and noise filtering.
|
||||
|
||||
**Attributes**:
|
||||
- **position_scale** (`float`) -- A factor to scale the delta position inputs.
|
||||
- **noise_threshold** (`float`) -- The magnitude below which delta inputs are considered noise and do
|
||||
not trigger an "enabled" state.
|
||||
Attributes:
|
||||
position_scale: A factor to scale the delta position inputs.
|
||||
noise_threshold: The magnitude below which delta inputs are considered noise
|
||||
and do not trigger an "enabled" state.
|
||||
"""
|
||||
|
||||
# Scale factors for delta movements
|
||||
|
||||
@@ -40,10 +40,10 @@ class DeviceProcessorStep(ProcessorStep):
|
||||
|
||||
This is crucial for preparing data for model training or inference on hardware like GPUs.
|
||||
|
||||
**Attributes**:
|
||||
- **device** (`str`) -- The target device for tensors (e.g., "cpu", "cuda", "cuda:0").
|
||||
- **float_dtype** (`str | None`) -- The target floating-point dtype as a string (e.g., "float32",
|
||||
"float16", "bfloat16"). If None, the dtype is not changed.
|
||||
Attributes:
|
||||
device: The target device for tensors (e.g., "cpu", "cuda", "cuda:0").
|
||||
float_dtype: The target floating-point dtype as a string (e.g., "float32", "float16", "bfloat16").
|
||||
If None, the dtype is not changed.
|
||||
"""
|
||||
|
||||
device: str = "cpu"
|
||||
|
||||
@@ -33,9 +33,10 @@ class Torch2NumpyActionProcessorStep(ActionProcessorStep):
|
||||
This step is useful when the output of a policy (typically a torch.Tensor)
|
||||
needs to be passed to an environment or component that expects a NumPy array.
|
||||
|
||||
**Attributes**:
|
||||
- **squeeze_batch_dim** (`bool`) -- If True, removes the first dimension of the array if it is of size
|
||||
1. This is useful for converting a batched action of size (1, D) to a single action of size (D,).
|
||||
Attributes:
|
||||
squeeze_batch_dim: If True, removes the first dimension of the array
|
||||
if it is of size 1. This is useful for converting a
|
||||
batched action of size (1, D) to a single action of size (D,).
|
||||
"""
|
||||
|
||||
squeeze_batch_dim: bool = True
|
||||
|
||||
@@ -101,8 +101,8 @@ class AddTeleopActionAsComplimentaryDataStep(ComplementaryDataProcessorStep):
|
||||
be available to downstream processors, for example, to override a policy's action
|
||||
during an intervention.
|
||||
|
||||
**Attributes**:
|
||||
- **teleop_device** (`Teleoperator`) -- The teleoperator instance to get the action from.
|
||||
Attributes:
|
||||
teleop_device: The teleoperator instance to get the action from.
|
||||
"""
|
||||
|
||||
teleop_device: "Teleoperator"
|
||||
@@ -137,9 +137,9 @@ class AddTeleopEventsAsInfoStep(InfoProcessorStep):
|
||||
This step extracts control events from teleoperators that support event-based
|
||||
interaction, making these signals available to other parts of the system.
|
||||
|
||||
**Attributes**:
|
||||
- **teleop_device** (`TeleopWithEvents`) -- An instance of a teleoperator that implements the
|
||||
`HasTeleopEvents` protocol.
|
||||
Attributes:
|
||||
teleop_device: An instance of a teleoperator that implements the
|
||||
`HasTeleopEvents` protocol.
|
||||
"""
|
||||
|
||||
teleop_device: TeleopWithEvents
|
||||
@@ -180,10 +180,10 @@ class ImageCropResizeProcessorStep(ObservationProcessorStep):
|
||||
the specified transformations. It handles device placement, moving tensors to the
|
||||
CPU if necessary for operations not supported on certain accelerators like MPS.
|
||||
|
||||
**Attributes**:
|
||||
- **crop_params_dict** (`dict[str, tuple[int, int, int, int]] | None`) -- A dictionary mapping image
|
||||
keys to cropping parameters (top, left, height, width).
|
||||
- **resize_size** (`tuple[int, int] | None`) -- A tuple (height, width) to resize all images to.
|
||||
Attributes:
|
||||
crop_params_dict: A dictionary mapping image keys to cropping parameters
|
||||
(top, left, height, width).
|
||||
resize_size: A tuple (height, width) to resize all images to.
|
||||
"""
|
||||
|
||||
crop_params_dict: dict[str, tuple[int, int, int, int]] | None = None
|
||||
@@ -267,9 +267,9 @@ class TimeLimitProcessorStep(TruncatedProcessorStep):
|
||||
"""
|
||||
Tracks episode steps and enforces a time limit by truncating the episode.
|
||||
|
||||
**Attributes**:
|
||||
- **max_episode_steps** (`int`) -- The maximum number of steps allowed per episode.
|
||||
- **current_step** (`int`) -- The current step count for the active episode.
|
||||
Attributes:
|
||||
max_episode_steps: The maximum number of steps allowed per episode.
|
||||
current_step: The current step count for the active episode.
|
||||
"""
|
||||
|
||||
max_episode_steps: int
|
||||
@@ -358,11 +358,11 @@ class GripperPenaltyProcessorStep(ProcessorStep):
|
||||
This discourages gripper oscillation while leaving "stay" and saturating-further
|
||||
commands unpenalized.
|
||||
|
||||
**Attributes**:
|
||||
- **penalty** (`float`) -- The negative reward value to apply.
|
||||
- **max_gripper_pos** (`float`) -- The maximum position value for the gripper, used for normalization.
|
||||
- **open_threshold** (`float`) -- Normalized state below which the gripper is considered "open".
|
||||
- **closed_threshold** (`float`) -- Normalized state above which the gripper is considered "closed".
|
||||
Attributes:
|
||||
penalty: The negative reward value to apply.
|
||||
max_gripper_pos: The maximum position value for the gripper, used for normalization.
|
||||
open_threshold: Normalized state below which the gripper is considered "open".
|
||||
closed_threshold: Normalized state above which the gripper is considered "closed".
|
||||
"""
|
||||
|
||||
penalty: float = -0.02
|
||||
@@ -456,10 +456,10 @@ class InterventionActionProcessorStep(ProcessorStep):
|
||||
this step replaces the policy's action with the human's teleoperated action.
|
||||
It also processes signals to terminate the episode or flag success.
|
||||
|
||||
**Attributes**:
|
||||
- **use_gripper** (`bool`) -- Whether to include the gripper in the teleoperated action.
|
||||
- **terminate_on_success** (`bool`) -- If True, automatically sets the `done` flag when a `success`
|
||||
event is received.
|
||||
Attributes:
|
||||
use_gripper: Whether to include the gripper in the teleoperated action.
|
||||
terminate_on_success: If True, automatically sets the `done` flag when a
|
||||
`success` event is received.
|
||||
"""
|
||||
|
||||
use_gripper: bool = False
|
||||
@@ -557,13 +557,13 @@ class RewardClassifierProcessorStep(ProcessorStep):
|
||||
This step uses a model to determine if the current state is successful, updating
|
||||
the reward and potentially terminating the episode.
|
||||
|
||||
**Attributes**:
|
||||
- **pretrained_path** (`str | None`) -- Path to the pretrained reward classifier model.
|
||||
- **device** (`str`) -- The device to run the classifier on.
|
||||
- **success_threshold** (`float`) -- The probability threshold to consider a prediction as successful.
|
||||
- **success_reward** (`float`) -- The reward value to assign on success.
|
||||
- **terminate_on_success** (`bool`) -- If True, terminates the episode upon successful classification.
|
||||
- **reward_classifier** (`Any`) -- The loaded classifier model instance.
|
||||
Attributes:
|
||||
pretrained_path: Path to the pretrained reward classifier model.
|
||||
device: The device to run the classifier on.
|
||||
success_threshold: The probability threshold to consider a prediction as successful.
|
||||
success_reward: The reward value to assign on success.
|
||||
terminate_on_success: If True, terminates the episode upon successful classification.
|
||||
reward_classifier: The loaded classifier model instance.
|
||||
"""
|
||||
|
||||
pretrained_path: str | None = None
|
||||
|
||||
@@ -647,15 +647,10 @@ def main():
|
||||
tags = set(tags).union({"robotics", "lerobot", policy_type})
|
||||
tags = list(tags)
|
||||
|
||||
# Generate model card through the free helper (PreTrainedPolicy.generate_model_card was
|
||||
# removed with the publisher redesign), then apply the metadata recovered above — the
|
||||
# migrated policy config does not carry the original repo's card fields.
|
||||
from lerobot.common.train_utils import generate_model_card
|
||||
|
||||
card = generate_model_card(policy.config)
|
||||
card.data.datasets = dataset_repo_id
|
||||
card.data.license = license
|
||||
card.data.tags = sorted(tags)
|
||||
# Generate model card
|
||||
card = policy.generate_model_card(
|
||||
dataset_repo_id=dataset_repo_id, model_type=policy_type, license=license, tags=tags
|
||||
)
|
||||
|
||||
# Save model card locally
|
||||
card.save(str(output_dir / "README.md"))
|
||||
|
||||
@@ -71,23 +71,22 @@ class _NormalizationMixin:
|
||||
)
|
||||
```
|
||||
|
||||
**Attributes**:
|
||||
- **features** (`dict[str, PolicyFeature]`) -- A dictionary mapping feature names to `PolicyFeature`
|
||||
objects, defining the data structure to be processed.
|
||||
- **norm_map** (`dict[FeatureType, NormalizationMode]`) -- A dictionary mapping `FeatureType` to
|
||||
`NormalizationMode`, specifying which normalization method to use for each type of feature.
|
||||
- **stats** (`dict[str, dict[str, Any]] | None`) -- A dictionary containing the normalization
|
||||
statistics (e.g., mean, std, min, max) for each feature.
|
||||
- **device** (`torch.device | str | None`) -- The PyTorch device on which to store and perform tensor
|
||||
operations.
|
||||
- **eps** (`float`) -- A small epsilon value to prevent division by zero in normalization
|
||||
calculations.
|
||||
- **normalize_observation_keys** (`set[str] | None`) -- An optional set of keys to selectively apply
|
||||
normalization to specific observation features.
|
||||
- **_tensor_stats** (`dict[str, dict[str, Tensor]]`) -- An internal dictionary holding the
|
||||
normalization statistics as PyTorch tensors.
|
||||
- **_stats_explicitly_provided** (`bool`) -- Internal flag tracking whether stats were explicitly
|
||||
provided during construction (used for override preservation).
|
||||
Attributes:
|
||||
features: A dictionary mapping feature names to `PolicyFeature` objects, defining
|
||||
the data structure to be processed.
|
||||
norm_map: A dictionary mapping `FeatureType` to `NormalizationMode`, specifying
|
||||
which normalization method to use for each type of feature.
|
||||
stats: A dictionary containing the normalization statistics (e.g., mean, std,
|
||||
min, max) for each feature.
|
||||
device: The PyTorch device on which to store and perform tensor operations.
|
||||
eps: A small epsilon value to prevent division by zero in normalization
|
||||
calculations.
|
||||
normalize_observation_keys: An optional set of keys to selectively apply
|
||||
normalization to specific observation features.
|
||||
_tensor_stats: An internal dictionary holding the normalization statistics as
|
||||
PyTorch tensors.
|
||||
_stats_explicitly_provided: Internal flag tracking whether stats were explicitly
|
||||
provided during construction (used for override preservation).
|
||||
"""
|
||||
|
||||
features: dict[str, PolicyFeature]
|
||||
|
||||
@@ -269,18 +269,13 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):
|
||||
data processing workflow. It's generic, allowing for custom input and output types,
|
||||
which are handled by the `to_transition` and `to_output` converters.
|
||||
|
||||
**Attributes**:
|
||||
- **steps** (`Sequence[ProcessorStep]`) -- A sequence of `ProcessorStep` objects that make up the
|
||||
pipeline.
|
||||
- **name** (`str`) -- A descriptive name for the pipeline.
|
||||
- **to_transition** (`Callable[[TInput], EnvTransition]`) -- A function to convert raw input data into
|
||||
the standardized `EnvTransition` format.
|
||||
- **to_output** (`Callable[[EnvTransition], TOutput]`) -- A function to convert the final
|
||||
`EnvTransition` into the desired output format.
|
||||
- **before_step_hooks** (`list[Callable[[int, EnvTransition], None]]`) -- A list of functions to be
|
||||
called before each step is executed.
|
||||
- **after_step_hooks** (`list[Callable[[int, EnvTransition], None]]`) -- A list of functions to be
|
||||
called after each step is executed.
|
||||
Attributes:
|
||||
steps: A sequence of `ProcessorStep` objects that make up the pipeline.
|
||||
name: A descriptive name for the pipeline.
|
||||
to_transition: A function to convert raw input data into the standardized `EnvTransition` format.
|
||||
to_output: A function to convert the final `EnvTransition` into the desired output format.
|
||||
before_step_hooks: A list of functions to be called before each step is executed.
|
||||
after_step_hooks: A list of functions to be called after each step is executed.
|
||||
"""
|
||||
|
||||
steps: Sequence[ProcessorStep] = field(default_factory=list)
|
||||
|
||||
@@ -91,11 +91,11 @@ class RelativeActionsProcessorStep(ProcessorStep):
|
||||
Caches the last seen state so a paired AbsoluteActionsProcessorStep can reverse
|
||||
the conversion during postprocessing.
|
||||
|
||||
**Attributes**:
|
||||
- **enabled** (`bool`) -- Whether to apply the relative conversion.
|
||||
- **exclude_joints** (`list[str]`) -- Joint names to keep absolute (not converted to relative).
|
||||
- **action_names** (`list[str] | None`) -- Action dimension names from dataset metadata, used to build
|
||||
the mask from exclude_joints. If None, all dims are converted.
|
||||
Attributes:
|
||||
enabled: Whether to apply the relative conversion.
|
||||
exclude_joints: Joint names to keep absolute (not converted to relative).
|
||||
action_names: Action dimension names from dataset metadata, used to build
|
||||
the mask from exclude_joints. If None, all dims are converted.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
@@ -168,10 +168,9 @@ class AbsoluteActionsProcessorStep(ProcessorStep):
|
||||
predicted relative offsets are converted back to absolute positions for execution.
|
||||
Reads the cached state from its paired RelativeActionsProcessorStep.
|
||||
|
||||
**Attributes**:
|
||||
- **enabled** (`bool`) -- Whether to apply the absolute conversion.
|
||||
- **relative_step** (`RelativeActionsProcessorStep | None`) -- Reference to the paired
|
||||
RelativeActionsProcessorStep that caches state.
|
||||
Attributes:
|
||||
enabled: Whether to apply the absolute conversion.
|
||||
relative_step: Reference to the paired RelativeActionsProcessorStep that caches state.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
|
||||
@@ -32,9 +32,10 @@ class RenameObservationsProcessorStep(ObservationProcessorStep):
|
||||
from an environment's format to the format expected by a LeRobot policy or
|
||||
other downstream components.
|
||||
|
||||
**Attributes**:
|
||||
- **rename_map** (`dict[str, str]`) -- A dictionary mapping from old key names to new key names. Keys
|
||||
present in an observation that are not in this map will be kept with their original names.
|
||||
Attributes:
|
||||
rename_map: A dictionary mapping from old key names to new key names.
|
||||
Keys present in an observation that are not in this map will
|
||||
be kept with their original names.
|
||||
"""
|
||||
|
||||
rename_map: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@@ -65,17 +65,15 @@ class TokenizerProcessorStep(ObservationProcessorStep):
|
||||
|
||||
Requires the `transformers` library to be installed.
|
||||
|
||||
**Attributes**:
|
||||
- **tokenizer_name** (`str | None`) -- The name of a pretrained tokenizer from the Hugging Face Hub
|
||||
(e.g., "bert-base-uncased").
|
||||
- **tokenizer** (`Any | None`) -- A pre-initialized tokenizer object. If provided, `tokenizer_name` is
|
||||
ignored.
|
||||
- **max_length** (`int`) -- The maximum length to pad or truncate sequences to.
|
||||
- **task_key** (`str`) -- The key in `complementary_data` where the task string is stored.
|
||||
- **padding_side** (`str`) -- The side to pad on ('left' or 'right').
|
||||
- **padding** (`str`) -- The padding strategy ('max_length', 'longest', etc.).
|
||||
- **truncation** (`bool`) -- Whether to truncate sequences longer than `max_length`.
|
||||
- **input_tokenizer** (`Any`) -- The internal tokenizer instance, loaded during initialization.
|
||||
Attributes:
|
||||
tokenizer_name: The name of a pretrained tokenizer from the Hugging Face Hub (e.g., "bert-base-uncased").
|
||||
tokenizer: A pre-initialized tokenizer object. If provided, `tokenizer_name` is ignored.
|
||||
max_length: The maximum length to pad or truncate sequences to.
|
||||
task_key: The key in `complementary_data` where the task string is stored.
|
||||
padding_side: The side to pad on ('left' or 'right').
|
||||
padding: The padding strategy ('max_length', 'longest', etc.).
|
||||
truncation: Whether to truncate sequences longer than `max_length`.
|
||||
input_tokenizer: The internal tokenizer instance, loaded during initialization.
|
||||
"""
|
||||
|
||||
tokenizer_name: str | None = None
|
||||
@@ -348,17 +346,12 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
|
||||
|
||||
Requires the `transformers` library to be installed.
|
||||
|
||||
**Attributes**:
|
||||
- **tokenizer_name** -- The name of a pretrained processor from the Hugging Face Hub (e.g.,
|
||||
"lerobot/fast-action-tokenizer").
|
||||
- **tokenizer** -- A pre-initialized processor/tokenizer object. If provided, `tokenizer_name` is
|
||||
ignored.
|
||||
- **trust_remote_code** (`bool`) -- Whether to trust remote code when loading the tokenizer (required
|
||||
for some tokenizers).
|
||||
- **action_tokenizer** (`Any`) -- The internal tokenizer/processor instance, loaded during
|
||||
initialization.
|
||||
- **paligemma_tokenizer_name** (`str`) -- The name of a pretrained PaliGemma tokenizer from the
|
||||
Hugging Face Hub (e.g., "google/paligemma-3b-pt-224").
|
||||
Attributes:
|
||||
tokenizer_name: The name of a pretrained processor from the Hugging Face Hub (e.g., "lerobot/fast-action-tokenizer").
|
||||
tokenizer: A pre-initialized processor/tokenizer object. If provided, `tokenizer_name` is ignored.
|
||||
trust_remote_code: Whether to trust remote code when loading the tokenizer (required for some tokenizers).
|
||||
action_tokenizer: The internal tokenizer/processor instance, loaded during initialization.
|
||||
paligemma_tokenizer_name: The name of a pretrained PaliGemma tokenizer from the Hugging Face Hub (e.g., "google/paligemma-3b-pt-224").
|
||||
"""
|
||||
|
||||
action_tokenizer_name: str | None = None
|
||||
|
||||
@@ -16,11 +16,12 @@ import abc
|
||||
import builtins
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub import HfApi, ModelCard, ModelCardData, hf_hub_download
|
||||
from huggingface_hub.constants import SAFETENSORS_SINGLE_FILE
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
from safetensors.torch import load_model as load_model_as_safetensor, save_model as save_model_as_safetensor
|
||||
@@ -60,22 +61,6 @@ class PreTrainedRewardModel(nn.Module, HubMixin, abc.ABC):
|
||||
raise TypeError(f"Class {cls.__name__} must define 'name'")
|
||||
|
||||
def _save_pretrained(self, save_directory: Path) -> None:
|
||||
"""Serialize this reward model's parameters (and config) into `save_directory`.
|
||||
|
||||
Safe to call on every rank: replicas carry identical weights, so only the main process
|
||||
writes (sharded reward models are rejected at config validation — no collective gather).
|
||||
|
||||
Args:
|
||||
save_directory (Path): Target directory for the reward model config (`config.json`)
|
||||
and `model.safetensors`.
|
||||
"""
|
||||
from lerobot.distributed.utils import is_main_process
|
||||
|
||||
# save_checkpoint calls this on every rank; replicas carry identical
|
||||
# weights, so the main process is the only writer. Sharded reward models are rejected
|
||||
# at config validation, so no collective gather is needed here.
|
||||
if not is_main_process():
|
||||
return
|
||||
self.config._save_pretrained(save_directory)
|
||||
model_to_save = self.module if hasattr(self, "module") else self
|
||||
save_model_as_safetensor(model_to_save, str(save_directory / SAFETENSORS_SINGLE_FILE))
|
||||
@@ -190,22 +175,53 @@ class PreTrainedRewardModel(nn.Module, HubMixin, abc.ABC):
|
||||
"""
|
||||
return type(self).forward is not PreTrainedRewardModel.forward
|
||||
|
||||
def push_model_to_hub(self, cfg: "TrainPipelineConfig") -> None:
|
||||
"""Publish this reward model to the Hub.
|
||||
def push_model_to_hub(self, cfg: "TrainPipelineConfig"):
|
||||
api = HfApi()
|
||||
repo_id = api.create_repo(
|
||||
repo_id=self.config.repo_id, private=self.config.private, exist_ok=True
|
||||
).repo_id
|
||||
|
||||
Deprecated: use :func:`lerobot.common.train_utils.publish_trained_model` instead.
|
||||
# Push the files to the repo in a single commit
|
||||
with TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
|
||||
saved_path = Path(tmp) / repo_id
|
||||
|
||||
Args:
|
||||
cfg (TrainPipelineConfig): The training config; saved as `train_config.json` and
|
||||
used to render the model card.
|
||||
"""
|
||||
from lerobot.common.train_utils import publish_trained_model
|
||||
self.save_pretrained(saved_path) # Calls _save_pretrained and stores model tensors
|
||||
|
||||
warnings.warn(
|
||||
"PreTrainedRewardModel.push_model_to_hub is deprecated and will be removed in a "
|
||||
"future version. Use lerobot.common.train_utils.publish_trained_model(cfg, model, "
|
||||
"preprocessor, postprocessor, dataset_meta) instead.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
card = self.generate_model_card(
|
||||
cfg.dataset.repo_id, self.config.type, self.config.license, self.config.tags
|
||||
)
|
||||
card.save(str(saved_path / "README.md"))
|
||||
|
||||
cfg.save_pretrained(saved_path) # Calls _save_pretrained and stores train config
|
||||
|
||||
commit_info = api.upload_folder(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
folder_path=saved_path,
|
||||
commit_message="Upload reward model weights, train config and readme",
|
||||
allow_patterns=["*.safetensors", "*.json", "*.yaml", "*.md"],
|
||||
ignore_patterns=["*.tmp", "*.log"],
|
||||
)
|
||||
|
||||
logging.info(f"Model pushed to {commit_info.repo_url.url}")
|
||||
|
||||
def generate_model_card(
|
||||
self, dataset_repo_id: str, model_type: str, license: str | None, tags: list[str] | None
|
||||
) -> ModelCard:
|
||||
card_data = ModelCardData(
|
||||
license=license or "apache-2.0",
|
||||
library_name="lerobot",
|
||||
pipeline_tag="robotics",
|
||||
tags=list(set(tags or []).union({"robotics", "lerobot", "reward-model", model_type})),
|
||||
model_name=model_type,
|
||||
datasets=dataset_repo_id,
|
||||
)
|
||||
publish_trained_model(cfg, self, None, None, None)
|
||||
|
||||
template_card = (
|
||||
files("lerobot.templates")
|
||||
.joinpath("lerobot_rewardmodel_modelcard_template.md")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
card = ModelCard.from_template(card_data, template_str=template_card)
|
||||
card.validate()
|
||||
return card
|
||||
|
||||
@@ -58,11 +58,12 @@ import builtins
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub import HfApi, hf_hub_download
|
||||
from huggingface_hub.constants import CONFIG_NAME
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
from torch import Tensor
|
||||
@@ -74,6 +75,9 @@ from lerobot.rewards.topreward.configuration_topreward import TOPRewardConfig
|
||||
from lerobot.rewards.topreward.processor_topreward import TOPREWARD_FEATURE_PREFIX, TOPREWARD_INPUT_KEYS
|
||||
from lerobot.utils.import_utils import _transformers_available, require_package
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lerobot.configs.train import TrainPipelineConfig
|
||||
|
||||
if TYPE_CHECKING or _transformers_available:
|
||||
from transformers import Qwen3VLForConditionalGeneration
|
||||
else:
|
||||
@@ -201,3 +205,34 @@ class TOPRewardModel(PreTrainedRewardModel):
|
||||
instance.to(config.device)
|
||||
instance.eval()
|
||||
return instance
|
||||
|
||||
def push_model_to_hub(self, cfg: TrainPipelineConfig):
|
||||
"""Push the TOPReward ``config.json`` + model card to the Hub."""
|
||||
api = HfApi()
|
||||
repo_id = api.create_repo(
|
||||
repo_id=self.config.repo_id, private=self.config.private, exist_ok=True
|
||||
).repo_id
|
||||
|
||||
with TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
|
||||
saved_path = Path(tmp) / repo_id
|
||||
saved_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.config._save_pretrained(saved_path)
|
||||
|
||||
card = self.generate_model_card(
|
||||
cfg.dataset.repo_id, self.config.type, self.config.license, self.config.tags
|
||||
)
|
||||
card.save(str(saved_path / "README.md"))
|
||||
|
||||
cfg.save_pretrained(saved_path)
|
||||
|
||||
commit_info = api.upload_folder(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
folder_path=saved_path,
|
||||
commit_message="Upload TOPReward config and readme",
|
||||
allow_patterns=["*.json", "*.yaml", "*.md"],
|
||||
ignore_patterns=["*.tmp", "*.log", "*.safetensors"],
|
||||
)
|
||||
|
||||
logger.info(f"Model pushed to {commit_info.repo_url.url}")
|
||||
|
||||
@@ -38,11 +38,11 @@ class JointVelocityProcessorStep(ObservationProcessorStep):
|
||||
difference between the current and the last observed joint positions. The
|
||||
resulting velocity vector is then concatenated to the original state vector.
|
||||
|
||||
**Attributes**:
|
||||
- **dt** (`float`) -- The time step (delta time) in seconds between observations, used for calculating
|
||||
velocity.
|
||||
- **last_joint_positions** (`torch.Tensor | None`) -- Stores the joint positions from the previous
|
||||
step to enable velocity calculation.
|
||||
Attributes:
|
||||
dt: The time step (delta time) in seconds between observations, used for
|
||||
calculating velocity.
|
||||
last_joint_positions: Stores the joint positions from the previous step
|
||||
to enable velocity calculation.
|
||||
"""
|
||||
|
||||
dt: float = 0.1
|
||||
@@ -138,9 +138,9 @@ class MotorCurrentProcessorStep(ObservationProcessorStep):
|
||||
This step queries the robot's hardware interface to get the present current
|
||||
for each motor and concatenates this information to the existing state vector.
|
||||
|
||||
**Attributes**:
|
||||
- **robot** (`Robot | None`) -- An instance of a `lerobot` Robot class that provides access to the
|
||||
hardware bus.
|
||||
Attributes:
|
||||
robot: An instance of a `lerobot` Robot class that provides access to
|
||||
the hardware bus.
|
||||
"""
|
||||
|
||||
robot: Robot | None = None
|
||||
|
||||
+10
-17
@@ -74,14 +74,13 @@ from torch.optim.optimizer import Optimizer
|
||||
from lerobot.cameras import opencv # noqa: F401
|
||||
from lerobot.common.train_utils import (
|
||||
get_step_checkpoint_dir,
|
||||
load_training_metadata,
|
||||
load_training_state as utils_load_training_state,
|
||||
save_checkpoint,
|
||||
update_last_checkpoint,
|
||||
)
|
||||
from lerobot.common.wandb_utils import WandBLogger
|
||||
from lerobot.configs import parser
|
||||
from lerobot.datasets import LeRobotDataset, make_dataset
|
||||
from lerobot.optim import load_optimizer_state
|
||||
from lerobot.policies import make_policy, make_pre_post_processors
|
||||
from lerobot.robots import so_follower # noqa: F401
|
||||
from lerobot.teleoperators import gamepad, so_leader # noqa: F401
|
||||
@@ -104,7 +103,7 @@ from lerobot.utils.constants import (
|
||||
from lerobot.utils.device_utils import get_safe_torch_device
|
||||
from lerobot.utils.io_utils import load_json, write_json
|
||||
from lerobot.utils.process import ProcessSignalHandler, ensure_multiprocessing_start_method
|
||||
from lerobot.utils.random_utils import load_rng_state, set_seed
|
||||
from lerobot.utils.random_utils import set_seed
|
||||
from lerobot.utils.utils import (
|
||||
format_big_number,
|
||||
init_logging,
|
||||
@@ -717,18 +716,15 @@ def load_training_state(
|
||||
algorithm-owned tensors) from the most recent checkpoint.
|
||||
|
||||
Args:
|
||||
cfg (TrainRLServerPipelineConfig): Training configuration; `cfg.resume` gates the load and
|
||||
`cfg.output_dir` locates the last checkpoint.
|
||||
optimizers (Optimizer | dict[str, Optimizer]): Optimizers to load state into.
|
||||
algorithm (RLAlgorithm | None, optional): Algorithm whose state dict should be restored.
|
||||
Required for full main-equivalent resume; the policy itself is restored separately via
|
||||
`make_policy`. Defaults to None.
|
||||
device (str | torch.device, optional): Device on which to place loaded algorithm tensors.
|
||||
Defaults to "cpu".
|
||||
cfg: Training configuration.
|
||||
optimizers: Optimizers to load state into.
|
||||
algorithm: Algorithm whose state dict should be restored.
|
||||
Required for full main-equivalent resume;
|
||||
the policy itself is restored separately via ``make_policy``.
|
||||
device: Device on which to place loaded algorithm tensors.
|
||||
|
||||
Returns:
|
||||
tuple[int | None, int | None]: `(optimization_step, interaction_step)`, or `(None, None)`
|
||||
when not resuming or when loading the training state fails.
|
||||
tuple: (optimization_step, interaction_step) or (None, None) if not resuming
|
||||
"""
|
||||
if not cfg.resume:
|
||||
return None, None
|
||||
@@ -740,10 +736,7 @@ def load_training_state(
|
||||
|
||||
try:
|
||||
# Restore optimizers + RNG + step from the standard `training_state/` folder
|
||||
training_state_dir = checkpoint_dir / TRAINING_STATE_DIR
|
||||
load_rng_state(training_state_dir)
|
||||
step = load_training_metadata(training_state_dir)["step"]
|
||||
optimizers = load_optimizer_state(optimizers, training_state_dir)
|
||||
step, optimizers, _ = utils_load_training_state(checkpoint_dir, optimizers, None)
|
||||
|
||||
# Restore algorithm-owned tensors
|
||||
if algorithm is not None:
|
||||
|
||||
@@ -29,18 +29,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BiOpenArmFollower(BimanualMixin, Robot):
|
||||
"""A bimanual pair of OpenArm follower arms driven as one robot."""
|
||||
"""
|
||||
Bimanual OpenArm Follower Arms
|
||||
"""
|
||||
|
||||
config_class = BiOpenArmFollowerConfig
|
||||
name = "bi_openarm_follower"
|
||||
|
||||
def __init__(self, config: BiOpenArmFollowerConfig):
|
||||
"""Build the robot from its configuration.
|
||||
|
||||
Args:
|
||||
config (`BiOpenArmFollowerConfig`):
|
||||
The robot's configuration. Its `port` and `cameras` determine what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -118,43 +114,19 @@ class BiOpenArmFollower(BimanualMixin, Robot):
|
||||
|
||||
@cached_property
|
||||
def observation_features(self) -> dict[str, type | tuple]:
|
||||
"""The values this robot reports, and their types or shapes.
|
||||
|
||||
Returns:
|
||||
`dict`: Keys as returned by [`~robots.Robot.get_observation`], mapped to a scalar type for
|
||||
proprioceptive values or to a `(height, width, channels)` shape for images.
|
||||
"""
|
||||
return {**self._motors_ft, **self._cameras_ft}
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The values this robot accepts, and their types.
|
||||
|
||||
Returns:
|
||||
`dict`: Keys accepted by [`~robots.Robot.send_action`], mapped to their type.
|
||||
"""
|
||||
return self._motors_ft
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one at a time.
|
||||
|
||||
Run this once when building the robot. Interactive: prompts you to connect the controller board to a
|
||||
single motor at a time.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Motor ID configuration is typically done via manufacturer tools for CAN motors."
|
||||
)
|
||||
|
||||
@check_if_not_connected
|
||||
def get_observation(self) -> RobotObservation:
|
||||
"""Read the robot's current state and a frame from each camera.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: Keys matching [`~robots.Robot.observation_features`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot is not connected.
|
||||
"""
|
||||
obs_dict: RobotObservation = {}
|
||||
|
||||
# Add "left_" prefix to per-arm keys; keep top-level camera keys unprefixed.
|
||||
@@ -174,23 +146,6 @@ class BiOpenArmFollower(BimanualMixin, Robot):
|
||||
custom_kp: dict[str, float] | None = None,
|
||||
custom_kd: dict[str, float] | None = None,
|
||||
) -> RobotAction:
|
||||
"""Command both arms to move towards a target configuration.
|
||||
|
||||
Args:
|
||||
action (`dict[str, Any]`):
|
||||
Target values, keyed as in [`~robots.Robot.action_features`], i.e. prefixed `left_` and
|
||||
`right_`.
|
||||
custom_kp (`dict[str, float]`, *optional*):
|
||||
Per-motor proportional gains for this step only. Defaults to each arm's `position_kp`.
|
||||
custom_kd (`dict[str, float]`, *optional*):
|
||||
Per-motor derivative gains for this step only. Defaults to each arm's `position_kd`.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: The action actually sent, which may be clipped by `max_relative_target`.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot is not connected.
|
||||
"""
|
||||
# Remove "left_" prefix
|
||||
left_action = {
|
||||
key.removeprefix("left_"): value for key, value in action.items() if key.startswith("left_")
|
||||
|
||||
@@ -25,28 +25,7 @@ from ..openarm_follower import OpenArmFollowerConfigBase
|
||||
@RobotConfig.register_subclass("bi_openarm_follower")
|
||||
@dataclass(kw_only=True)
|
||||
class BiOpenArmFollowerConfig(RobotConfig):
|
||||
"""Configuration for a bimanual pair of OpenArm follower arms.
|
||||
|
||||
The two arms are configured independently, then driven as one robot: observation and action keys from
|
||||
each arm are prefixed with `left_` and `right_`.
|
||||
|
||||
Calibration is per arm, taken from each arm config's own settings.
|
||||
|
||||
Args:
|
||||
id (`str`, *optional*, defaults to `"bi_openarm_follower"`):
|
||||
Identifier for the pair as a whole.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Unused at this level; each arm calibrates through its own config.
|
||||
left_arm_config (`OpenArmFollowerConfigBase`):
|
||||
Configuration for the left arm, including its own CAN interface. Set its `side` to `"left"` so
|
||||
the correct joint limits apply.
|
||||
right_arm_config (`OpenArmFollowerConfigBase`):
|
||||
Configuration for the right arm, including its own CAN interface. Set its `side` to `"right"`.
|
||||
cameras (`dict[str, CameraConfig]`, *optional*):
|
||||
Cameras not attached to either arm, such as an overhead view. These keys appear in
|
||||
observations unchanged, whereas cameras declared on an arm config are prefixed with that
|
||||
arm's side.
|
||||
"""
|
||||
"""Configuration class for Bi OpenArm Follower robots."""
|
||||
|
||||
id: str | None = "bi_openarm_follower"
|
||||
|
||||
|
||||
@@ -39,12 +39,6 @@ class BiRebotB601Follower(BimanualMixin, Robot):
|
||||
name = "bi_rebot_b601_follower"
|
||||
|
||||
def __init__(self, config: BiRebotB601FollowerConfig):
|
||||
"""Build the robot from its configuration.
|
||||
|
||||
Args:
|
||||
config (`BiRebotB601FollowerConfig`):
|
||||
The robot's configuration. Its `port` and `cameras` determine what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -126,33 +120,14 @@ class BiRebotB601Follower(BimanualMixin, Robot):
|
||||
|
||||
@cached_property
|
||||
def observation_features(self) -> dict[str, type | tuple]:
|
||||
"""The values this robot reports, and their types or shapes.
|
||||
|
||||
Returns:
|
||||
`dict`: Keys as returned by [`~robots.Robot.get_observation`], mapped to a scalar type for
|
||||
proprioceptive values or to a `(height, width, channels)` shape for images.
|
||||
"""
|
||||
return {**self._motors_ft, **self._cameras_ft}
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The values this robot accepts, and their types.
|
||||
|
||||
Returns:
|
||||
`dict`: Keys accepted by [`~robots.Robot.send_action`], mapped to their type.
|
||||
"""
|
||||
return self._motors_ft
|
||||
|
||||
@check_if_not_connected
|
||||
def get_observation(self) -> RobotObservation:
|
||||
"""Read the robot's current state and a frame from each camera.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: Keys matching [`~robots.Robot.observation_features`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot is not connected.
|
||||
"""
|
||||
obs_dict: RobotObservation = {}
|
||||
for k, v in self.left_arm.get_observation().items():
|
||||
obs_dict[k if k in self._top_level_cam_keys else f"left_{k}"] = v
|
||||
@@ -162,18 +137,6 @@ class BiRebotB601Follower(BimanualMixin, Robot):
|
||||
|
||||
@check_if_not_connected
|
||||
def send_action(self, action: RobotAction) -> RobotAction:
|
||||
"""Command the robot to move towards a target configuration.
|
||||
|
||||
Args:
|
||||
action (`dict[str, Any]`):
|
||||
Target values, keyed as in [`~robots.Robot.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: The action actually sent, which may be clipped by `max_relative_target`.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot is not connected.
|
||||
"""
|
||||
left_action = {
|
||||
key.removeprefix("left_"): value for key, value in action.items() if key.startswith("left_")
|
||||
}
|
||||
|
||||
@@ -25,27 +25,7 @@ from ..rebot_b601_follower import RebotB601FollowerConfig
|
||||
@RobotConfig.register_subclass("bi_rebot_b601_follower")
|
||||
@dataclass
|
||||
class BiRebotB601FollowerConfig(RobotConfig):
|
||||
"""Configuration for a bimanual pair of reBot B601-DM follower arms.
|
||||
|
||||
The two arms are configured independently, then driven as one robot: observation and action keys from
|
||||
each arm are prefixed with `left_` and `right_`.
|
||||
|
||||
Calibration is per arm, taken from each arm config's own `id` and `calibration_dir`.
|
||||
|
||||
Args:
|
||||
left_arm_config (`RebotB601FollowerConfig`):
|
||||
Configuration for the left arm, including its own `port` and CAN settings.
|
||||
right_arm_config (`RebotB601FollowerConfig`):
|
||||
Configuration for the right arm, including its own `port` and CAN settings.
|
||||
cameras (`dict[str, CameraConfig]`, *optional*):
|
||||
Cameras not attached to either arm, such as an overhead view. These keys appear in
|
||||
observations unchanged, whereas cameras declared on an arm config are prefixed with that
|
||||
arm's side.
|
||||
id (`str`, *optional*):
|
||||
Identifier for the pair as a whole.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Unused at this level; each arm calibrates through its own config.
|
||||
"""
|
||||
"""Configuration class for the bimanual reBot B601-DM follower robot."""
|
||||
|
||||
left_arm_config: RebotB601FollowerConfig
|
||||
right_arm_config: RebotB601FollowerConfig
|
||||
|
||||
@@ -29,18 +29,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BiSOFollower(BimanualMixin, Robot):
|
||||
"""A bimanual pair of [SO follower arms](https://github.com/TheRobotStudio/SO-ARM100) by TheRobotStudio."""
|
||||
"""
|
||||
[Bimanual SO Follower Arms](https://github.com/TheRobotStudio/SO-ARM100) designed by TheRobotStudio
|
||||
"""
|
||||
|
||||
config_class = BiSOFollowerConfig
|
||||
name = "bi_so_follower"
|
||||
|
||||
def __init__(self, config: BiSOFollowerConfig):
|
||||
"""Build the robot from its configuration.
|
||||
|
||||
Args:
|
||||
config (`BiSOFollowerConfig`):
|
||||
The robot's configuration. Its `port` and `cameras` determine what is connected.
|
||||
"""
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
|
||||
@@ -111,42 +107,18 @@ class BiSOFollower(BimanualMixin, Robot):
|
||||
|
||||
@cached_property
|
||||
def observation_features(self) -> dict[str, type | tuple]:
|
||||
"""The values this robot reports, and their types or shapes.
|
||||
|
||||
Returns:
|
||||
`dict`: Keys as returned by [`~robots.Robot.get_observation`], mapped to a scalar type for
|
||||
proprioceptive values or to a `(height, width, channels)` shape for images.
|
||||
"""
|
||||
return {**self._motors_ft, **self._cameras_ft}
|
||||
|
||||
@cached_property
|
||||
def action_features(self) -> dict[str, type]:
|
||||
"""The values this robot accepts, and their types.
|
||||
|
||||
Returns:
|
||||
`dict`: Keys accepted by [`~robots.Robot.send_action`], mapped to their type.
|
||||
"""
|
||||
return self._motors_ft
|
||||
|
||||
def setup_motors(self) -> None:
|
||||
"""Assign each motor its bus ID, one at a time.
|
||||
|
||||
Run this once when building the robot. Interactive: prompts you to connect the controller board to a
|
||||
single motor at a time.
|
||||
"""
|
||||
self.left_arm.setup_motors()
|
||||
self.right_arm.setup_motors()
|
||||
|
||||
@check_if_not_connected
|
||||
def get_observation(self) -> RobotObservation:
|
||||
"""Read the robot's current state and a frame from each camera.
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: Keys matching [`~robots.Robot.observation_features`].
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot is not connected.
|
||||
"""
|
||||
obs_dict: RobotObservation = {}
|
||||
|
||||
# Add "left_" prefix to per-arm keys; keep top-level camera keys unprefixed.
|
||||
@@ -162,18 +134,6 @@ class BiSOFollower(BimanualMixin, Robot):
|
||||
@check_if_not_connected
|
||||
def send_action(self, action: RobotAction) -> RobotAction:
|
||||
# Remove "left_" prefix
|
||||
"""Command the robot to move towards a target configuration.
|
||||
|
||||
Args:
|
||||
action (`dict[str, Any]`):
|
||||
Target values, keyed as in [`~robots.Robot.action_features`].
|
||||
|
||||
Returns:
|
||||
`dict[str, Any]`: The action actually sent, which may be clipped by `max_relative_target`.
|
||||
|
||||
Raises:
|
||||
DeviceNotConnectedError: If the robot is not connected.
|
||||
"""
|
||||
left_action = {
|
||||
key.removeprefix("left_"): value for key, value in action.items() if key.startswith("left_")
|
||||
}
|
||||
|
||||
@@ -25,27 +25,7 @@ from ..so_follower import SOFollowerConfig
|
||||
@RobotConfig.register_subclass("bi_so_follower")
|
||||
@dataclass
|
||||
class BiSOFollowerConfig(RobotConfig):
|
||||
"""Configuration for a bimanual pair of SO follower arms.
|
||||
|
||||
The two arms are configured independently, then driven as one robot: observation and action keys from
|
||||
each arm are prefixed with `left_` and `right_`.
|
||||
|
||||
Calibration is per arm, taken from each arm config's own `id` and `calibration_dir`.
|
||||
|
||||
Args:
|
||||
left_arm_config (`SOFollowerConfig`):
|
||||
Configuration for the left arm, including its own `port`.
|
||||
right_arm_config (`SOFollowerConfig`):
|
||||
Configuration for the right arm, including its own `port`.
|
||||
cameras (`dict[str, CameraConfig]`, *optional*):
|
||||
Cameras not attached to either arm, such as an overhead view. These keys appear in
|
||||
observations unchanged, whereas cameras declared on an arm config are prefixed with that
|
||||
arm's side.
|
||||
id (`str`, *optional*):
|
||||
Identifier for the pair as a whole.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Unused at this level; each arm calibrates through its own config.
|
||||
"""
|
||||
"""Configuration class for Bi SO Follower robots."""
|
||||
|
||||
left_arm_config: SOFollowerConfig
|
||||
right_arm_config: SOFollowerConfig
|
||||
|
||||
@@ -21,33 +21,12 @@ import draccus
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class RobotConfig(draccus.ChoiceRegistry, abc.ABC):
|
||||
"""Base configuration shared by every robot.
|
||||
|
||||
Concrete robots subclass this and register themselves with
|
||||
`@RobotConfig.register_subclass("name")`, which is what makes `--robot.type=name` work on the command
|
||||
line. Subclasses inherit the two fields below and must document them alongside their own.
|
||||
|
||||
Args:
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular unit, used to tell apart several robots of the same type. It
|
||||
also names the calibration file, so keep it stable for a given piece of hardware.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to a per-robot directory under the
|
||||
LeRobot calibration home.
|
||||
"""
|
||||
|
||||
# Allows to distinguish between different robots of the same type
|
||||
id: str | None = None
|
||||
# Directory to store calibration file
|
||||
calibration_dir: Path | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that every configured camera specifies the fields a robot requires.
|
||||
|
||||
Raises:
|
||||
ValueError: If a camera does not set `width`, `height` and `fps`. A robot records frames at a
|
||||
fixed shape, so these cannot be left to the driver's defaults.
|
||||
"""
|
||||
if hasattr(self, "cameras") and self.cameras:
|
||||
for _, config in self.cameras.items():
|
||||
for attr in ["width", "height", "fps"]:
|
||||
@@ -58,10 +37,4 @@ class RobotConfig(draccus.ChoiceRegistry, abc.ABC):
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
"""The registered name of this robot type.
|
||||
|
||||
Returns:
|
||||
`str`: The name passed to `@RobotConfig.register_subclass`, e.g. `"so101_follower"`. This is
|
||||
what `make_robot_from_config` dispatches on and what a user writes as `--robot.type=...`.
|
||||
"""
|
||||
return self.get_choice_name(self.__class__)
|
||||
|
||||
@@ -23,18 +23,13 @@ from ..config import RobotConfig
|
||||
@RobotConfig.register_subclass("earthrover_mini_plus")
|
||||
@dataclass
|
||||
class EarthRoverMiniPlusConfig(RobotConfig):
|
||||
"""Configuration for the EarthRover Mini Plus rover.
|
||||
"""Configuration for EarthRover Mini Plus robot using Frodobots SDK.
|
||||
|
||||
This robot is driven over the cloud through the Frodobots SDK's HTTP API rather than a local bus, so
|
||||
there is no serial port and no LeRobot calibration file. Camera frames come from SDK HTTP endpoints.
|
||||
This robot uses cloud-based control via the Frodobots SDK HTTP API.
|
||||
Camera frames are accessed directly through SDK HTTP endpoints.
|
||||
|
||||
Args:
|
||||
sdk_url (`str`, *optional*, defaults to `"http://localhost:8000"`):
|
||||
Base URL of the Frodobots SDK server. Commands and camera frames both go through it.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular rover.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Unused: the rover exposes no calibration.
|
||||
Attributes:
|
||||
sdk_url: URL of the Frodobots SDK server (default: http://localhost:8000)
|
||||
"""
|
||||
|
||||
sdk_url: str = "http://localhost:8000"
|
||||
|
||||
@@ -70,7 +70,8 @@ OBS_WHEEL_RPM_3 = "wheel_rpm_3"
|
||||
|
||||
|
||||
class EarthRoverMiniPlus(Robot):
|
||||
"""EarthRover Mini Plus robot controlled via Frodobots SDK HTTP API.
|
||||
"""
|
||||
EarthRover Mini Plus robot controlled via Frodobots SDK HTTP API.
|
||||
|
||||
This robot uses cloud-based control through the Frodobots SDK instead of direct
|
||||
hardware connection. Cameras stream via WebRTC through Agora cloud, and control
|
||||
@@ -81,9 +82,9 @@ class EarthRoverMiniPlus(Robot):
|
||||
- Linear and angular velocity control
|
||||
- Battery and orientation telemetry
|
||||
|
||||
**Attributes**:
|
||||
- **config** -- Robot configuration
|
||||
- **sdk_base_url** -- URL of the Frodobots SDK server (default: http://localhost:8000)
|
||||
Attributes:
|
||||
config: Robot configuration
|
||||
sdk_base_url: URL of the Frodobots SDK server (default: http://localhost:8000)
|
||||
"""
|
||||
|
||||
config_class = EarthRoverMiniPlusConfig
|
||||
@@ -129,6 +130,7 @@ class EarthRoverMiniPlus(Robot):
|
||||
DeviceAlreadyConnectedError: If robot is already connected
|
||||
DeviceNotConnectedError: If cannot connect to SDK server
|
||||
"""
|
||||
|
||||
# Verify SDK is running and accessible
|
||||
try:
|
||||
response = requests.get(f"{self.sdk_base_url}/data", timeout=10.0)
|
||||
@@ -278,6 +280,7 @@ class EarthRoverMiniPlus(Robot):
|
||||
Robot telemetry is retrieved from /data endpoint.
|
||||
All SDK values are normalized to appropriate ranges for dataset recording.
|
||||
"""
|
||||
|
||||
observation = {}
|
||||
|
||||
# Get camera images from SDK
|
||||
@@ -367,6 +370,7 @@ class EarthRoverMiniPlus(Robot):
|
||||
Raises:
|
||||
DeviceNotConnectedError: If robot is not connected
|
||||
"""
|
||||
|
||||
# Stop the robot before disconnecting
|
||||
try:
|
||||
self._send_command_to_sdk(0.0, 0.0)
|
||||
|
||||
@@ -24,27 +24,6 @@ from ..config import RobotConfig
|
||||
@RobotConfig.register_subclass("hope_jr_hand")
|
||||
@dataclass
|
||||
class HopeJrHandConfig(RobotConfig):
|
||||
"""Configuration for one Hope Jr hand.
|
||||
|
||||
Each hand is a separate robot, so a two-handed setup uses two of these with different `side` and
|
||||
`port` values.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the hand is connected to. Run `lerobot-find-port` to identify it.
|
||||
side (`str`):
|
||||
Which hand this is, `"left"` or `"right"`. Determines the motor layout, so it must match the
|
||||
hardware.
|
||||
disable_torque_on_disconnect (`bool`, *optional*, defaults to `True`):
|
||||
Whether to release the motors on disconnect.
|
||||
cameras (`dict[str, CameraConfig]`, *optional*):
|
||||
Cameras to read alongside the hand's joint positions.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular hand; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
port: str # Port to connect to the hand
|
||||
side: str # "left" / "right"
|
||||
|
||||
@@ -53,12 +32,6 @@ class HopeJrHandConfig(RobotConfig):
|
||||
cameras: dict[str, CameraConfig] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate the camera settings and the hand side.
|
||||
|
||||
Raises:
|
||||
ValueError: If `side` is not `"left"` or `"right"`, or if a camera omits `width`, `height` or
|
||||
`fps`.
|
||||
"""
|
||||
super().__post_init__()
|
||||
if self.side not in ["right", "left"]:
|
||||
raise ValueError(self.side)
|
||||
@@ -67,26 +40,6 @@ class HopeJrHandConfig(RobotConfig):
|
||||
@RobotConfig.register_subclass("hope_jr_arm")
|
||||
@dataclass
|
||||
class HopeJrArmConfig(RobotConfig):
|
||||
"""Configuration for one Hope Jr arm.
|
||||
|
||||
Args:
|
||||
port (`str`):
|
||||
Serial port the arm is connected to. Run `lerobot-find-port` to identify it.
|
||||
disable_torque_on_disconnect (`bool`, *optional*, defaults to `True`):
|
||||
Whether to release the motors on disconnect. Leave `True` unless the arm is holding a load it
|
||||
must not drop.
|
||||
max_relative_target (`float | dict[str, float]`, *optional*):
|
||||
Caps how far a single action may move the arm from its present position, as a safety limit. A
|
||||
scalar applies to every motor; a dict maps motor name to a per-motor cap. `None` disables
|
||||
clipping.
|
||||
cameras (`dict[str, CameraConfig]`, *optional*):
|
||||
Cameras to read alongside the arm's joint positions.
|
||||
id (`str`, *optional*):
|
||||
Identifier for this particular arm; also names its calibration file.
|
||||
calibration_dir (`Path`, *optional*):
|
||||
Where to read and write the calibration file. Defaults to the LeRobot calibration home.
|
||||
"""
|
||||
|
||||
port: str # Port to connect to the hand
|
||||
disable_torque_on_disconnect: bool = True
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user