mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-30 16:09:44 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nikodem Bartnik
|
a24d10f5bb | ||
|
Nikodem Bartnik
|
32279544ea |
@@ -0,0 +1,172 @@
|
||||
- sections:
|
||||
- local: index
|
||||
title: LeRobot
|
||||
- local: installation
|
||||
title: Installation
|
||||
- local: cheat-sheet
|
||||
title: Cheat sheet
|
||||
title: Get started
|
||||
- sections:
|
||||
- local: il_robots
|
||||
title: Imitation Learning for Robots
|
||||
- local: bring_your_own_policies
|
||||
title: Adding a Policy
|
||||
- local: integrate_hardware
|
||||
title: Bring Your Own Hardware
|
||||
- local: hilserl
|
||||
title: Train a Robot with RL
|
||||
- local: hilserl_sim
|
||||
title: Train RL in Simulation
|
||||
- local: multi_gpu_training
|
||||
title: Multi GPU training
|
||||
- local: hil_data_collection
|
||||
title: Human In the Loop Data Collection
|
||||
- local: peft_training
|
||||
title: Training with PEFT (e.g., LoRA)
|
||||
- local: rename_map
|
||||
title: Using Rename Map and Empty Cameras
|
||||
title: "Tutorials"
|
||||
- sections:
|
||||
- local: hardware_guide
|
||||
title: Compute Hardware Guide
|
||||
- local: torch_accelerators
|
||||
title: PyTorch accelerators
|
||||
title: "Compute & Hardware"
|
||||
- sections:
|
||||
- local: lerobot-dataset-v3
|
||||
title: Using LeRobotDataset
|
||||
- local: porting_datasets_v3
|
||||
title: Porting Large Datasets
|
||||
- local: using_dataset_tools
|
||||
title: Using the Dataset Tools
|
||||
- local: language_and_recipes
|
||||
title: Language Columns and Recipes
|
||||
- local: tools
|
||||
title: Tools
|
||||
- local: video_encoding_parameters
|
||||
title: Video encoding parameters
|
||||
- local: streaming_video_encoding
|
||||
title: Streaming Video Encoding
|
||||
title: "Datasets"
|
||||
- sections:
|
||||
- local: act
|
||||
title: ACT
|
||||
- local: smolvla
|
||||
title: SmolVLA
|
||||
- local: pi0
|
||||
title: π₀ (Pi0)
|
||||
- local: pi0fast
|
||||
title: π₀-FAST (Pi0Fast)
|
||||
- local: pi05
|
||||
title: π₀.₅ (Pi05)
|
||||
- local: eo1
|
||||
title: EO-1
|
||||
- local: groot
|
||||
title: NVIDIA GR00T N1.5
|
||||
- local: xvla
|
||||
title: X-VLA
|
||||
- local: multi_task_dit
|
||||
title: Multitask DiT Policy
|
||||
- local: walloss
|
||||
title: WALL-OSS
|
||||
title: "Policies"
|
||||
- sections:
|
||||
- local: sarm
|
||||
title: SARM
|
||||
title: "Reward Models"
|
||||
- sections:
|
||||
- local: inference
|
||||
title: Policy Deployment (lerobot-rollout)
|
||||
- local: async
|
||||
title: Use Async Inference
|
||||
- local: rtc
|
||||
title: Real-Time Chunking (RTC)
|
||||
title: "Inference"
|
||||
- sections:
|
||||
- local: envhub
|
||||
title: Environments from the Hub
|
||||
- local: envhub_leisaac
|
||||
title: Control & Train Robots in Sim (LeIsaac)
|
||||
title: "Simulation"
|
||||
- sections:
|
||||
- local: adding_benchmarks
|
||||
title: Adding a New Benchmark
|
||||
- local: libero
|
||||
title: LIBERO
|
||||
- local: libero_plus
|
||||
title: LIBERO-plus
|
||||
- local: metaworld
|
||||
title: Meta-World
|
||||
- local: robotwin
|
||||
title: RoboTwin 2.0
|
||||
- local: robocasa
|
||||
title: RoboCasa365
|
||||
- local: robocerebra
|
||||
title: RoboCerebra
|
||||
- local: robomme
|
||||
title: RoboMME
|
||||
- local: envhub_isaaclab_arena
|
||||
title: NVIDIA IsaacLab Arena Environments
|
||||
- local: vlabench
|
||||
title: VLABench
|
||||
title: "Benchmarks"
|
||||
- sections:
|
||||
- local: introduction_processors
|
||||
title: Introduction to Robot Processors
|
||||
- local: debug_processor_pipeline
|
||||
title: Debug your processor pipeline
|
||||
- local: implement_your_own_processor
|
||||
title: Implement your own processor
|
||||
- local: processors_robots_teleop
|
||||
title: Processors for Robots and Teleoperators
|
||||
- local: env_processor
|
||||
title: Environment Processors
|
||||
- local: action_representations
|
||||
title: Action Representations
|
||||
title: "Robot Processors"
|
||||
- sections:
|
||||
- local: so101
|
||||
title: SO-101
|
||||
- local: so100
|
||||
title: SO-100
|
||||
- local: koch
|
||||
title: Koch v1.1
|
||||
- local: lekiwi
|
||||
title: LeKiwi
|
||||
- local: hope_jr
|
||||
title: Hope Jr
|
||||
- local: reachy2
|
||||
title: Reachy 2
|
||||
- local: unitree_g1
|
||||
title: Unitree G1
|
||||
- local: earthrover_mini_plus
|
||||
title: Earth Rover Mini
|
||||
- local: omx
|
||||
title: OMX
|
||||
- local: openarm
|
||||
title: OpenArm
|
||||
- local: rebot_b601
|
||||
title: reBot B601-DM
|
||||
title: "Robots"
|
||||
- sections:
|
||||
- local: phone_teleop
|
||||
title: Phone
|
||||
title: "Teleoperators"
|
||||
- sections:
|
||||
- local: cameras
|
||||
title: Cameras
|
||||
title: "Sensors"
|
||||
- sections:
|
||||
- local: notebooks
|
||||
title: Notebooks
|
||||
- local: feetech
|
||||
title: Updating Feetech Firmware
|
||||
- local: damiao
|
||||
title: Damiao Motors and CAN Bus
|
||||
title: "Resources"
|
||||
- sections:
|
||||
- local: contributing
|
||||
title: Contribute to LeRobot
|
||||
- local: backwardcomp
|
||||
title: Backward compatibility
|
||||
title: "About"
|
||||
+168
-126
@@ -1,172 +1,214 @@
|
||||
# LeRobot documentation table of contents
|
||||
#
|
||||
# Ordering principle: gentle onboarding first, advanced/custom work last.
|
||||
# Within each top-level section the same rule applies — concept/overview pages
|
||||
# before reference/per-item pages.
|
||||
#
|
||||
# Pages marked "NEW (to create)" do not yet exist as .mdx files; they are
|
||||
# placeholders for the redesign and must be authored before the docs build.
|
||||
|
||||
- sections:
|
||||
- local: index
|
||||
title: LeRobot
|
||||
title: 🤗 LeRobot
|
||||
- local: quickstart # NEW (to create) — 15-min zero-to-trained-ACT path
|
||||
title: Quickstart
|
||||
- local: installation
|
||||
title: Installation
|
||||
- local: core_concepts # NEW (to create) — datasets, policies, processors, robots, envs in one mental model
|
||||
title: Core concepts
|
||||
- local: cheat-sheet
|
||||
title: Cheat sheet
|
||||
title: Command cheat sheet
|
||||
title: Get started
|
||||
|
||||
- sections:
|
||||
- local: il_robots
|
||||
title: Imitation Learning for Robots
|
||||
- local: bring_your_own_policies
|
||||
title: Adding a Policy
|
||||
- local: integrate_hardware
|
||||
title: Bring Your Own Hardware
|
||||
- local: hilserl
|
||||
title: Train a Robot with RL
|
||||
- local: hilserl_sim
|
||||
title: Train RL in Simulation
|
||||
- local: multi_gpu_training
|
||||
title: Multi GPU training
|
||||
title: Imitation learning end-to-end
|
||||
- local: hil_data_collection
|
||||
title: Human In the Loop Data Collection
|
||||
- local: peft_training
|
||||
title: Training with PEFT (e.g., LoRA)
|
||||
title: Human-in-the-loop data collection
|
||||
- local: inference
|
||||
title: Deploying a trained policy
|
||||
- local: rename_map
|
||||
title: Using Rename Map and Empty Cameras
|
||||
title: "Tutorials"
|
||||
title: Matching dataset keys to a policy (rename map)
|
||||
title: Your first project
|
||||
|
||||
- sections:
|
||||
- local: hardware_guide
|
||||
title: Compute Hardware Guide
|
||||
title: Compute hardware guide
|
||||
- local: torch_accelerators
|
||||
title: PyTorch accelerators
|
||||
title: "Compute & Hardware"
|
||||
- local: multi_gpu_training
|
||||
title: Multi-GPU training
|
||||
- local: peft_training
|
||||
title: Parameter-efficient fine-tuning (LoRA)
|
||||
title: Training
|
||||
|
||||
- sections:
|
||||
- local: lerobot-dataset-v3
|
||||
title: Using LeRobotDataset
|
||||
- local: porting_datasets_v3
|
||||
title: Porting Large Datasets
|
||||
- local: using_dataset_tools
|
||||
title: Using the Dataset Tools
|
||||
title: Dataset tools
|
||||
- local: language_and_recipes
|
||||
title: Language Columns and Recipes
|
||||
title: Language columns & recipes
|
||||
- local: tools
|
||||
title: Tools
|
||||
title: Tool calls in datasets
|
||||
- local: video_encoding_parameters
|
||||
title: Video encoding parameters
|
||||
- local: streaming_video_encoding
|
||||
title: Streaming Video Encoding
|
||||
title: "Datasets"
|
||||
title: Streaming video encoding
|
||||
- local: porting_datasets_v3
|
||||
title: Porting datasets to v3
|
||||
title: Datasets
|
||||
|
||||
- sections:
|
||||
- local: act
|
||||
title: ACT
|
||||
- local: smolvla
|
||||
title: SmolVLA
|
||||
- local: pi0
|
||||
title: π₀ (Pi0)
|
||||
- local: pi0fast
|
||||
title: π₀-FAST (Pi0Fast)
|
||||
- local: pi05
|
||||
title: π₀.₅ (Pi05)
|
||||
- local: eo1
|
||||
title: EO-1
|
||||
- local: groot
|
||||
title: NVIDIA GR00T N1.5
|
||||
- local: xvla
|
||||
title: X-VLA
|
||||
- local: multi_task_dit
|
||||
title: Multitask DiT Policy
|
||||
- local: walloss
|
||||
title: WALL-OSS
|
||||
title: "Policies"
|
||||
- local: policies_overview # NEW (to create) — concept hub + "choose a policy" decision guide
|
||||
title: Choosing a policy
|
||||
- sections:
|
||||
- local: act
|
||||
title: ACT
|
||||
- local: smolvla
|
||||
title: SmolVLA
|
||||
- local: pi0
|
||||
title: π₀ (Pi0)
|
||||
- local: pi0fast
|
||||
title: π₀-FAST
|
||||
- local: pi05
|
||||
title: π₀.₅ (Pi05)
|
||||
- local: eo1
|
||||
title: EO-1
|
||||
- local: groot
|
||||
title: NVIDIA GR00T N1.5
|
||||
- local: xvla
|
||||
title: X-VLA
|
||||
- local: walloss
|
||||
title: WALL-OSS
|
||||
- local: multi_task_dit
|
||||
title: Multitask DiT
|
||||
title: Policy reference
|
||||
title: Policies
|
||||
|
||||
- sections:
|
||||
- local: sarm
|
||||
title: SARM
|
||||
title: "Reward Models"
|
||||
- sections:
|
||||
- local: inference
|
||||
title: Policy Deployment (lerobot-rollout)
|
||||
- local: async
|
||||
title: Use Async Inference
|
||||
title: Async inference
|
||||
- local: rtc
|
||||
title: Real-Time Chunking (RTC)
|
||||
title: "Inference"
|
||||
title: Real-time chunking (RTC)
|
||||
title: Real-time deployment
|
||||
|
||||
- sections:
|
||||
- local: hilserl
|
||||
title: Train a robot with RL (HIL-SERL)
|
||||
- local: hilserl_sim
|
||||
title: Train RL in simulation
|
||||
- local: sarm
|
||||
title: SARM reward model
|
||||
title: Reinforcement learning
|
||||
|
||||
- sections:
|
||||
- local: envhub
|
||||
title: Environments from the Hub
|
||||
- local: envhub_leisaac
|
||||
title: Control & Train Robots in Sim (LeIsaac)
|
||||
title: "Simulation"
|
||||
- sections:
|
||||
- local: adding_benchmarks
|
||||
title: Adding a New Benchmark
|
||||
- local: libero
|
||||
title: LIBERO
|
||||
- local: libero_plus
|
||||
title: LIBERO-plus
|
||||
- local: metaworld
|
||||
title: Meta-World
|
||||
- local: robotwin
|
||||
title: RoboTwin 2.0
|
||||
- local: robocasa
|
||||
title: RoboCasa365
|
||||
- local: robocerebra
|
||||
title: RoboCerebra
|
||||
- local: robomme
|
||||
title: RoboMME
|
||||
title: LeIsaac — control & train in sim
|
||||
- local: envhub_isaaclab_arena
|
||||
title: NVIDIA IsaacLab Arena Environments
|
||||
- local: vlabench
|
||||
title: VLABench
|
||||
title: "Benchmarks"
|
||||
title: NVIDIA IsaacLab Arena environments
|
||||
- sections:
|
||||
- local: libero
|
||||
title: LIBERO
|
||||
- local: libero_plus
|
||||
title: LIBERO-plus
|
||||
- local: metaworld
|
||||
title: Meta-World
|
||||
- local: robotwin
|
||||
title: RoboTwin 2.0
|
||||
- local: robocasa
|
||||
title: RoboCasa365
|
||||
- local: robocerebra
|
||||
title: RoboCerebra
|
||||
- local: robomme
|
||||
title: RoboMME
|
||||
- local: vlabench
|
||||
title: VLABench
|
||||
title: Benchmark suites
|
||||
title: Simulation & benchmarks
|
||||
|
||||
- sections:
|
||||
- local: introduction_processors
|
||||
title: Introduction to Robot Processors
|
||||
- local: debug_processor_pipeline
|
||||
title: Debug your processor pipeline
|
||||
- local: implement_your_own_processor
|
||||
title: Implement your own processor
|
||||
title: Introduction to processors
|
||||
- local: processors_robots_teleop
|
||||
title: Processors for Robots and Teleoperators
|
||||
title: Processors for robots & teleoperators
|
||||
- local: env_processor
|
||||
title: Environment Processors
|
||||
title: Environment processors
|
||||
- local: action_representations
|
||||
title: Action Representations
|
||||
title: "Robot Processors"
|
||||
title: Action representations
|
||||
- local: debug_processor_pipeline
|
||||
title: Debugging a pipeline
|
||||
- local: implement_your_own_processor
|
||||
title: Implementing your own processor
|
||||
title: Processors
|
||||
|
||||
- sections:
|
||||
- local: so101
|
||||
title: SO-101
|
||||
- local: so100
|
||||
title: SO-100
|
||||
- local: koch
|
||||
title: Koch v1.1
|
||||
- local: lekiwi
|
||||
title: LeKiwi
|
||||
- local: hope_jr
|
||||
title: Hope Jr
|
||||
- local: reachy2
|
||||
title: Reachy 2
|
||||
- local: unitree_g1
|
||||
title: Unitree G1
|
||||
- local: earthrover_mini_plus
|
||||
title: Earth Rover Mini
|
||||
- local: omx
|
||||
title: OMX
|
||||
- local: openarm
|
||||
title: OpenArm
|
||||
- local: rebot_b601
|
||||
title: reBot B601-DM
|
||||
title: "Robots"
|
||||
- sections:
|
||||
- local: phone_teleop
|
||||
title: Phone
|
||||
title: "Teleoperators"
|
||||
- sections:
|
||||
- local: so101
|
||||
title: SO-101
|
||||
- local: so100
|
||||
title: SO-100
|
||||
- local: koch
|
||||
title: Koch v1.1
|
||||
- local: omx
|
||||
title: OMX
|
||||
- local: openarm
|
||||
title: OpenArm
|
||||
title: Low-cost arms
|
||||
- sections:
|
||||
- local: lekiwi
|
||||
title: LeKiwi
|
||||
- local: earthrover_mini_plus
|
||||
title: Earth Rover Mini
|
||||
title: Mobile platforms
|
||||
- sections:
|
||||
- local: hope_jr
|
||||
title: Hope Jr
|
||||
- local: reachy2
|
||||
title: Reachy 2
|
||||
- local: unitree_g1
|
||||
title: Unitree G1
|
||||
title: Bimanual & humanoid
|
||||
- sections:
|
||||
- local: rebot_b601
|
||||
title: reBot B601-DM
|
||||
title: Research & industrial
|
||||
title: Supported robots
|
||||
|
||||
- sections:
|
||||
- local: cameras
|
||||
title: Cameras
|
||||
title: "Sensors"
|
||||
- sections:
|
||||
- local: notebooks
|
||||
title: Notebooks
|
||||
- local: phone_teleop
|
||||
title: Phone teleoperation
|
||||
- local: feetech
|
||||
title: Updating Feetech Firmware
|
||||
title: Feetech firmware update
|
||||
- local: damiao
|
||||
title: Damiao Motors and CAN Bus
|
||||
title: "Resources"
|
||||
title: Damiao motors & CAN bus
|
||||
title: Sensors, teleop & motors
|
||||
|
||||
- sections:
|
||||
- local: contributing
|
||||
title: Contribute to LeRobot
|
||||
- local: integrate_hardware
|
||||
title: Bring your own hardware
|
||||
- local: bring_your_own_policies
|
||||
title: Add a new policy
|
||||
- local: adding_benchmarks
|
||||
title: Add a new benchmark
|
||||
title: Extend LeRobot
|
||||
|
||||
- sections:
|
||||
- local: troubleshooting # NEW (to create) — common errors: USB, calibration drift, CUDA OOM, video decoding…
|
||||
title: Troubleshooting & FAQ
|
||||
- local: glossary # NEW (to create) — episode, action chunk, leader/follower, teleop, processor…
|
||||
title: Glossary
|
||||
- local: notebooks
|
||||
title: Example notebooks
|
||||
- local: backwardcomp
|
||||
title: Backward compatibility
|
||||
title: "About"
|
||||
title: Reference
|
||||
|
||||
- sections:
|
||||
- local: contributing
|
||||
title: Contributing to LeRobot
|
||||
title: About
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
# Quickstart
|
||||
|
||||
This is the **shortest path** from an unboxed SO-101 to a policy that drives your own robot. Every step is copy-paste; replace the **`<placeholders>`** with the values for your setup.
|
||||
|
||||
By the end you will have:
|
||||
|
||||
- A calibrated SO-101 leader + follower pair.
|
||||
- A dataset of 30 episodes pushed to the Hugging Face Hub.
|
||||
- A trained ACT policy (~20k steps) running on your robot via `lerobot-rollout`.
|
||||
|
||||
> [!NOTE]
|
||||
> **How long will this take?**
|
||||
> Recording 30 episodes is roughly 30–60 minutes of teleoperation. Training ACT for 20k steps takes ~1.5h on an A100, a few hours on a laptop RTX 3060, longer on Apple Silicon (`mps`). The commands themselves are quick — most of the wall-clock is data collection and training.
|
||||
|
||||
> [!TIP]
|
||||
> If you only want to **understand the codebase** or **train on an existing dataset without hardware**, this page isn't for you. Read [Core concepts](./core_concepts) first, then jump to [Imitation learning end-to-end](./il_robots).
|
||||
|
||||
---
|
||||
|
||||
## Before you start
|
||||
|
||||
You need:
|
||||
|
||||
- An **assembled SO-101 leader + follower pair**. If your robot is not assembled yet, follow the [SO-101 assembly guide](./so101) and come back here.
|
||||
- **One or two cameras** (USB webcam works fine).
|
||||
- A **CUDA GPU with ≥ 6 GB VRAM** (ACT is light — a laptop RTX 3060 works). Apple Silicon (`mps`) and CPU are supported but slower. See the [compute hardware guide](./hardware_guide) for sizing.
|
||||
- A **Hugging Face account** — datasets and the trained policy will be pushed to your Hub.
|
||||
|
||||
If any of the above is missing, fix it first; the rest of the page assumes it.
|
||||
|
||||
---
|
||||
|
||||
## Step 1 — Install LeRobot
|
||||
|
||||
Follow the full [Installation Guide](./installation) for environment setup, then add the SO-101 motor stack and log in to the Hub:
|
||||
|
||||
```bash
|
||||
pip install 'lerobot[feetech]'
|
||||
git lfs install && git lfs pull
|
||||
hf auth login # paste a token from https://huggingface.co/settings/tokens
|
||||
```
|
||||
|
||||
Sanity check — the CLI entry points should be available:
|
||||
|
||||
```bash
|
||||
lerobot-find-port --help
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 2 — Identify USB ports and motor IDs
|
||||
|
||||
Plug **only the follower arm** in (USB + power) and run:
|
||||
|
||||
```bash
|
||||
lerobot-find-port
|
||||
```
|
||||
|
||||
When prompted, unplug it and press Enter. Note the printed port — that's your `<FOLLOWER_PORT>`. Repeat with only the **leader arm** plugged in to get `<LEADER_PORT>`.
|
||||
|
||||
> [!TIP]
|
||||
> On Linux, USB ports look like `/dev/ttyACM0`; on macOS like `/dev/tty.usbmodem...`. On Linux you may need `sudo chmod 666 /dev/ttyACM0` to grant access.
|
||||
|
||||
If your motors are brand-new (or repurposed), set their IDs and baudrate **once per arm**:
|
||||
|
||||
```bash
|
||||
lerobot-setup-motors --robot.type=so101_follower --robot.port=<FOLLOWER_PORT>
|
||||
lerobot-setup-motors --teleop.type=so101_leader --teleop.port=<LEADER_PORT>
|
||||
```
|
||||
|
||||
The script walks you through connecting motors one at a time. Full details: [SO-101 → Configure the motors](./so101#configure-the-motors).
|
||||
|
||||
---
|
||||
|
||||
## Step 3 — Calibrate
|
||||
|
||||
Center every joint roughly in the middle of its range, then run:
|
||||
|
||||
```bash
|
||||
lerobot-calibrate \
|
||||
--robot.type=so101_follower \
|
||||
--robot.port=<FOLLOWER_PORT> \
|
||||
--robot.id=my_follower
|
||||
|
||||
lerobot-calibrate \
|
||||
--teleop.type=so101_leader \
|
||||
--teleop.port=<LEADER_PORT> \
|
||||
--teleop.id=my_leader
|
||||
```
|
||||
|
||||
After pressing Enter, sweep each joint through its full range of motion, then press Enter again to finish.
|
||||
|
||||
> [!WARNING]
|
||||
> The `--robot.id` / `--teleop.id` values (`my_follower`, `my_leader`) become the **calibration keys**. Reuse the same IDs in every later command — that's how LeRobot finds the calibration on disk.
|
||||
|
||||
Watch the [calibration video](./so101#calibrate) if anything is unclear.
|
||||
|
||||
---
|
||||
|
||||
## Step 4 — Teleoperate (sanity check, no recording)
|
||||
|
||||
Before recording anything, confirm the leader drives the follower correctly:
|
||||
|
||||
```bash
|
||||
lerobot-teleoperate \
|
||||
--robot.type=so101_follower \
|
||||
--robot.port=<FOLLOWER_PORT> \
|
||||
--robot.id=my_follower \
|
||||
--robot.cameras="{ top: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30} }" \
|
||||
--teleop.type=so101_leader \
|
||||
--teleop.port=<LEADER_PORT> \
|
||||
--teleop.id=my_leader \
|
||||
--display_data=true
|
||||
```
|
||||
|
||||
A Rerun window should open showing the camera feed and joint angles. Move the leader — the follower should mirror it in real time. If it doesn't, see [Troubleshooting & FAQ](./troubleshooting).
|
||||
|
||||
Don't know which camera index is which? Run `lerobot-find-cameras` — it saves a frame from each detected camera so you can pick the right one.
|
||||
|
||||
---
|
||||
|
||||
## Step 5 — Record a dataset (30 episodes)
|
||||
|
||||
Now record demonstrations. Pick a short, repeatable task (e.g. *"put the red brick in the bowl"*). The dataset is pushed to the Hub under your username:
|
||||
|
||||
```bash
|
||||
export HF_USER=<your-hf-username>
|
||||
|
||||
lerobot-record \
|
||||
--robot.type=so101_follower \
|
||||
--robot.port=<FOLLOWER_PORT> \
|
||||
--robot.id=my_follower \
|
||||
--robot.cameras="{ top: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}, wrist: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30} }" \
|
||||
--teleop.type=so101_leader \
|
||||
--teleop.port=<LEADER_PORT> \
|
||||
--teleop.id=my_leader \
|
||||
--dataset.repo_id=${HF_USER}/so101_quickstart \
|
||||
--dataset.num_episodes=30 \
|
||||
--dataset.single_task="Put the red brick in the bowl" \
|
||||
--dataset.streaming_encoding=true \
|
||||
--display_data=true
|
||||
```
|
||||
|
||||
**Keyboard controls during recording:**
|
||||
|
||||
- **`→` (Right Arrow)** — save the current episode and move to the next.
|
||||
- **`←` (Left Arrow)** — discard the current episode and retry.
|
||||
- **`Esc`** — stop, encode videos, and upload to the Hub.
|
||||
|
||||
> [!TIP]
|
||||
> **Quality beats quantity.** 30 clean, varied episodes (different brick positions, lighting, camera shake) train a much better policy than 100 identical ones. Move the object around. Vary your speed slightly.
|
||||
|
||||
When you're done, your dataset lives at `https://huggingface.co/datasets/${HF_USER}/so101_quickstart`. You can preview it in the browser. For deeper recording options (resume, multiple tasks, custom processors), see [Imitation learning end-to-end → Record](./il_robots#record-a-dataset).
|
||||
|
||||
---
|
||||
|
||||
## Step 6 — Train ACT
|
||||
|
||||
ACT (Action Chunking Transformer) is the right default for a first run — small, fast, and works well on 30 episodes.
|
||||
|
||||
```bash
|
||||
lerobot-train \
|
||||
--dataset.repo_id=${HF_USER}/so101_quickstart \
|
||||
--policy.type=act \
|
||||
--output_dir=outputs/train/act_so101_quickstart \
|
||||
--job_name=act_so101_quickstart \
|
||||
--policy.device=cuda \
|
||||
--policy.repo_id=${HF_USER}/act_so101_quickstart \
|
||||
--steps=20000 \
|
||||
--wandb.enable=true
|
||||
```
|
||||
|
||||
A few notes:
|
||||
|
||||
- Replace `--policy.device=cuda` with `mps` on Apple Silicon, or `cpu` if you have no GPU (very slow — not recommended for a real run).
|
||||
- `--wandb.enable=true` is optional. If you use it, run `wandb login` first. Otherwise drop the flag.
|
||||
- Checkpoints land in `outputs/train/act_so101_quickstart/checkpoints/`. The final model is also pushed to the Hub at the `--policy.repo_id` you specified.
|
||||
- To resume from an interruption: `lerobot-train --config_path=outputs/train/act_so101_quickstart/checkpoints/last/pretrained_model/train_config.json --resume=true`.
|
||||
|
||||
> [!TIP]
|
||||
> **No GPU locally?** Train on Google Colab using the [ACT notebook](./notebooks#training-act), or rent a GPU via [Hugging Face Jobs](./il_robots#train-using-hugging-face-jobs) — pay-as-you-go, no setup.
|
||||
|
||||
For why ACT is the default and when to switch to SmolVLA, Pi0, or another policy, see [Choosing a policy](./policies_overview).
|
||||
|
||||
---
|
||||
|
||||
## Step 7 — Run your policy on the robot
|
||||
|
||||
Deploy with `lerobot-rollout`. **Use the same camera layout you used while recording** — keys and resolutions must match.
|
||||
|
||||
```bash
|
||||
lerobot-rollout \
|
||||
--strategy.type=base \
|
||||
--policy.path=${HF_USER}/act_so101_quickstart \
|
||||
--robot.type=so101_follower \
|
||||
--robot.port=<FOLLOWER_PORT> \
|
||||
--robot.id=my_follower \
|
||||
--robot.cameras="{ top: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}, wrist: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30} }" \
|
||||
--task="Put the red brick in the bowl" \
|
||||
--duration=60
|
||||
```
|
||||
|
||||
`--duration` is in seconds — leave it off to run until you stop the script. You should see the follower arm move on its own, attempting the task.
|
||||
|
||||
If observations from the robot use different keys than the policy expects, you'll need a [rename map](./rename_map). If latency matters, look at [async inference](./async) and [real-time chunking](./rtc).
|
||||
|
||||
---
|
||||
|
||||
## You're done 🎉
|
||||
|
||||
You now have a working IL pipeline end-to-end. From here, the natural next steps are:
|
||||
|
||||
- **Improve the policy** — record more diverse episodes, train longer, or try a stronger model. See [Choosing a policy](./policies_overview).
|
||||
- **Go deeper on imitation learning** — [Imitation learning end-to-end](./il_robots) covers multi-camera setups, multi-task datasets, episode replay, evaluation, and Hugging Face Jobs.
|
||||
- **Try RL with a human in the loop** — [HIL-SERL](./hilserl) trains a policy that improves while you correct it.
|
||||
- **Use a different robot** — see [Supported robots](./so101) for low-cost arms, mobile platforms, bimanual, and humanoid.
|
||||
- **Build something new** — [Bring your own hardware](./integrate_hardware) and [Add a new policy](./bring_your_own_policies).
|
||||
|
||||
Stuck on something? Check [Troubleshooting & FAQ](./troubleshooting), or ask on [Discord](https://discord.gg/s3KuuzsPFb).
|
||||
@@ -255,7 +255,8 @@ def extract_path_fields_from_config(config_path: str, path_fields: list[str]) ->
|
||||
remaining = config_data[field]
|
||||
if remaining:
|
||||
_config_yaml_overrides[field] = _flatten_to_cli_args(remaining)
|
||||
del config_data[field]
|
||||
else:
|
||||
del config_data[field]
|
||||
modified = True
|
||||
|
||||
if not modified:
|
||||
@@ -310,13 +311,7 @@ def wrap(config_path: Path | None = None) -> Callable[[F], F]:
|
||||
cli_args = filter_arg("config_path", cli_args)
|
||||
cfg = argtype.from_pretrained(config_path_cli, cli_args=cli_args)
|
||||
else:
|
||||
if config_path_cli:
|
||||
cli_args = filter_arg("config_path", cli_args)
|
||||
cfg = draccus.parse(
|
||||
config_class=argtype,
|
||||
config_path=config_path_cli or config_path,
|
||||
args=cli_args,
|
||||
)
|
||||
cfg = draccus.parse(config_class=argtype, config_path=config_path, args=cli_args)
|
||||
response = fn(cfg, *args, **kwargs)
|
||||
return response
|
||||
|
||||
|
||||
@@ -60,7 +60,6 @@ class Eagle25VLPreTrainedModel(PreTrainedModel):
|
||||
"SiglipEncoderLayer",
|
||||
]
|
||||
_skip_keys_device_placement = "past_key_values"
|
||||
_supports_flash_attn = True
|
||||
_supports_flash_attn_2 = True
|
||||
_supports_cache_class = True
|
||||
_supports_static_cache = True
|
||||
|
||||
@@ -124,6 +124,7 @@ class Eagle25VLProcessor(ProcessorMixin):
|
||||
"videos_kwargs",
|
||||
"text_kwargs",
|
||||
]
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -206,11 +206,7 @@ def _build_eagle_processor(tokenizer_assets_repo: str = DEFAULT_TOKENIZER_ASSETS
|
||||
"Vendor files are copied during model creation. Create the policy/model first, "
|
||||
"or call ensure_eagle_cache_ready() before building processors."
|
||||
)
|
||||
proc = AutoProcessor.from_pretrained(
|
||||
str(cache_dir),
|
||||
trust_remote_code=True,
|
||||
fix_mistral_regex=False,
|
||||
)
|
||||
proc = AutoProcessor.from_pretrained(str(cache_dir), trust_remote_code=True, use_fast=True)
|
||||
proc.tokenizer.padding_side = "left"
|
||||
return proc
|
||||
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
"""Tests for policy.path support in YAML config files (issue #2957)."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from unittest.mock import patch
|
||||
|
||||
import yaml
|
||||
|
||||
from lerobot.configs import parser
|
||||
from lerobot.configs.parser import (
|
||||
_config_path_args,
|
||||
_config_yaml_overrides,
|
||||
@@ -20,8 +16,7 @@ from lerobot.configs.parser import (
|
||||
|
||||
|
||||
def test_extract_path_fields_from_yaml():
|
||||
"""Test that policy.path is extracted from a YAML config and the policy block
|
||||
is removed entirely (siblings are captured separately as cli_overrides)."""
|
||||
"""Test that policy.path is extracted from a YAML config and removed."""
|
||||
config = {
|
||||
"dataset": {"repo_id": "lerobot/pusht"},
|
||||
"policy": {"type": "smolvla", "path": "lerobot/smolvla_base", "push_to_hub": False},
|
||||
@@ -31,33 +26,26 @@ def test_extract_path_fields_from_yaml():
|
||||
config_path = f.name
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
||||
|
||||
# Path should be extracted and stored
|
||||
assert _config_path_args["policy"] == "lerobot/smolvla_base"
|
||||
|
||||
# Cleaned config should not have the policy block at all -- draccus must not
|
||||
# try to decode it as PreTrainedConfig; the actual config comes from
|
||||
# from_pretrained(path) with the captured overrides applied on top.
|
||||
# Cleaned config should not have the path field
|
||||
with open(cleaned_path) as f:
|
||||
cleaned = yaml.safe_load(f)
|
||||
assert "policy" not in cleaned
|
||||
assert "path" not in cleaned["policy"]
|
||||
assert cleaned["policy"]["type"] == "smolvla"
|
||||
assert cleaned["policy"]["push_to_hub"] is False
|
||||
|
||||
# Original dataset should be untouched
|
||||
assert cleaned["dataset"]["repo_id"] == "lerobot/pusht"
|
||||
|
||||
# Sibling overrides (excluding type/path) captured for from_pretrained.
|
||||
overrides = get_yaml_overrides("policy")
|
||||
assert any("push_to_hub=false" in o for o in overrides)
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
|
||||
|
||||
def test_extract_path_fields_from_json():
|
||||
"""Test that policy.path is extracted from a JSON config and the policy
|
||||
block is removed entirely."""
|
||||
"""Test that policy.path is extracted from a JSON config."""
|
||||
config = {
|
||||
"policy": {"type": "act", "path": "some/local/path"},
|
||||
}
|
||||
@@ -66,17 +54,15 @@ def test_extract_path_fields_from_json():
|
||||
config_path = f.name
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
||||
|
||||
assert _config_path_args["policy"] == "some/local/path"
|
||||
|
||||
with open(cleaned_path) as f:
|
||||
cleaned = json.load(f)
|
||||
assert "policy" not in cleaned
|
||||
assert "path" not in cleaned["policy"]
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
|
||||
|
||||
def test_extract_no_path_returns_original():
|
||||
@@ -230,91 +216,3 @@ def test_flatten_nested_with_bools():
|
||||
args = _flatten_to_cli_args(d)
|
||||
assert "--optimizer.use_warmup=true" in args
|
||||
assert "--optimizer.lr=0.01" in args
|
||||
|
||||
|
||||
def test_extract_removes_field_with_siblings_and_no_type():
|
||||
"""Regression: when policy.path has siblings but no type:, the entire policy
|
||||
block must still be removed from the cleaned config. Otherwise draccus tries
|
||||
to decode the leftover dict as PreTrainedConfig and crashes on the missing
|
||||
type discriminator.
|
||||
"""
|
||||
config = {
|
||||
"dataset": {"repo_id": "lerobot/pusht"},
|
||||
"policy": {
|
||||
"path": "lerobot/smolvla_base",
|
||||
"n_action_steps": 10,
|
||||
"dtype": "bfloat16",
|
||||
},
|
||||
}
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
|
||||
yaml.dump(config, f)
|
||||
config_path = f.name
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
cleaned_path = extract_path_fields_from_config(config_path, ["policy"])
|
||||
|
||||
with open(cleaned_path) as f:
|
||||
cleaned = yaml.safe_load(f) or {}
|
||||
assert "policy" not in cleaned, "policy block should be fully removed when path is present"
|
||||
assert cleaned["dataset"]["repo_id"] == "lerobot/pusht"
|
||||
assert _config_path_args["policy"] == "lerobot/smolvla_base"
|
||||
overrides = get_yaml_overrides("policy")
|
||||
assert any("n_action_steps=10" in o for o in overrides)
|
||||
assert any("dtype=bfloat16" in o for o in overrides)
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DummyNested:
|
||||
foo: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DummyConfig:
|
||||
nested: _DummyNested = field(default_factory=_DummyNested)
|
||||
other: str = "default"
|
||||
|
||||
@classmethod
|
||||
def __get_path_fields__(cls):
|
||||
return ["nested"]
|
||||
|
||||
|
||||
def test_wrap_uses_cleaned_config_for_draccus_parse():
|
||||
"""Regression: wrap() updates config_path_cli to point at the cleaned temp
|
||||
file but must propagate that to the draccus.parse fallback branch. Without
|
||||
the fix, cli_args still contains --config_path=<original> and draccus reads
|
||||
the original YAML with `path:` still in it, crashing on the unknown field.
|
||||
"""
|
||||
config = {
|
||||
"nested": {"path": "some/checkpoint", "foo": 42},
|
||||
"other": "set-via-yaml",
|
||||
}
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
|
||||
yaml.dump(config, f)
|
||||
config_path = f.name
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
@parser.wrap()
|
||||
def main(cfg: _DummyConfig) -> _DummyConfig:
|
||||
captured["cfg"] = cfg
|
||||
return cfg
|
||||
|
||||
with patch.object(sys, "argv", ["prog", f"--config_path={config_path}"]):
|
||||
main()
|
||||
|
||||
assert captured["cfg"].other == "set-via-yaml"
|
||||
assert _config_path_args["nested"] == "some/checkpoint"
|
||||
# Cleaned config dropped `nested:` entirely; defaults stand for this wrapper
|
||||
# class (a real PreTrainedConfig would now load the checkpoint and apply
|
||||
# the captured yaml_overrides via from_pretrained()).
|
||||
assert captured["cfg"].nested.foo == 0
|
||||
|
||||
_config_path_args.clear()
|
||||
_config_yaml_overrides.clear()
|
||||
|
||||
Reference in New Issue
Block a user