mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-22 03:59:42 +00:00
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
This commit is contained in:
committed by
Adil Zouitine
parent
35612c61e1
commit
14c2ece004
@@ -3,6 +3,7 @@
|
|||||||
This tutorial will teach you how to use RobotProcessor, a powerful pipeline system for preprocessing robot observations and postprocessing actions. You'll learn how to create modular, composable data transformations that make your robot learning pipelines more maintainable and reproducible.
|
This tutorial will teach you how to use RobotProcessor, a powerful pipeline system for preprocessing robot observations and postprocessing actions. You'll learn how to create modular, composable data transformations that make your robot learning pipelines more maintainable and reproducible.
|
||||||
|
|
||||||
**You'll learn:**
|
**You'll learn:**
|
||||||
|
|
||||||
1. What RobotProcessor is and why it's better than alternatives
|
1. What RobotProcessor is and why it's better than alternatives
|
||||||
2. How to create and compose processor steps
|
2. How to create and compose processor steps
|
||||||
3. How to save, load, and share your processors
|
3. How to save, load, and share your processors
|
||||||
@@ -13,10 +14,13 @@ This tutorial will teach you how to use RobotProcessor, a powerful pipeline syst
|
|||||||
Before diving into code, let's understand the real problems RobotProcessor solves. If you've worked with robots and learned policies, you've likely encountered these challenges:
|
Before diving into code, let's understand the real problems RobotProcessor solves. If you've worked with robots and learned policies, you've likely encountered these challenges:
|
||||||
|
|
||||||
### Observation Key Mismatches
|
### Observation Key Mismatches
|
||||||
|
|
||||||
Your environment might output observations with keys like `"rgb_camera_front"` and `"joint_positions"`, but your pre-trained policy expects `"observation.images.wrist"` and `"observation.state"`. You need to rename these keys consistently across your pipeline.
|
Your environment might output observations with keys like `"rgb_camera_front"` and `"joint_positions"`, but your pre-trained policy expects `"observation.images.wrist"` and `"observation.state"`. You need to rename these keys consistently across your pipeline.
|
||||||
|
|
||||||
### Image Preprocessing Requirements
|
### Image Preprocessing Requirements
|
||||||
|
|
||||||
Your robot's camera captures 1920x1080 RGB images, but your policy was trained on 224x224 cropped images focusing on the workspace. You need to:
|
Your robot's camera captures 1920x1080 RGB images, but your policy was trained on 224x224 cropped images focusing on the workspace. You need to:
|
||||||
|
|
||||||
- Crop out the relevant workspace area
|
- Crop out the relevant workspace area
|
||||||
- Resize to the expected dimensions
|
- Resize to the expected dimensions
|
||||||
- Convert from uint8 [0,255] to float32 [0,1]
|
- Convert from uint8 [0,255] to float32 [0,1]
|
||||||
@@ -24,27 +28,35 @@ Your robot's camera captures 1920x1080 RGB images, but your policy was trained o
|
|||||||
- Move everything to GPU efficiently
|
- Move everything to GPU efficiently
|
||||||
|
|
||||||
### Coordinate System Transformations
|
### Coordinate System Transformations
|
||||||
|
|
||||||
Your policy outputs desired end-effector positions in Cartesian space, but your robot expects joint angles. You need to:
|
Your policy outputs desired end-effector positions in Cartesian space, but your robot expects joint angles. You need to:
|
||||||
|
|
||||||
- Apply inverse kinematics to convert positions to joint angles
|
- Apply inverse kinematics to convert positions to joint angles
|
||||||
- Handle singularities and out-of-reach positions
|
- Handle singularities and out-of-reach positions
|
||||||
- Smooth the resulting trajectories to avoid jerky movements
|
- Smooth the resulting trajectories to avoid jerky movements
|
||||||
- Clip values to respect joint limits
|
- Clip values to respect joint limits
|
||||||
|
|
||||||
### State Augmentation
|
### State Augmentation
|
||||||
|
|
||||||
Your environment provides joint positions, but your policy also needs velocities and accelerations. You need to:
|
Your environment provides joint positions, but your policy also needs velocities and accelerations. You need to:
|
||||||
|
|
||||||
- Calculate velocities from position differences
|
- Calculate velocities from position differences
|
||||||
- Maintain a buffer of past states
|
- Maintain a buffer of past states
|
||||||
- Handle episode boundaries correctly
|
- Handle episode boundaries correctly
|
||||||
|
|
||||||
### Multi-Robot Deployment
|
### Multi-Robot Deployment
|
||||||
|
|
||||||
You want to deploy the same policy on different robots with varying:
|
You want to deploy the same policy on different robots with varying:
|
||||||
|
|
||||||
- Camera setups (different resolutions, positions)
|
- Camera setups (different resolutions, positions)
|
||||||
- Joint configurations (6-DOF vs 7-DOF arms)
|
- Joint configurations (6-DOF vs 7-DOF arms)
|
||||||
- Control frequencies (10Hz vs 30Hz)
|
- Control frequencies (10Hz vs 30Hz)
|
||||||
- Safety limits and workspace boundaries
|
- Safety limits and workspace boundaries
|
||||||
|
|
||||||
### Data Collection and Training
|
### Data Collection and Training
|
||||||
|
|
||||||
During data collection, you need to:
|
During data collection, you need to:
|
||||||
|
|
||||||
- Record raw sensor data for dataset creation
|
- Record raw sensor data for dataset creation
|
||||||
- Apply the same preprocessing as during deployment
|
- Apply the same preprocessing as during deployment
|
||||||
- Ensure reproducibility across different operators
|
- Ensure reproducibility across different operators
|
||||||
@@ -87,6 +99,7 @@ def preprocess_observation(obs, device='cuda'):
|
|||||||
```
|
```
|
||||||
|
|
||||||
This approach has several problems:
|
This approach has several problems:
|
||||||
|
|
||||||
- **Mixed concerns**: Image processing, tensor conversion, device management all in one function
|
- **Mixed concerns**: Image processing, tensor conversion, device management all in one function
|
||||||
- **Hard to test**: Can't test image processing separately from state processing
|
- **Hard to test**: Can't test image processing separately from state processing
|
||||||
- **Device management scattered**: Easy to forget `.to(device)` calls
|
- **Device management scattered**: Easy to forget `.to(device)` calls
|
||||||
@@ -345,16 +358,18 @@ for step in range(1000):
|
|||||||
### When to Use Different Output Formats
|
### When to Use Different Output Formats
|
||||||
|
|
||||||
**Use EnvTransition tuple output when:**
|
**Use EnvTransition tuple output when:**
|
||||||
|
|
||||||
- Environment interaction and real-time control
|
- Environment interaction and real-time control
|
||||||
- You need to access individual transition components frequently
|
- You need to access individual transition components frequently
|
||||||
- Performance is critical (avoids dictionary creation overhead)
|
- Performance is critical (avoids dictionary creation overhead)
|
||||||
- Working with gym environments that expect tuple format
|
- Working with gym environments that expect tuple format
|
||||||
|
|
||||||
**Use batch dictionary output when:**
|
**Use batch dictionary output when:**
|
||||||
|
|
||||||
- Training with LeRobot datasets
|
- Training with LeRobot datasets
|
||||||
- Working with DataLoaders and batched processing
|
- Working with DataLoaders and batched processing
|
||||||
- Interfacing with existing LeRobot training code
|
- Interfacing with existing LeRobot training code
|
||||||
- You need the standardized "next.*" key format
|
- You need the standardized "next.\*" key format
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# For environment interaction - use tuple output
|
# For environment interaction - use tuple output
|
||||||
@@ -384,7 +399,7 @@ for step in range(1000):
|
|||||||
|
|
||||||
### Why "next.reward", "next.done", "next.truncated"?
|
### Why "next.reward", "next.done", "next.truncated"?
|
||||||
|
|
||||||
The default conversion uses "next.*" prefixes because this matches the standard format used by LeRobot datasets and follows the convention that rewards, done flags, and truncated flags are the result of taking an action (i.e., they come from the "next" state):
|
The default conversion uses "next.\*" prefixes because this matches the standard format used by LeRobot datasets and follows the convention that rewards, done flags, and truncated flags are the result of taking an action (i.e., they come from the "next" state):
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# Standard RL transition format
|
# Standard RL transition format
|
||||||
@@ -822,6 +837,7 @@ loaded_processor = RobotProcessor.from_pretrained(
|
|||||||
### How Overrides Work
|
### How Overrides Work
|
||||||
|
|
||||||
The `overrides` parameter is a dictionary where:
|
The `overrides` parameter is a dictionary where:
|
||||||
|
|
||||||
- **Keys** are step identifiers (class names for unregistered steps, registry names for registered steps)
|
- **Keys** are step identifiers (class names for unregistered steps, registry names for registered steps)
|
||||||
- **Values** are dictionaries of parameter overrides that get merged with saved configurations
|
- **Values** are dictionaries of parameter overrides that get merged with saved configurations
|
||||||
|
|
||||||
@@ -833,6 +849,7 @@ overrides = {
|
|||||||
```
|
```
|
||||||
|
|
||||||
The loading process:
|
The loading process:
|
||||||
|
|
||||||
1. Load saved configuration from JSON
|
1. Load saved configuration from JSON
|
||||||
2. For each step, check if overrides exist for that step
|
2. For each step, check if overrides exist for that step
|
||||||
3. Merge override parameters with saved parameters (overrides take precedence)
|
3. Merge override parameters with saved parameters (overrides take precedence)
|
||||||
@@ -981,6 +998,7 @@ real_processor = RobotProcessor.from_pretrained(
|
|||||||
The override system uses exact string matching:
|
The override system uses exact string matching:
|
||||||
|
|
||||||
#### For Registered Steps
|
#### For Registered Steps
|
||||||
|
|
||||||
Use the registry name (the string passed to `@ProcessorStepRegistry.register()`):
|
Use the registry name (the string passed to `@ProcessorStepRegistry.register()`):
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -993,6 +1011,7 @@ overrides = {"my_custom_step": {"param": "value"}}
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### For Unregistered Steps
|
#### For Unregistered Steps
|
||||||
|
|
||||||
Use the exact class name:
|
Use the exact class name:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -1008,6 +1027,7 @@ overrides = {"MyUnregisteredStep": {"param": "value"}}
|
|||||||
The override system provides helpful error messages:
|
The override system provides helpful error messages:
|
||||||
|
|
||||||
#### Invalid Override Keys
|
#### Invalid Override Keys
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# This will raise KeyError with helpful message
|
# This will raise KeyError with helpful message
|
||||||
overrides = {"NonExistentStep": {"param": "value"}}
|
overrides = {"NonExistentStep": {"param": "value"}}
|
||||||
@@ -1021,6 +1041,7 @@ except KeyError as e:
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### Instantiation Errors
|
#### Instantiation Errors
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# Invalid parameter types are caught
|
# Invalid parameter types are caught
|
||||||
overrides = {"MyStep": {"numeric_param": "not_a_number"}}
|
overrides = {"MyStep": {"numeric_param": "not_a_number"}}
|
||||||
@@ -1068,6 +1089,7 @@ overrides = {
|
|||||||
### Best Practices for Overrides
|
### Best Practices for Overrides
|
||||||
|
|
||||||
#### 1. Design Steps for Overrides
|
#### 1. Design Steps for Overrides
|
||||||
|
|
||||||
When creating steps that need non-serializable objects, design them with overrides in mind:
|
When creating steps that need non-serializable objects, design them with overrides in mind:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -1095,6 +1117,7 @@ class WellDesignedStep:
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### 2. Use Registry Names for Clarity
|
#### 2. Use Registry Names for Clarity
|
||||||
|
|
||||||
Register steps with descriptive names to make overrides clearer:
|
Register steps with descriptive names to make overrides clearer:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -1114,6 +1137,7 @@ overrides = {
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### 3. Document Override Requirements
|
#### 3. Document Override Requirements
|
||||||
|
|
||||||
Include clear documentation about what overrides are needed:
|
Include clear documentation about what overrides are needed:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -1134,6 +1158,7 @@ class VisionProcessor:
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### 4. Environment-Specific Configuration Files
|
#### 4. Environment-Specific Configuration Files
|
||||||
|
|
||||||
Create configuration helpers for different deployment environments:
|
Create configuration helpers for different deployment environments:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -1387,7 +1412,6 @@ class WorkspaceCropper:
|
|||||||
return (obs, *transition[1:])
|
return (obs, *transition[1:])
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
### Building Complete Pipelines for Different Robots
|
### Building Complete Pipelines for Different Robots
|
||||||
|
|
||||||
Now you can compose these steps into robot-specific pipelines:
|
Now you can compose these steps into robot-specific pipelines:
|
||||||
@@ -1435,6 +1459,7 @@ processor = RobotProcessor.from_pretrained("my-lab/robot-a-act-processor")
|
|||||||
```
|
```
|
||||||
|
|
||||||
The beauty of this approach is that:
|
The beauty of this approach is that:
|
||||||
|
|
||||||
1. **Each transformation is tested independently** - The IK solver can be validated separately from image cropping
|
1. **Each transformation is tested independently** - The IK solver can be validated separately from image cropping
|
||||||
2. **Configurations are shareable** - Other labs can use your exact preprocessing setup
|
2. **Configurations are shareable** - Other labs can use your exact preprocessing setup
|
||||||
3. **Pipelines are composable** - Mix and match steps for different robots
|
3. **Pipelines are composable** - Mix and match steps for different robots
|
||||||
@@ -1443,6 +1468,7 @@ The beauty of this approach is that:
|
|||||||
## Best Practices for Processor Steps
|
## Best Practices for Processor Steps
|
||||||
|
|
||||||
### 1. Always Check for None
|
### 1. Always Check for None
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def __call__(self, transition: EnvTransition) -> EnvTransition:
|
def __call__(self, transition: EnvTransition) -> EnvTransition:
|
||||||
obs = transition[TransitionIndex.OBSERVATION]
|
obs = transition[TransitionIndex.OBSERVATION]
|
||||||
@@ -1457,6 +1483,7 @@ def __call__(self, transition: EnvTransition) -> EnvTransition:
|
|||||||
```
|
```
|
||||||
|
|
||||||
### 2. Preserve Transition Structure
|
### 2. Preserve Transition Structure
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# Good - preserve all elements
|
# Good - preserve all elements
|
||||||
return (modified_obs, *transition[1:])
|
return (modified_obs, *transition[1:])
|
||||||
@@ -1466,6 +1493,7 @@ return (modified_obs, None, 0.0, False, False, {}, {})
|
|||||||
```
|
```
|
||||||
|
|
||||||
### 3. Clone When Storing State
|
### 3. Clone When Storing State
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def __call__(self, transition: EnvTransition) -> EnvTransition:
|
def __call__(self, transition: EnvTransition) -> EnvTransition:
|
||||||
obs = transition[TransitionIndex.OBSERVATION]
|
obs = transition[TransitionIndex.OBSERVATION]
|
||||||
@@ -1479,6 +1507,7 @@ def __call__(self, transition: EnvTransition) -> EnvTransition:
|
|||||||
```
|
```
|
||||||
|
|
||||||
### 4. Handle Device Transfers in state_dict
|
### 4. Handle Device Transfers in state_dict
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def state_dict(self) -> Dict[str, torch.Tensor]:
|
def state_dict(self) -> Dict[str, torch.Tensor]:
|
||||||
if self.buffer is None:
|
if self.buffer is None:
|
||||||
@@ -1689,6 +1718,7 @@ RobotProcessor provides a powerful, modular approach to data preprocessing in ro
|
|||||||
- **Flexible conversion**: Customize `to_transition` and `to_output` functions for specific needs
|
- **Flexible conversion**: Customize `to_transition` and `to_output` functions for specific needs
|
||||||
|
|
||||||
Key advantages of the dual format approach:
|
Key advantages of the dual format approach:
|
||||||
|
|
||||||
- **Environment interaction**: Use tuple format for real-time robot control
|
- **Environment interaction**: Use tuple format for real-time robot control
|
||||||
- **Training/evaluation**: Use batch format for dataset processing and model training
|
- **Training/evaluation**: Use batch format for dataset processing and model training
|
||||||
- **Seamless integration**: Same processor works with both formats automatically
|
- **Seamless integration**: Same processor works with both formats automatically
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, Mapping
|
from typing import Any
|
||||||
|
from collections.abc import Mapping
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ import os
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable, Iterable, Protocol, Sequence
|
from typing import Any, Protocol
|
||||||
|
from collections.abc import Callable, Iterable, Sequence
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from huggingface_hub import ModelHubMixin, hf_hub_download
|
from huggingface_hub import ModelHubMixin, hf_hub_download
|
||||||
@@ -502,10 +503,10 @@ class RobotProcessor(ModelHubMixin):
|
|||||||
Loading with overrides for non-serializable objects:
|
Loading with overrides for non-serializable objects:
|
||||||
```python
|
```python
|
||||||
import gym
|
import gym
|
||||||
|
|
||||||
env = gym.make("CartPole-v1")
|
env = gym.make("CartPole-v1")
|
||||||
processor = RobotProcessor.from_pretrained(
|
processor = RobotProcessor.from_pretrained(
|
||||||
"username/cartpole-processor",
|
"username/cartpole-processor", overrides={"ActionRepeatStep": {"env": env}}
|
||||||
overrides={"ActionRepeatStep": {"env": env}}
|
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -515,8 +516,8 @@ class RobotProcessor(ModelHubMixin):
|
|||||||
"path/to/processor",
|
"path/to/processor",
|
||||||
overrides={
|
overrides={
|
||||||
"CustomStep": {"param1": "new_value"},
|
"CustomStep": {"param1": "new_value"},
|
||||||
"device_processor": {"device": "cuda:1"} # For registered steps
|
"device_processor": {"device": "cuda:1"}, # For registered steps
|
||||||
}
|
},
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
@@ -913,7 +914,7 @@ class InfoProcessor:
|
|||||||
|
|
||||||
def info(self, info):
|
def info(self, info):
|
||||||
info = info.copy() # Create a copy to avoid modifying the original
|
info = info.copy() # Create a copy to avoid modifying the original
|
||||||
info['steps'] = self.step_count
|
info["steps"] = self.step_count
|
||||||
self.step_count += 1
|
self.step_count += 1
|
||||||
return info
|
return info
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
---
|
---
|
||||||
library_name: lerobot
|
library_name: lerobot
|
||||||
tags:
|
tags:
|
||||||
- robotics
|
- robotics
|
||||||
- lerobot
|
- lerobot
|
||||||
- safetensors
|
- safetensors
|
||||||
pipeline_tag: robotics
|
pipeline_tag: robotics
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -20,6 +20,7 @@ The RobotProcessor provides a modular architecture for processing robot environm
|
|||||||
### EnvTransition Structure
|
### EnvTransition Structure
|
||||||
|
|
||||||
An `EnvTransition` is a 7-tuple containing:
|
An `EnvTransition` is a 7-tuple containing:
|
||||||
|
|
||||||
1. **observation**: Current state observation
|
1. **observation**: Current state observation
|
||||||
2. **action**: Action taken (can be None)
|
2. **action**: Action taken (can be None)
|
||||||
3. **reward**: Reward received (float or None)
|
3. **reward**: Reward received (float or None)
|
||||||
|
|||||||
@@ -902,17 +902,17 @@ class MockStepWithNonSerializableParam:
|
|||||||
|
|
||||||
return (obs, action, reward, done, truncated, info, comp_data)
|
return (obs, action, reward, done, truncated, info, comp_data)
|
||||||
|
|
||||||
def get_config(self) -> Dict[str, Any]:
|
def get_config(self) -> dict[str, Any]:
|
||||||
# Note: env is intentionally NOT included here as it's not serializable
|
# Note: env is intentionally NOT included here as it's not serializable
|
||||||
return {
|
return {
|
||||||
"name": self.name,
|
"name": self.name,
|
||||||
"multiplier": self.multiplier,
|
"multiplier": self.multiplier,
|
||||||
}
|
}
|
||||||
|
|
||||||
def state_dict(self) -> Dict[str, torch.Tensor]:
|
def state_dict(self) -> dict[str, torch.Tensor]:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def load_state_dict(self, state: Dict[str, torch.Tensor]) -> None:
|
def load_state_dict(self, state: dict[str, torch.Tensor]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def reset(self) -> None:
|
def reset(self) -> None:
|
||||||
@@ -936,16 +936,16 @@ class RegisteredMockStep:
|
|||||||
|
|
||||||
return (obs, action, reward, done, truncated, info, comp_data)
|
return (obs, action, reward, done, truncated, info, comp_data)
|
||||||
|
|
||||||
def get_config(self) -> Dict[str, Any]:
|
def get_config(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"value": self.value,
|
"value": self.value,
|
||||||
"device": self.device,
|
"device": self.device,
|
||||||
}
|
}
|
||||||
|
|
||||||
def state_dict(self) -> Dict[str, torch.Tensor]:
|
def state_dict(self) -> dict[str, torch.Tensor]:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def load_state_dict(self, state: Dict[str, torch.Tensor]) -> None:
|
def load_state_dict(self, state: dict[str, torch.Tensor]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def reset(self) -> None:
|
def reset(self) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user