Merge branch 'main' into codex/language-recipes

test(collate): expect preserved language columns
feat(data): add recipe-driven language supervision
2026-07-30 21:19:40 +00:00 · 2026-07-28 12:50:11 +02:00 · 2026-07-28 12:39:15 +02:00 · 2026-07-28 10:04:28 +02:00
206 changed files with 1364 additions and 2134 deletions
@@ -61,20 +61,16 @@ Full details in [`docs/source/so101.mdx`](./docs/source/so101.mdx) and [`docs/so
 **4.1 Install**

 ```bash
-# uv (recommended — see AGENTS.md and CLAUDE.md)
-uv sync --locked --extra feetech          # SO-100/SO-101 motor stack
-# uv sync --locked --extra all            # everything
-# uv sync --locked --extra smolvla        # add SmolVLA deps
-
-# pip (alternative, e.g. when not working from source)
-# pip install 'lerobot[feetech]'
-# pip install 'lerobot[all]'
-# pip install 'lerobot[smolvla]'
-
+pip install 'lerobot[feetech]'              # SO-100/SO-101 motor stack
+# pip install 'lerobot[all]'                # everything
+# pip install 'lerobot[aloha,pusht]'        # specific features
+# pip install 'lerobot[smolvla]'            # add SmolVLA deps
 git lfs install && git lfs pull
-hf auth login                             # required to push datasets/policies
+hf auth login                               # required to push datasets/policies
 ```

+Contributors can alternatively use `uv sync --locked --extra feetech` (see `AGENTS.md`).
+
 **4.2 Find USB ports** — run once per arm, unplug when prompted.

 ```bash
@@ -68,16 +68,17 @@ ENV HOME=/home/user_lerobot \
 # issues with MuJoCo and OpenGL drivers.
 RUN uv venv --python python${PYTHON_VERSION}

-# Install third-party dependencies separately for layer caching
+# Install Python dependencies for caching
 COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
-RUN uv sync --locked --extra all --no-install-project --no-cache
+COPY --chown=user_lerobot:user_lerobot src/ src/
+
+RUN uv sync --locked --extra all --no-cache

 RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas

-# Copy the application source code and install the local project
+# Copy the rest of the application source code
 # Make sure to have the git-LFS files for testing
 COPY --chown=user_lerobot:user_lerobot . .
-RUN uv sync --locked --extra all --no-cache

 # Set the default command
 CMD ["/bin/bash"]
@@ -60,14 +60,15 @@ ENV HOME=/home/user_lerobot \
 # run other Python projects in the same container without dependency conflicts.
 RUN uv venv

-# Install third-party dependencies separately for layer caching
+# Install Python dependencies for caching
 COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml uv.lock README.md MANIFEST.in ./
-RUN uv sync --locked --extra all --no-install-project --no-cache
+COPY --chown=user_lerobot:user_lerobot src/ src/

-# Copy the application code and install the local project
+RUN uv sync --locked --extra all --no-cache
+
+# Copy the rest of the application code
 # Make sure to have the git-LFS files for testing
 COPY --chown=user_lerobot:user_lerobot . .
-RUN uv sync --locked --extra all --no-cache

 # Set the default command
 CMD ["/bin/bash"]
@@ -58,7 +58,7 @@ final_action = postprocessor(action)

 ## Hardware API redesign

-PR [#777](https://github.com/huggingface/lerobot/pull/777) improves the LeRobot calibration but is **not backward-compatible**. Below is an overview of what changed and how you can continue to work with datasets created before this pull request.
+PR [#777](https://github.com/huggingface/lerobot/pull/777) improves the LeRobot calibration but is **not backward-compatible**. Below is a overview of what changed and how you can continue to work with datasets created before this pull request.

 ### What changed?

@@ -129,8 +129,8 @@ python examples/backward_compatibility/replay.py \

 Policies output actions in the same format as the datasets (`torch.Tensors`). Therefore, the same transformations should be applied.

-To find these transformations, we recommend first replaying an episode of the dataset your policy was trained on using the section above.
-Then, add these same transformations to your inference script (shown here in the `record.py` script):
+To find these transformations, we recommend to first try and and replay an episode of the dataset your policy was trained on using the section above.
+Then, add these same transformations on your inference script (shown here in the `record.py` script):

 ```diff
 action_values = predict_action(
@@ -136,10 +136,6 @@ config = RealSenseCameraConfig(
    height=480,
    color_mode=ColorMode.RGB,
    use_depth=True,
-    # Optional fixed color controls. Omit them to leave the current sensor settings unchanged.
-    exposure=120,
-    gain=64,
-    white_balance=4600,
    rotation=Cv2Rotation.NO_ROTATION
 )

@@ -158,15 +154,6 @@ finally:
 ```
 <!-- prettier-ignore-end -->

-Manual color controls disable the corresponding automatic exposure or white-balance mode. Their
-supported ranges vary by camera model; an invalid value raises an error at connection time that
-includes the range reported by the sensor. Requesting an unsupported control also raises an error.
-Omitted controls leave the sensor's existing automatic or manual setting unchanged. These options
-require `use_rgb=True`.
-
-On the RealSense D405, the color stream is provided by the Stereo Module, so changing manual
-exposure or gain also affects the depth stream.
-
 </hfoption>
 </hfoptions>

@@ -88,6 +88,20 @@ policy_preprocessor = NormalizerProcessorStep(stats=dataset_stats)

 The same policy can work with different environment processors, and the same environment processor can work with different policies:

+````python
+# Use SmolVLA policy with LIBERO environment
+# Use SmolVLA policy with LIBERO environment
+libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(
+    env_cfg=libero_cfg,
+    policy_cfg=smolvla_cfg,
+)
+smolvla_preprocessor, smolvla_postprocessor = make_pre_post_processors(smolvla_cfg)
+# Or use ACT policy with the same LIBERO environment
+libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(
+    env_cfg=libero_cfg,
+    policy_cfg=act_cfg,
+)
+act_preprocessor, act_postprocessor = make_pre_post_processors(act_cfg)
 ```python
 # Use SmolVLA policy with LIBERO environment
 libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(
@@ -102,7 +116,6 @@ libero_preprocessor, libero_postprocessor = make_env_pre_post_processors(
    policy_cfg=act_cfg,
 )
 act_preprocessor, act_postprocessor = make_pre_post_processors(act_cfg)
-```

 ### 3. **Easier Experimentation**

@@ -132,7 +145,7 @@ class LiberoVelocityProcessorStep(ObservationProcessorStep):
        state = torch.cat([eef_pos, eef_axisangle, eef_vel,
                          gripper_pos, gripper_vel], dim=-1)  # 14D
        return state
-```
+````

 ### 4. **Cleaner Environment Code**

@@ -40,10 +40,10 @@ This tutorial guides you through updating the firmware of Feetech motors using t
 For each motor you want to update:

 1. **Select the motor** from the list by clicking on it
-2. **Click the Upgrade tab**:
-3. **Click the Online button**:
-   - If a potential firmware update is found, it will be displayed in the box
-4. **Click the Upgrade button**:
+2. **Click on Upgrade tab**:
+3. **Click on Online button**:
+   - If an potential firmware update is found, it will be displayed in the box
+4. **Click on Upgrade button**:
   - The update progress will be displayed

 ## Step 6: Verify Update
@@ -59,7 +59,6 @@ The `lerobot-rollout --strategy.type=dagger` mode requires **teleoperators with

 - `bi_openarm_mini` - Bimanual OpenArm Mini
 - `so_leader` - SO100 / SO101 leader arm
- `bi_so_leader` - Bimanual SO100 / SO101 leader arms

 > [!IMPORTANT]
 > The provided commands default to `bi_openarm_follower` + `bi_openarm_mini`.
@@ -211,7 +211,7 @@ Record, Replay and Train with Hope-JR is still experimental.

 ### Record

-This step records the dataset, which can be seen as an example [here](https://huggingface.co/datasets/nepyope/hand_record_test_with_video_data).
+This step records the dataset, which can be seen as an example [here](https://huggingface.co/datasets/nepyope/hand_record_test_with_video_data/settings).

 ```bash
 lerobot-record \
@@ -18,7 +18,7 @@ If you're using Feetech or Dynamixel motors, LeRobot provides built-in bus inter
 - [`DynamixelMotorsBus`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/motors/dynamixel/dynamixel.py) – for controlling Dynamixel servos

 Please refer to the [`MotorsBus`](https://github.com/huggingface/lerobot/blob/main/src/lerobot/motors/motors_bus.py) abstract class to learn about its API.
-For a good example of how it can be used, you can have a look at our own [SO101 follower implementation](https://github.com/huggingface/lerobot/blob/main/src/lerobot/robots/so_follower/so_follower.py)
+For a good example of how it can be used, you can have a look at our own [SO101 follower implementation](https://github.com/huggingface/lerobot/blob/main/src/lerobot/robots/so_follower/so101_follower/so101_follower.py)

 Use these if compatible. Otherwise, you'll need to find or write a Python interface (not covered in this tutorial):

@@ -141,6 +141,17 @@ sample["target_message_indices"]

 The renderer does not apply a tokenizer chat template. Policy processors decide how to serialize the messages for their backbone, which keeps the same dataset usable across SmolVLA, Pi0.5, and any future VLM that expects OpenAI-style chat messages.

+## Blends
+
+Blend recipes select one weighted sub-recipe deterministically from the sample index.
+`recipes/subtask_mem.yaml` trains the compact core blend — high-level subtask prediction, low-level execution, and memory. `recipes/subtask_mem_vqa_speech.yaml` is the fuller variant that also adds VQA and spoken interjection responses.
+
+A message recipe with a supervised assistant turn on the `low_level` stream trains
+the π0.5 paper's joint sequence instead of a blend: the target span gets text CE
+while also conditioning the action losses in the same forward.
+`recipes/subtask_joint.yaml` is the provided example; pair it with
+`--policy.joint_subtask_conditioning=true` at inference.
+
 ## Graceful absence

 If both language columns are missing, `None`, or empty, `RenderMessagesStep` is a no-op.
@@ -51,7 +51,7 @@ In addition to these instructions, you need to install the Feetech SDK & ZeroMQ
 pip install -e ".[lekiwi]"
 ```

-Great 🤗! You are now done installing LeRobot, and we can begin assembling the SO100/SO101 arms and the mobile base 🤖.
+Great :hugs:! You are now done installing LeRobot, and we can begin assembling the SO100/SO101 arms and the mobile base :robot:.
 Every time you now want to use LeRobot, you can go to the `~/lerobot` folder where we installed LeRobot and run one of the commands.

 # Step-by-Step Assembly Instructions
@@ -174,7 +174,7 @@ The model takes images, text instructions, and robot state as input, and outputs

 ## Reproducing π₀Fast results

-We reproduce the results of π₀Fast on the LIBERO benchmark using the LeRobot implementation. We take the LeRobot PiFast base model [lerobot/pi0fast-base](https://huggingface.co/lerobot/pi0fast-base) and finetune for an additional 40k steps in bfloat16, with batch size of 256 on 8 H100 GPUs using the [HuggingFace LIBERO dataset](https://huggingface.co/datasets/HuggingFaceVLA/libero).
+We reproduce the results of π₀Fast on the LIBERO benchmark using the LeRobot implementation. We take the LeRobot PiFast base model [lerobot/pi0fast-base](https://huggingface.co/lerobot/pi0fast-base) and finetune for an additional 40kk steps in bfloat16, with batch size of 256 on 8 H100 GPUs using the [HuggingFace LIBERO dataset](https://huggingface.co/datasets/HuggingFaceVLA/libero).

 The finetuned model can be found here:

@@ -22,7 +22,7 @@ With processors, you choose the learning features you want to use for your polic
 ## Three pipelines

 We often compose three pipelines. Depending on your setup, some can be empty if action and observation spaces already match.
-Each of these pipelines handles different conversions between different action and observation spaces. Below is a quick explanation of each pipeline.
+Each of these pipelines handle different conversions between different action and observation spaces. Below is a quick explanation of each pipeline.

 1. Pipeline 1: Teleop action space → dataset action space (phone pose → EE targets)
 2. Pipeline 2: Dataset action space → robot command space (EE targets → joints)
@@ -74,15 +74,15 @@ In the phone to SO-100 follower examples we use the following adapters:
 - `robot_action_to_transition`: transforms the teleop action dict to a pipeline transition.
 - `transition_to_robot_action`: transforms the pipeline transition to a robot action dict.
 - `observation_to_transition`: transforms the robot observation dict to a pipeline transition.
- `transition_to_observation`: transforms the pipeline transition to an observation dict.
+- `transition_to_observation`: transforms the pipeline transition to a observation dict.

-Check out [src/lerobot/processor/converters.py](https://github.com/huggingface/lerobot/blob/main/src/lerobot/processor/converters.py) for more details.
+Checkout [src/lerobot/processor/converters.py](https://github.com/huggingface/lerobot/blob/main/src/lerobot/processor/converters.py) for more details.

 ## Dataset feature contracts

 Dataset features are determined by the keys saved in the dataset. Each step can declare what features it modifies in a contract called `transform_features(...)`. Once you build a processor, the processor can then aggregate all of these features with `aggregate_pipeline_dataset_features()` and merge multiple feature dicts with `combine_feature_dicts(...)`.

-Below is an example of how we declare features with the `transform_features` method in the phone to SO-100 follower examples:
+Below is and example of how we declare features with the `transform_features` method in the phone to SO-100 follower examples:

 ```python
    def transform_features(
@@ -82,8 +82,6 @@ By default the env samples objects only from the `lightwheel` registry (what `--

 All eval snippets below mirror the CI command (see `.github/workflows/benchmark_tests.yml`). The `--rename_map` argument maps RoboCasa's native camera keys (`robot0_agentview_left` / `robot0_eye_in_hand` / `robot0_agentview_right`) onto the three-camera (`camera1` / `camera2` / `camera3`) input layout the released `smolvla_robocasa` policy was trained on.

-By default, each task uses the rollout horizon registered by RoboCasa. Set `--env.episode_length=<steps>` to apply the same explicit horizon to every selected task.
-
 ### Single-task evaluation (recommended for quick iteration)

 ```bash
@@ -57,7 +57,7 @@ policy_cfg.rtc_config = RTCConfig(
 policy = PI0Policy.from_pretrained("lerobot/pi0_base", policy_cfg=policy_cfg, device="cuda")

 # Now use predict_action_chunk with RTC parameters
-inference_delay = 4  # How many steps of inference latency, this value should be calculated based on the inference latency of the policy
+inference_delay = 4  # How many steps of inference latency, this values should be calculated based on the inference latency of the policy

 # Initialize the action queue
 action_queue = ActionQueue(policy_cfg.rtc_config)
@@ -100,7 +100,7 @@ Typical values: 8-12 steps
 RTCConfig(execution_horizon=10)
 ```

-**`max_guidance_weight`**: How strongly to enforce consistency with the previous chunk. This is a hyperparameter that can be tuned to balance the smoothness of the transitions and the reactivity of the policy. For 10 steps flow matching (SmolVLA, Pi0, Pi0.5), a value of 10.0 is an optimal value.
+**`max_guidance_weight`**: How strongly to enforce consistency with the previous chunk. This is a hyperparameter that can be tuned to balance the smoothness of the transitions and the reactivity of the policy. For 10 steps flow matching (SmolVLA, Pi0, Pi0.5), a value of 10.0 is a optimal value.

 **`prefix_attention_schedule`**: How to weight consistency across the overlap region.

@@ -93,7 +93,7 @@ lerobot-train --help

 ## Evaluate the finetuned model and run it in real-time

-Similarly for when recording an episode, it is recommended that you are logged in to the HuggingFace Hub. You can follow the corresponding steps: [Record a dataset](./il_robots#record-a-dataset).
+Similarly for when recording an episode, it is recommended that you are logged in to the HuggingFace Hub. You can follow the corresponding steps: [Record a dataset](./il_robots).
 Once you are logged in, you can run inference in your setup by doing:

 ```bash
@@ -338,7 +338,7 @@ It is advisable to install one 3-pin cable in the motor after placing them befor
 <hfoption id="Leader">

 - Mount the leader holder onto the wrist and secure it with 4 M3x6mm screws.
- Attach the handle to the leader holder using 1 M2x6mm screw.
+- Attach the handle to motor 5 using 1 M2x6mm screw.
 - Insert the gripper motor, secure it with 2 M2x6mm screws on each side, attach a motor horn using a M3x6mm horn screw.
 - Attach the follower trigger with 4 M3x6mm screws.

@@ -50,11 +50,11 @@ lerobot-edit-dataset \
 Divide a dataset into multiple subsets.

 ```bash
-# Split by fractions (e.g. 60% train, 20% val, 20% test)
+# Split by fractions (e.g. 80% train, 20% test, 20% val)
 lerobot-edit-dataset \
    --repo_id lerobot/pusht \
    --operation.type split \
-    --operation.splits '{"train": 0.6, "val": 0.2, "test": 0.2}'
+    --operation.splits '{"train": 0.8, "test": 0.2, "val": 0.2}'

 # Split by specific episode indices
 lerobot-edit-dataset \
@@ -44,7 +44,6 @@ from typing import Protocol

 import numpy as np

-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -57,6 +56,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
    EEBoundsAndSafety,
    InverseKinematicsEEToJoints,
 )
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.constants import HF_LEROBOT_CALIBRATION, HF_LEROBOT_HOME, TELEOPERATORS
 from lerobot.utils.robot_utils import precise_sleep

@@ -38,7 +38,7 @@ from typing import TYPE_CHECKING

 import numpy as np

-from lerobot.lerobot_types import RobotAction
+from lerobot.types import RobotAction

 from .base import _GRIPPER_MOTOR_SCALE, IsaacTeleopTeleoperator, _isaacteleop_available
 from .config_isaac_teleop import SO101LeaderArmConfig
@@ -32,7 +32,7 @@ from typing import TYPE_CHECKING, Any

 import numpy as np

-from lerobot.lerobot_types import RobotAction
+from lerobot.types import RobotAction

 from .base import IsaacTeleopTeleoperator, _isaacteleop_available
 from .config_isaac_teleop import XRControllerConfig
@@ -26,8 +26,8 @@ from __future__ import annotations
 from dataclasses import dataclass

 from lerobot.configs.types import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import RobotAction
 from lerobot.processor import ProcessorStepRegistry, RobotActionProcessorStep
+from lerobot.types import RobotAction
 from lerobot.utils.rotation import Rotation

 from .base import _GRIPPER_MOTOR_SCALE
@@ -21,7 +21,6 @@ from lerobot.cameras.opencv import OpenCVCameraConfig
 from lerobot.common.control_utils import predict_action
 from lerobot.configs import FeatureType, PolicyFeature
 from lerobot.datasets import LeRobotDataset, aggregate_pipeline_dataset_features, create_initial_features
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.policies import make_pre_post_processors
 from lerobot.policies.act import ACTPolicy
@@ -39,6 +38,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
    ForwardKinematicsJointsToEE,
    InverseKinematicsEEToJoints,
 )
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION, OBS_STR
 from lerobot.utils.feature_utils import build_dataset_frame, combine_feature_dicts
 from lerobot.utils.keyboard_input import init_keyboard_listener
@@ -16,7 +16,6 @@

 from lerobot.cameras.opencv import OpenCVCameraConfig
 from lerobot.datasets import LeRobotDataset, aggregate_pipeline_dataset_features, create_initial_features
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -37,6 +36,7 @@ from lerobot.scripts.lerobot_record import record_loop
 from lerobot.teleoperators.phone import Phone, PhoneConfig
 from lerobot.teleoperators.phone.config_phone import PhoneOS
 from lerobot.teleoperators.phone.phone_processor import MapPhoneActionToRobotAction
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.feature_utils import combine_feature_dicts
 from lerobot.utils.keyboard_input import init_keyboard_listener
 from lerobot.utils.utils import log_say
@@ -17,7 +17,6 @@
 import time

 from lerobot.datasets import LeRobotDataset
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -28,6 +27,7 @@ from lerobot.robots.so_follower import SO100Follower, SO100FollowerConfig
 from lerobot.robots.so_follower.robot_kinematic_processor import (
    InverseKinematicsEEToJoints,
 )
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION
 from lerobot.utils.robot_utils import precise_sleep
 from lerobot.utils.utils import log_say
@@ -27,7 +27,6 @@ Highlight, or DAgger via ``lerobot-rollout --strategy.type=...``.

 from lerobot.cameras.opencv import OpenCVCameraConfig
 from lerobot.configs import PreTrainedConfig
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -44,6 +43,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
 from lerobot.rollout import BaseStrategyConfig, RolloutConfig, build_rollout_context
 from lerobot.rollout.inference import SyncInferenceConfig
 from lerobot.rollout.strategies import BaseStrategy
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.process import ProcessSignalHandler
 from lerobot.utils.utils import init_logging

@@ -15,7 +15,6 @@

 import time

-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -32,6 +31,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
 from lerobot.teleoperators.phone import Phone, PhoneConfig
 from lerobot.teleoperators.phone.config_phone import PhoneOS
 from lerobot.teleoperators.phone.phone_processor import MapPhoneActionToRobotAction
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.robot_utils import precise_sleep
 from lerobot.utils.visualization_utils import init_rerun, log_rerun_data

@@ -21,7 +21,6 @@ from lerobot.cameras.opencv import OpenCVCameraConfig
 from lerobot.common.control_utils import predict_action
 from lerobot.configs import FeatureType, PolicyFeature
 from lerobot.datasets import LeRobotDataset, aggregate_pipeline_dataset_features, create_initial_features
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.policies import make_pre_post_processors
 from lerobot.policies.act import ACTPolicy
@@ -39,6 +38,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
    ForwardKinematicsJointsToEE,
    InverseKinematicsEEToJoints,
 )
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION, OBS_STR
 from lerobot.utils.feature_utils import build_dataset_frame, combine_feature_dicts
 from lerobot.utils.keyboard_input import init_keyboard_listener
@@ -17,7 +17,6 @@

 from lerobot.cameras.opencv import OpenCVCameraConfig
 from lerobot.datasets import LeRobotDataset, aggregate_pipeline_dataset_features, create_initial_features
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -34,6 +33,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
 )
 from lerobot.scripts.lerobot_record import record_loop
 from lerobot.teleoperators.so_leader import SO100Leader, SO100LeaderConfig
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.feature_utils import combine_feature_dicts
 from lerobot.utils.keyboard_input import init_keyboard_listener
 from lerobot.utils.utils import log_say
@@ -18,7 +18,6 @@
 import time

 from lerobot.datasets import LeRobotDataset
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -29,6 +28,7 @@ from lerobot.robots.so_follower import SO100Follower, SO100FollowerConfig
 from lerobot.robots.so_follower.robot_kinematic_processor import (
    InverseKinematicsEEToJoints,
 )
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION
 from lerobot.utils.robot_utils import precise_sleep
 from lerobot.utils.utils import log_say
@@ -25,7 +25,6 @@ forward/inverse kinematics.

 from lerobot.cameras.opencv import OpenCVCameraConfig
 from lerobot.configs import PreTrainedConfig
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -42,6 +41,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
 from lerobot.rollout import BaseStrategyConfig, RolloutConfig, build_rollout_context
 from lerobot.rollout.inference import SyncInferenceConfig
 from lerobot.rollout.strategies import BaseStrategy
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.process import ProcessSignalHandler
 from lerobot.utils.utils import init_logging

@@ -16,7 +16,6 @@

 import time

-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.model.kinematics import RobotKinematics
 from lerobot.processor import (
    RobotProcessorPipeline,
@@ -31,6 +30,7 @@ from lerobot.robots.so_follower.robot_kinematic_processor import (
    InverseKinematicsEEToJoints,
 )
 from lerobot.teleoperators.so_leader import SO100Leader, SO100LeaderConfig
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.robot_utils import precise_sleep
 from lerobot.utils.visualization_utils import init_rerun, log_rerun_data

@@ -67,7 +67,7 @@ dependencies = [
    "einops>=0.8.0,<0.9.0",

    # Config & Hub
-    "draccus>=0.11.6,<0.12.0",
+    "draccus==0.10.0", # TODO: Relax version constraint
    "huggingface-hub>=1.0.0,<2.0.0",
    "requests>=2.32.0,<3.0.0",

@@ -494,19 +494,6 @@ ignore_errors = true
 module = "lerobot.envs.*"
 ignore_errors = false

-[[tool.mypy.overrides]]
-module = "lerobot.annotations.*"
-ignore_errors = false
-disallow_untyped_defs = true
-disallow_incomplete_defs = true
-check_untyped_defs = true
-
-[[tool.mypy.overrides]]
-module = "lerobot.transforms.*"
-ignore_errors = false
-disallow_untyped_defs = true
-disallow_incomplete_defs = true
-check_untyped_defs = true

 # [[tool.mypy.overrides]]
 # module = "lerobot.utils.*"
@@ -38,7 +38,6 @@ import draccus
 import grpc
 import torch

-from lerobot.lerobot_types import PolicyAction
 from lerobot.policies import get_policy_class, make_pre_post_processors
 from lerobot.processor import PolicyProcessorPipeline
 from lerobot.transport import (
@@ -46,6 +45,7 @@ from lerobot.transport import (
    services_pb2_grpc,  # type: ignore
 )
 from lerobot.transport.utils import receive_bytes_in_chunks
+from lerobot.types import PolicyAction

 from .configs import PolicyServerConfig
 from .constants import SUPPORTED_POLICIES
@@ -120,22 +120,14 @@ class OpenCVCamera(Camera):
        self.rotation: int | None = get_cv2_rotation(config.rotation)
        self.backend: int = config.backend

-        self.capture_width: int | None = None
-        self.capture_height: int | None = None
-        self._reset_connection_settings()
+        if self.height and self.width:
+            self.capture_width, self.capture_height = self.width, self.height
+            if self.rotation in [cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE]:
+                self.capture_width, self.capture_height = self.height, self.width

    def __str__(self) -> str:
        return f"{self.__class__.__name__}({self.index_or_path})"

-    def _reset_connection_settings(self) -> None:
-        """Restore settings that may have been auto-detected during a failed connection."""
-        self.fps = self.config.fps
-        self.width = self.config.width
-        self.height = self.config.height
-        self.capture_width, self.capture_height = self.width, self.height
-        if self.rotation in [cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE]:
-            self.capture_width, self.capture_height = self.height, self.width
-
    @property
    def is_connected(self) -> bool:
        """Checks if the camera is currently connected and opened."""
@@ -172,25 +164,17 @@ class OpenCVCamera(Camera):
                f"Failed to open {self}.Run `lerobot-find-cameras opencv` to find available cameras."
            )

-        try:
-            self._configure_capture_settings()
-            self._start_read_thread()
+        self._configure_capture_settings()
+        self._start_read_thread()

-            if warmup and self.warmup_s > 0:
-                start_time = time.time()
-                while time.time() - start_time < self.warmup_s:
-                    self.async_read(timeout_ms=self.warmup_s * 1000)
-                    time.sleep(0.1)
-                with self.frame_lock:
-                    if self.latest_frame is None:
-                        raise ConnectionError(f"{self} failed to capture frames during warmup.")
-        except BaseException:
-            try:
-                self._cleanup_resources()
-            except Exception:
-                logger.exception(f"Failed to fully clean up {self} after connect() failed.")
-            self._reset_connection_settings()
-            raise
+        if warmup and self.warmup_s > 0:
+            start_time = time.time()
+            while time.time() - start_time < self.warmup_s:
+                self.async_read(timeout_ms=self.warmup_s * 1000)
+                time.sleep(0.1)
+            with self.frame_lock:
+                if self.latest_frame is None:
+                    raise ConnectionError(f"{self} failed to capture frames during warmup.")

        logger.info(f"{self} connected.")

@@ -328,36 +312,32 @@ class OpenCVCamera(Camera):

        for target in targets_to_scan:
            camera = cv2.VideoCapture(target)
-            try:
-                if camera.isOpened():
-                    default_width = int(camera.get(cv2.CAP_PROP_FRAME_WIDTH))
-                    default_height = int(camera.get(cv2.CAP_PROP_FRAME_HEIGHT))
-                    default_fps = camera.get(cv2.CAP_PROP_FPS)
-                    default_format = camera.get(cv2.CAP_PROP_FORMAT)
+            if camera.isOpened():
+                default_width = int(camera.get(cv2.CAP_PROP_FRAME_WIDTH))
+                default_height = int(camera.get(cv2.CAP_PROP_FRAME_HEIGHT))
+                default_fps = camera.get(cv2.CAP_PROP_FPS)
+                default_format = camera.get(cv2.CAP_PROP_FORMAT)

-                    # Get FOURCC code and convert to string
-                    default_fourcc_code = camera.get(cv2.CAP_PROP_FOURCC)
-                    default_fourcc_code_int = int(default_fourcc_code)
-                    default_fourcc = "".join(
-                        [chr((default_fourcc_code_int >> 8 * i) & 0xFF) for i in range(4)]
-                    )
+                # Get FOURCC code and convert to string
+                default_fourcc_code = camera.get(cv2.CAP_PROP_FOURCC)
+                default_fourcc_code_int = int(default_fourcc_code)
+                default_fourcc = "".join([chr((default_fourcc_code_int >> 8 * i) & 0xFF) for i in range(4)])

-                    camera_info = {
-                        "name": f"OpenCV Camera @ {target}",
-                        "type": "OpenCV",
-                        "id": target,
-                        "backend_api": camera.getBackendName(),
-                        "default_stream_profile": {
-                            "format": default_format,
-                            "fourcc": default_fourcc,
-                            "width": default_width,
-                            "height": default_height,
-                            "fps": default_fps,
-                        },
-                    }
+                camera_info = {
+                    "name": f"OpenCV Camera @ {target}",
+                    "type": "OpenCV",
+                    "id": target,
+                    "backend_api": camera.getBackendName(),
+                    "default_stream_profile": {
+                        "format": default_format,
+                        "fourcc": default_fourcc,
+                        "width": default_width,
+                        "height": default_height,
+                        "fps": default_fps,
+                    },
+                }

-                    found_cameras_info.append(camera_info)
-            finally:
+                found_cameras_info.append(camera_info)
                camera.release()

        return found_cameras_info
@@ -516,26 +496,6 @@ class OpenCVCamera(Camera):
            self.latest_timestamp = None
            self.new_frame_event.clear()

-    def _cleanup_resources(self) -> None:
-        """Stop background reads and release the capture, including after partial setup."""
-        read_thread = self.thread
-        videocapture = self.videocapture
-
-        try:
-            self._stop_read_thread()
-        finally:
-            self.videocapture = None
-            try:
-                if videocapture is not None:
-                    videocapture.release()
-            finally:
-                # Releasing the device may unblock a hardware read that outlived
-                # the first bounded join in _stop_read_thread().
-                if read_thread is not None and read_thread.is_alive():
-                    read_thread.join(timeout=2.0)
-                    if read_thread.is_alive():  # pragma: no cover
-                        logger.warning(f"{self} read thread remained alive after releasing the capture.")
-
    @check_if_not_connected
    def async_read(self, timeout_ms: float = 200) -> NDArray[Any]:
        """
@@ -626,6 +586,16 @@ class OpenCVCamera(Camera):
        if not self.is_connected and self.thread is None:
            raise DeviceNotConnectedError(f"{self} not connected.")

-        self._cleanup_resources()
+        if self.thread is not None:
+            self._stop_read_thread()
+
+        if self.videocapture is not None:
+            self.videocapture.release()
+            self.videocapture = None
+
+        with self.frame_lock:
+            self.latest_frame = None
+            self.latest_timestamp = None
+            self.new_frame_event.clear()

        logger.info(f"{self} disconnected.")
@@ -121,9 +121,6 @@ class RealSenseCamera(Camera):

        self.config = config

-        self.width: int | None = config.width
-        self.height: int | None = config.height
-
        if config.serial_number_or_name.isdigit():
            self.serial_number = config.serial_number_or_name
        else:
@@ -134,9 +131,6 @@ class RealSenseCamera(Camera):
        self.use_rgb = config.use_rgb
        self.use_depth = config.use_depth
        self.warmup_s = config.warmup_s
-        self.exposure: int | None = config.exposure
-        self.gain: int | None = config.gain
-        self.white_balance: int | None = config.white_balance

        self.rs_pipeline: rs.pipeline | None = None
        self.rs_profile: rs.pipeline_profile | None = None
@@ -151,23 +145,14 @@ class RealSenseCamera(Camera):

        self.rotation: int | None = get_cv2_rotation(config.rotation)

-        self.capture_width: int | None = None
-        self.capture_height: int | None = None
-        self._reset_connection_settings()
+        if self.height and self.width:
+            self.capture_width, self.capture_height = self.width, self.height
+            if self.rotation in [cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE]:
+                self.capture_width, self.capture_height = self.height, self.width

    def __str__(self) -> str:
        return f"{self.__class__.__name__}({self.serial_number})"

-    def _reset_connection_settings(self) -> None:
-        """Restore settings that may have been auto-detected during a failed connection."""
-        self.fps = self.config.fps
-        self.width = self.config.width
-        self.height = self.config.height
-        self.warmup_s = self.config.warmup_s
-        self.capture_width, self.capture_height = self.width, self.height
-        if self.rotation in [cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE]:
-            self.capture_width, self.capture_height = self.height, self.width
-
    @property
    def is_connected(self) -> bool:
        """Checks if the camera pipeline is started and streams are active."""
@@ -187,8 +172,7 @@ class RealSenseCamera(Camera):

        Raises:
            DeviceAlreadyConnectedError: If the camera is already connected.
-            ValueError: If the configuration is invalid, a requested sensor option is unsupported,
-                or a requested sensor value is invalid.
+            ValueError: If the configuration is invalid (e.g., missing serial/name, name not unique).
            ConnectionError: If the camera is found but fails to start the pipeline or no RealSense devices are detected at all.
            RuntimeError: If the pipeline starts but fails to apply requested settings.
        """
@@ -206,31 +190,22 @@ class RealSenseCamera(Camera):
                f"Failed to open {self}.Run `lerobot-find-cameras realsense` to find available cameras."
            ) from e

-        try:
-            self._configure_capture_settings()
-            self._configure_sensor_options()
-            self._start_read_thread()
+        self._configure_capture_settings()
+        self._start_read_thread()

-            # NOTE(Steven/Caroline): Enforcing at least one second of warmup as RS cameras need a bit of time before the first read. If we don't wait, the first read from the warmup will raise.
-            self.warmup_s = max(self.warmup_s, 1)
+        # NOTE(Steven/Caroline): Enforcing at least one second of warmup as RS cameras need a bit of time before the first read. If we don't wait, the first read from the warmup will raise.
+        self.warmup_s = max(self.warmup_s, 1)

-            warmup_read = self.async_read if self.use_rgb else self.async_read_depth
-            start_time = time.time()
-            while time.time() - start_time < self.warmup_s:
-                warmup_read(timeout_ms=self.warmup_s * 1000)
-                time.sleep(0.1)
-            with self.frame_lock:
-                if (self.use_rgb and self.latest_color_frame is None) or (
-                    self.use_depth and self.latest_depth_frame is None
-                ):
-                    raise ConnectionError(f"{self} failed to capture frames during warmup.")
-        except BaseException:
-            try:
-                self._cleanup_resources()
-            except Exception:
-                logger.exception(f"Failed to fully clean up {self} after connect() failed.")
-            self._reset_connection_settings()
-            raise
+        warmup_read = self.async_read if self.use_rgb else self.async_read_depth
+        start_time = time.time()
+        while time.time() - start_time < self.warmup_s:
+            warmup_read(timeout_ms=self.warmup_s * 1000)
+            time.sleep(0.1)
+        with self.frame_lock:
+            if (self.use_rgb and self.latest_color_frame is None) or (
+                self.use_depth and self.latest_depth_frame is None
+            ):
+                raise ConnectionError(f"{self} failed to capture frames during warmup.")

        logger.info(f"{self} connected.")

@@ -364,111 +339,6 @@ class RealSenseCamera(Camera):
        self.new_frame_event.clear()
        return self._async_read(timeout_ms=10000, read_depth=read_depth)

-    def _get_color_sensor(self) -> "rs.sensor":
-        """Returns the sensor that controls the color stream.
-
-        Most RealSense cameras expose "RGB Camera" for color. The D405 has no
-        separate RGB module — its color stream comes from "Stereo Module".
-        We try RGB Camera first, then fall back to Stereo Module.
-        """
-        if self.rs_profile is None:
-            raise RuntimeError(f"{self}: rs_profile must be initialized before use.")
-
-        device = self.rs_profile.get_device()
-        sensors = {s.get_info(rs.camera_info.name): s for s in device.query_sensors()}
-
-        for name in ("RGB Camera", "Stereo Module"):
-            if name in sensors:
-                return sensors[name]
-
-        available = list(sensors.keys())
-        raise RuntimeError(f"{self}: no color sensor found. Available sensors: {available}")
-
-    def _set_sensor_option(self, sensor: "rs.sensor", option: "rs.option", value: float, label: str) -> None:
-        """Sets a sensor option, re-raising range errors with actionable diagnostics."""
-        try:
-            sensor.set_option(option, value)
-        except Exception as e:
-            range_info = ""
-            try:
-                option_range = sensor.get_option_range(option)
-                range_info = (
-                    f" (supported range: min={option_range.min}, max={option_range.max}, "
-                    f"step={option_range.step}, default={option_range.default})"
-                )
-            except Exception:
-                range_info = " (option range unavailable)"
-            raise ValueError(
-                f"{self}: failed to set {label} to {value}{range_info}. Original error: {e}"
-            ) from e
-
-    def _configure_sensor_options(self) -> None:
-        """Applies manual sensor options (exposure, gain, white balance) to the color sensor.
-
-        When exposure or gain is set, auto-exposure is disabled first. When white_balance
-        is set, auto white balance is disabled first. An omitted option is left unchanged,
-        and configuration is skipped entirely if all options are omitted.
-
-        Raises:
-            ValueError: If the sensor does not support a requested option or a requested
-                value is invalid. Invalid-value errors include the option name, requested
-                value, and supported range when available.
-        """
-        if self.exposure is None and self.gain is None and self.white_balance is None:
-            return
-
-        color_sensor = self._get_color_sensor()
-
-        requested_options = (
-            (rs.option.exposure, self.exposure, "exposure"),
-            (rs.option.gain, self.gain, "gain"),
-            (rs.option.white_balance, self.white_balance, "white balance"),
-        )
-        unsupported_options = [
-            label
-            for option, value, label in requested_options
-            if value is not None and not color_sensor.supports(option)
-        ]
-        if unsupported_options:
-            raise ValueError(
-                f"{self}: color sensor does not support requested manual options: {unsupported_options}."
-            )
-
-        manual_exposure_requested = self.exposure is not None or self.gain is not None
-        if manual_exposure_requested:
-            if color_sensor.supports(rs.option.enable_auto_exposure):
-                self._set_sensor_option(color_sensor, rs.option.enable_auto_exposure, 0, "auto-exposure")
-                logger.info(f"{self} auto-exposure disabled.")
-            else:
-                logger.warning(
-                    f"{self} sensor does not support disabling auto-exposure; "
-                    "applying manual exposure/gain directly."
-                )
-
-        if self.exposure is not None:
-            self._set_sensor_option(color_sensor, rs.option.exposure, self.exposure, "exposure")
-            logger.info(f"{self} exposure set to {self.exposure}.")
-
-        if self.gain is not None:
-            self._set_sensor_option(color_sensor, rs.option.gain, self.gain, "gain")
-            logger.info(f"{self} gain set to {self.gain}.")
-
-        if self.white_balance is not None:
-            if color_sensor.supports(rs.option.enable_auto_white_balance):
-                self._set_sensor_option(
-                    color_sensor, rs.option.enable_auto_white_balance, 0, "auto white balance"
-                )
-                logger.info(f"{self} auto white balance disabled.")
-            else:
-                logger.warning(
-                    f"{self} sensor does not support disabling auto white balance; "
-                    "applying manual white balance directly."
-                )
-            self._set_sensor_option(
-                color_sensor, rs.option.white_balance, self.white_balance, "white balance"
-            )
-            logger.info(f"{self} white balance set to {self.white_balance}.")
-
    @check_if_not_connected
    def read_depth(self, timeout_ms: int = 200) -> NDArray[Any]:
        """
@@ -671,27 +541,6 @@ class RealSenseCamera(Camera):
            self.latest_timestamp = None
            self.new_frame_event.clear()

-    def _cleanup_resources(self) -> None:
-        """Stop background reads and stop the pipeline, including after partial setup."""
-        read_thread = self.thread
-        rs_pipeline = self.rs_pipeline
-
-        try:
-            self._stop_read_thread()
-        finally:
-            self.rs_pipeline = None
-            self.rs_profile = None
-            try:
-                if rs_pipeline is not None:
-                    rs_pipeline.stop()
-            finally:
-                # Stopping the pipeline may unblock a hardware read that outlived
-                # the first bounded join in _stop_read_thread().
-                if read_thread is not None and read_thread.is_alive():
-                    read_thread.join(timeout=2.0)
-                    if read_thread.is_alive():  # pragma: no cover
-                        logger.warning(f"{self} read thread remained alive after stopping the pipeline.")
-
    def _async_read(self, timeout_ms: float, read_depth: bool = False) -> NDArray[Any]:
        """Shared helper for :meth:`async_read`/:meth:`async_read_depth`: return the latest buffered frame."""
        if self.thread is None or not self.thread.is_alive():
@@ -835,5 +684,18 @@ class RealSenseCamera(Camera):
                f"Attempted to disconnect {self}, but it appears already disconnected."
            )

-        self._cleanup_resources()
+        if self.thread is not None:
+            self._stop_read_thread()
+
+        if self.rs_pipeline is not None:
+            self.rs_pipeline.stop()
+            self.rs_pipeline = None
+            self.rs_profile = None
+
+        with self.frame_lock:
+            self.latest_color_frame = None
+            self.latest_depth_frame = None
+            self.latest_timestamp = None
+            self.new_frame_event.clear()
+
        logger.info(f"{self} disconnected.")
@@ -46,17 +46,6 @@ class RealSenseCameraConfig(CameraConfig):
        use_depth: Whether to enable depth stream. Defaults to False.
        rotation: Image rotation setting (0°, 90°, 180°, or 270°). Defaults to no rotation.
        warmup_s: Time reading frames before returning from connect (in seconds)
-        exposure: Manual exposure value for the color sensor. When set, auto-exposure is
-            disabled and this fixed value is used. Valid ranges are camera-model specific
-            and reported if the value is rejected. Defaults to None (leave unchanged).
-        gain: Manual gain value for the color sensor. When set, auto-exposure is disabled
-            and this fixed gain is used, which also freezes exposure at its current value
-            when no exposure is configured. Valid ranges are camera-model specific and
-            reported if the value is rejected. Defaults to None (leave unchanged).
-        white_balance: Manual white balance value for the color sensor. When set, auto
-            white balance is disabled and this fixed value is used. Valid ranges are
-            camera-model specific and reported if the value is rejected. Defaults to None
-            (leave unchanged).

    Note:
        - Either name or serial_number must be specified.
@@ -72,9 +61,6 @@ class RealSenseCameraConfig(CameraConfig):
    use_depth: bool = False
    rotation: Cv2Rotation = Cv2Rotation.NO_ROTATION
    warmup_s: int = 1
-    exposure: int | None = None
-    gain: int | None = None
-    white_balance: int | None = None

    def __post_init__(self) -> None:
        self.color_mode = ColorMode(self.color_mode)
@@ -83,18 +69,6 @@ class RealSenseCameraConfig(CameraConfig):
        if not self.use_rgb and not self.use_depth:
            raise ValueError("At least one of `use_rgb` or `use_depth` must be enabled.")

-        manual_color_options = {
-            "exposure": self.exposure,
-            "gain": self.gain,
-            "white_balance": self.white_balance,
-        }
-        configured_color_options = [name for name, value in manual_color_options.items() if value is not None]
-        if configured_color_options and not self.use_rgb:
-            raise ValueError(
-                "Manual color sensor options require `use_rgb=True`. "
-                f"Configured options: {configured_color_options}."
-            )
-
        values = (self.fps, self.width, self.height)
        if any(v is not None for v in values) and any(v is None for v in values):
            raise ValueError(
@@ -35,9 +35,9 @@ else:

 if TYPE_CHECKING:
    from lerobot.datasets import LeRobotDataset
-from lerobot.lerobot_types import PolicyAction
 from lerobot.processor import PolicyProcessorPipeline
 from lerobot.robots import Robot
+from lerobot.types import PolicyAction


 def predict_action(
@@ -71,19 +71,13 @@ class DatasetRecordConfig:
    # Number of threads per encoder instance. None = auto (codec default).
    # Lower values reduce CPU usage, maps to 'lp' (via svtav1-params) for libsvtav1 and 'threads' for h264/hevc..
    encoder_threads: int | None = None
-    # Skip appending the date-time tag to repo_id, keeping the user-provided name as-is
-    # (e.g. self-managed versioned names intended for a later `lerobot-edit-dataset merge`).
-    no_stamp: bool = False

    def stamp_repo_id(self) -> None:
        """Append a date-time tag to ``repo_id`` so each recording session gets a unique name.

        Must be called explicitly at dataset *creation* time — not on resume,
        where the existing ``repo_id`` (already stamped) must be preserved.
-        No-op when ``no_stamp`` is set, preserving a user-managed ``repo_id``.
        """
-        if self.no_stamp:
-            return
        if self.repo_id:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            self.repo_id = f"{self.repo_id}_{timestamp}"
@@ -33,6 +33,8 @@ class DatasetConfig:
    # looked up under $HF_LEROBOT_HOME/repo_id and Hub downloads use a revision-safe cache under $HF_LEROBOT_HOME/hub.
    root: str | None = None
    episodes: list[int] | None = None
+    # Episode indices to drop (e.g. corrupt or heterogeneous ones). Applied on top of `episodes`.
+    exclude_episodes: list[int] | None = None
    image_transforms: ImageTransformsConfig = field(default_factory=ImageTransformsConfig)
    revision: str | None = None
    use_imagenet_stats: bool = True
@@ -62,6 +64,10 @@ class DatasetConfig:
            if len(self.episodes) != len(set(self.episodes)):
                duplicates = sorted({ep for ep in self.episodes if self.episodes.count(ep) > 1})
                raise ValueError(f"Episode indices contain duplicates: {duplicates}")
+        if self.exclude_episodes is not None and any(ep < 0 for ep in self.exclude_episodes):
+            raise ValueError(
+                f"exclude_episodes must be non-negative, got: {[ep for ep in self.exclude_episodes if ep < 0]}"
+            )


@dataclass
@@ -163,10 +163,8 @@ class PreTrainedConfig(draccus.ChoiceRegistry, HubMixin, abc.ABC):  # type: igno
        return None

    def _save_pretrained(self, save_directory: Path) -> None:
-        # Encode against the base class so draccus includes the choice "type" key,
-        # which `from_pretrained` needs to resolve the concrete subclass.
-        with open(save_directory / CONFIG_NAME, "w") as f:
-            json.dump(draccus.encode(self, PreTrainedConfig), f, indent=4)
+        with open(save_directory / CONFIG_NAME, "w") as f, draccus.config_type("json"):
+            draccus.dump(self, f, indent=4)

    @classmethod
    def from_pretrained(
@@ -78,7 +78,7 @@ class MessageTurn:
            raise ValueError(f"Unsupported message stream: {self.stream!r}")
        if self.content is None and self.tool_calls_from is None:
            raise ValueError("MessageTurn.content is required unless tool_calls_from is set.")
-        if self.content is not None and not isinstance(self.content, (str, list)):
+        if self.content is not None and not isinstance(self.content, str | list):
            raise TypeError("MessageTurn.content must be a string, a list of HF-style blocks, or None.")
        if isinstance(self.content, list):
            for block in self.content:
@@ -147,7 +147,7 @@ class TrainingRecipe:
        return cls.from_dict(data)

    def _validate_message_recipe(self) -> None:
-        """Ensure every templated binding is known and at least one turn is a target."""
+        """Validate bindings and require text or low-level action supervision."""
        assert self.messages is not None
        known_bindings = set(DEFAULT_BINDINGS) | set(self.bindings or {}) | {"task"}

@@ -156,8 +156,14 @@ class TrainingRecipe:
            if missing:
                raise ValueError(f"MessageTurn references unknown binding(s): {sorted(missing)}")

-        if not any(turn.target for turn in self.messages):
-            raise ValueError("Message recipes must contain at least one target turn.")
+        has_target = any(turn.target for turn in self.messages)
+        has_low_level = any(turn.stream == "low_level" for turn in self.messages)
+        if not (has_target or has_low_level):
+            raise ValueError(
+                "Message recipes must contain at least one supervised turn — "
+                "either ``target: true`` (text CE) or ``stream: low_level`` "
+                "(flow/action loss)."
+            )

    def _validate_blend_recipe(self) -> None:
        """Ensure each blend component is a non-empty, weighted message recipe."""
@@ -0,0 +1,16 @@
+# Predicts subtasks from tasks and trains subtask-conditioned action flow without memory or plans.
+# Requires `subtask` annotations; samples with missing `if_present` bindings do not render.
+
+blend:
+
+  high_level_subtask:
+    weight: 0.30
+    messages:
+      - {role: user, content: "${task}", stream: high_level}
+      - {role: assistant, content: "${subtask}", stream: high_level, target: true, if_present: subtask}
+
+  low_level_execution:
+    weight: 0.70
+    messages:
+      # The low-level stream trains action flow on the generated or annotated subtask.
+      - {role: user, content: "${subtask}", stream: low_level, if_present: subtask}
@@ -0,0 +1,13 @@
+# Paper-style joint sequence (pi0.5 §IV-B): one sample supervises the subtask
+# text with CE and, because the assistant turn is part of the prefix, conditions
+# the FAST and flow action losses on the same annotated subtask in one forward.
+# The supervised span is attended causally; the action losses see task + subtask.
+#
+# Pair with `--policy.joint_subtask_conditioning=true` at inference so the flow
+# prefix reproduces this layout (task turn with state + causal generated subtask).
+# Samples without a `subtask` annotation fall back to a plain task-prompt
+# low-level sample via `if_present`.
+
+messages:
+  - {role: user, content: "${task}", stream: low_level}
+  - {role: assistant, content: "${subtask}", stream: low_level, target: true, if_present: subtask}
@@ -0,0 +1,30 @@
+# Trains subtask prediction, subtask-conditioned action flow, and memory updates without plans.
+# Requires `subtask` and `memory`; missing `if_present` bindings skip the affected sub-recipe.
+
+blend:
+
+  high_level_subtask:
+    weight: 0.25
+    messages:
+      - {role: user, content: "${task}", stream: high_level}
+      - {role: assistant, content: "${subtask}", stream: high_level, target: true, if_present: subtask}
+
+  low_level_execution:
+    weight: 0.60
+    messages:
+      # The low-level stream trains action flow on the generated or annotated subtask.
+      - {role: user, content: "${subtask}", stream: low_level, if_present: subtask}
+
+  memory_update:
+    # `active_at` densifies sparse boundaries while preserving the prior-memory/subtask mapping.
+    # Inference controls update timing through `subtask_change` events.
+    weight: 0.15
+    bindings:
+      prior_memory: "nth_prev(style=memory, offset=1)"
+      current_memory: "active_at(t, style=memory)"
+      completed_subtask: "nth_prev(style=subtask, offset=1)"
+    messages:
+      - {role: user, content: "${task}", stream: high_level}
+      - {role: assistant, content: "Previous memory: ${prior_memory}", stream: high_level, if_present: prior_memory}
+      - {role: user, content: "Completed subtask: ${completed_subtask}", stream: high_level, if_present: completed_subtask}
+      - {role: assistant, content: "${current_memory}", stream: high_level, target: true, if_present: current_memory}
@@ -0,0 +1,70 @@
+# Adds memory, spoken interjection responses, and camera-grounded VQA to subtask/action training.
+# Missing optional annotations skip only their sub-recipe; `say` tool calls tokenize as `<say>...</say>`.
+
+blend:
+
+  high_level_subtask:
+    weight: 0.25
+    messages:
+      - {role: user, content: "${task}", stream: high_level}
+      - {role: assistant, content: "${subtask}", stream: high_level, target: true, if_present: subtask}
+
+  low_level_execution:
+    weight: 0.40
+    messages:
+      # The low-level stream trains action flow on the generated or annotated subtask.
+      - {role: user, content: "${subtask}", stream: low_level, if_present: subtask}
+
+  memory_update:
+    # `active_at` densifies sparse boundaries while preserving the prior-memory/subtask mapping.
+    # Inference controls update timing through `subtask_change` events.
+    weight: 0.10
+    bindings:
+      prior_memory: "nth_prev(style=memory, offset=1)"
+      current_memory: "active_at(t, style=memory)"
+      completed_subtask: "nth_prev(style=subtask, offset=1)"
+    messages:
+      - {role: user, content: "${task}", stream: high_level}
+      - {role: assistant, content: "Previous memory: ${prior_memory}", stream: high_level, if_present: prior_memory}
+      - {role: user, content: "Completed subtask: ${completed_subtask}", stream: high_level, if_present: completed_subtask}
+      - {role: assistant, content: "${current_memory}", stream: high_level, target: true, if_present: current_memory}
+
+  user_interjection_response:
+    weight: 0.10
+    bindings:
+      interjection: "emitted_at(t, style=interjection)"
+      speech: "emitted_at(t, role=assistant, tool_name=say)"
+    messages:
+      - {role: user, content: "${task}", stream: high_level}
+      - {role: user, content: "${interjection}", stream: high_level, if_present: interjection}
+      # The assistant target is a `say` tool call flattened to a `<say>...</say>` marker.
+      - {role: assistant, stream: high_level, target: true, if_present: speech, tool_calls_from: speech}
+
+  # Each camera uses a separate VQA sub-recipe for view-specific binding.
+  ask_vqa_top:
+    weight: 0.075
+    bindings:
+      vqa_query: "emitted_at(t, style=vqa, role=user, camera=observation.images.front)"
+      vqa: "emitted_at(t, style=vqa, role=assistant, camera=observation.images.front)"
+    messages:
+      - role: user
+        stream: high_level
+        if_present: vqa_query
+        content:
+          - {type: image, feature: observation.images.front}
+          - {type: text, text: "${vqa_query}"}
+      - {role: assistant, content: "${vqa}", stream: high_level, target: true, if_present: vqa}
+
+  ask_vqa_wrist:
+    weight: 0.075
+    bindings:
+      vqa_query: "emitted_at(t, style=vqa, role=user, camera=observation.images.wrist)"
+      vqa: "emitted_at(t, style=vqa, role=assistant, camera=observation.images.wrist)"
+    messages:
+      - role: user
+        stream: high_level
+        if_present: vqa_query
+        content:
+          - {type: image, feature: observation.images.wrist}
+          - {type: text, text: "${vqa_query}"}
+      - {role: assistant, content: "${vqa}", stream: high_level, target: true, if_present: vqa}
@@ -103,10 +103,8 @@ class RewardModelConfig(draccus.ChoiceRegistry, HubMixin, abc.ABC):
        pass

    def _save_pretrained(self, save_directory: Path) -> None:
-        # Encode against the base class so draccus includes the choice "type" key,
-        # which `from_pretrained` needs to resolve the concrete subclass.
-        with open(save_directory / CONFIG_NAME, "w") as f:
-            json.dump(draccus.encode(self, RewardModelConfig), f, indent=4)
+        with open(save_directory / CONFIG_NAME, "w") as f, draccus.config_type("json"):
+            draccus.dump(self, f, indent=4)

    @classmethod
    def from_pretrained(
@@ -194,11 +194,7 @@ class TrainPipelineConfig(HubMixin):
            )

        if Path(config_path).resolve().exists():
-            # `config_path` may point at the checkpoint's train_config.json or at its
-            # pretrained_model/ directory (both documented above) — resolve either to
-            # the pretrained_model/ directory.
-            config_path_obj = Path(config_path)
-            policy_dir = config_path_obj.parent if config_path_obj.is_file() else config_path_obj
+            policy_dir = Path(config_path).parent
            self.checkpoint_path = policy_dir.parent
        elif self.job.is_remote:
            return
@@ -19,7 +19,6 @@ import copy
 import logging
 import shutil
 from pathlib import Path
-from typing import Any, NotRequired, TypedDict

 import datasets
 import pandas as pd
@@ -50,32 +49,8 @@ from .utils import (
 )
 from .video_utils import concatenate_video_files, get_video_duration_in_s

-logger = logging.getLogger(__name__)

-type FeatureDict = dict[str, dict[str, Any]]
-type ChunkFile = tuple[int, int]
-
-
-class IndexState(TypedDict):
-    chunk: int
-    file: int
-    src_to_dst: NotRequired[dict[ChunkFile, ChunkFile]]
-
-
-class VideoIndex(TypedDict):
-    chunk: int
-    file: int
-    latest_duration: float
-    episode_duration: float
-    src_to_offset: NotRequired[dict[ChunkFile, float]]
-    src_to_dst: NotRequired[dict[ChunkFile, ChunkFile]]
-    dst_file_durations: NotRequired[dict[ChunkFile, float]]
-
-
-type VideoIndexState = dict[str, VideoIndex]
-
-
-def merge_video_feature_info_for_aggregate(all_metadata: list[LeRobotDatasetMetadata]) -> FeatureDict:
+def merge_video_feature_info_for_aggregate(all_metadata: list[LeRobotDatasetMetadata]) -> dict[str, dict]:
    """Create a merged video feature info dictionary for aggregation. The video encoder info is merged field-by-field: each key is kept only when every source agrees; otherwise that key is set to ``null`` (or ``{}`` for ``video.extra_options``) and a warning is logged.

    Args:
@@ -84,14 +59,14 @@ def merge_video_feature_info_for_aggregate(all_metadata: list[LeRobotDatasetMeta
    Returns:
        dict: A dictionary of merged video feature info.
    """
-    merged_info: FeatureDict = copy.deepcopy(all_metadata[0].features)
+    merged_info = copy.deepcopy(all_metadata[0].features)
    video_keys = [k for k in merged_info if merged_info[k].get("dtype") == "video"]

    for vk in video_keys:
        video_infos = [m.features.get(vk, {}).get("info") or {} for m in all_metadata]
        base_video_info = video_infos[0]

-        merged_encoder_info: dict[str, Any] = {}
+        merged_encoder_info: dict = {}
        fallback_keys: list[str] = []
        for info_key in VIDEO_ENCODER_INFO_KEYS:
            values = [info.get(info_key, None) for info in video_infos]
@@ -105,7 +80,7 @@ def merge_video_feature_info_for_aggregate(all_metadata: list[LeRobotDatasetMeta
                merged_encoder_info[info_key] = {} if info_key == "video.extra_options" else None

        if fallback_keys:
-            logger.warning(
+            logging.warning(
                f"Merging heterogeneous or incomplete video encoder metadata for feature {vk}. "
                f"Setting these keys to null: {fallback_keys}.",
            )
@@ -117,7 +92,7 @@ def merge_video_feature_info_for_aggregate(all_metadata: list[LeRobotDatasetMeta
    return merged_info


-def validate_all_metadata(all_metadata: list[LeRobotDatasetMetadata]) -> tuple[int, str | None, FeatureDict]:
+def validate_all_metadata(all_metadata: list[LeRobotDatasetMetadata]):
    """Validates that all dataset metadata have consistent properties.

    Ensures all datasets have the same fps, robot_type, and features to guarantee
@@ -154,9 +129,7 @@ def validate_all_metadata(all_metadata: list[LeRobotDatasetMetadata]) -> tuple[i
    return fps, robot_type, features


-def update_data_df(
-    df: pd.DataFrame, src_meta: LeRobotDatasetMetadata, dst_meta: LeRobotDatasetMetadata
-) -> pd.DataFrame:
+def update_data_df(df, src_meta, dst_meta):
    """Updates a data DataFrame with new indices and task mappings for aggregation.

    Adjusts episode indices, frame indices, and task indices to account for
@@ -181,12 +154,12 @@ def update_data_df(


 def update_meta_data(
-    df: pd.DataFrame,
-    dst_meta: LeRobotDatasetMetadata,
-    meta_idx: IndexState,
-    data_idx: IndexState,
-    videos_idx: VideoIndexState,
-) -> pd.DataFrame:
+    df,
+    dst_meta,
+    meta_idx,
+    data_idx,
+    videos_idx,
+):
    """Updates metadata DataFrame with new chunk, file, and timestamp indices.

    Adjusts all indices and timestamps to account for previously aggregated
@@ -316,7 +289,7 @@ def aggregate_datasets(
    chunk_size: int | None = None,
    concatenate_videos: bool = True,
    concatenate_data: bool = True,
-) -> None:
+):
    """Aggregates multiple LeRobot datasets into a single unified dataset.

    This is the main function that orchestrates the aggregation process by:
@@ -336,7 +309,7 @@ def aggregate_datasets(
        concatenate_videos: When False, keep one mp4 per source file instead of packing into shards.
        concatenate_data: When False, keep one parquet per source file instead of packing into shards.
    """
-    logger.info("Start aggregate_datasets")
+    logging.info("Start aggregate_datasets")

    if data_files_size_in_mb is None:
        data_files_size_in_mb = DEFAULT_DATA_FILE_SIZE_IN_MB
@@ -368,15 +341,15 @@ def aggregate_datasets(
        video_files_size_in_mb=video_files_size_in_mb,
    )

-    logger.info("Find all tasks")
+    logging.info("Find all tasks")
    unique_tasks = pd.concat([m.tasks for m in all_metadata]).index.unique()
    dst_meta.tasks = pd.DataFrame(
        {"task_index": range(len(unique_tasks))}, index=pd.Index(unique_tasks, name="task")
    )

-    meta_idx: IndexState = {"chunk": 0, "file": 0}
-    data_idx: IndexState = {"chunk": 0, "file": 0}
-    videos_idx: VideoIndexState = {
+    meta_idx = {"chunk": 0, "file": 0}
+    data_idx = {"chunk": 0, "file": 0}
+    videos_idx = {
        key: {"chunk": 0, "file": 0, "latest_duration": 0, "episode_duration": 0} for key in video_keys
    }

@@ -400,17 +373,12 @@ def aggregate_datasets(
        dst_meta.info.total_frames += src_meta.total_frames

    finalize_aggregation(dst_meta, all_metadata)
-    logger.info("Aggregation complete.")
+    logging.info("Aggregation complete.")


 def aggregate_videos(
-    src_meta: LeRobotDatasetMetadata,
-    dst_meta: LeRobotDatasetMetadata,
-    videos_idx: VideoIndexState,
-    video_files_size_in_mb: float,
-    chunk_size: int,
-    concatenate_videos: bool = True,
-) -> VideoIndexState:
+    src_meta, dst_meta, videos_idx, video_files_size_in_mb, chunk_size, concatenate_videos=True
+):
    """Aggregates video chunks from a source dataset into the destination dataset.

    Handles video file concatenation and rotation based on file size limits.
@@ -438,16 +406,15 @@ def aggregate_videos(
            videos_idx[key]["dst_file_durations"] = {}

    for key, video_idx in videos_idx.items():
-        unique_chunk_file_pairs: list[ChunkFile] = sorted(
-            {
-                (chunk, file)
-                for chunk, file in zip(
-                    src_meta.episodes[f"videos/{key}/chunk_index"],
-                    src_meta.episodes[f"videos/{key}/file_index"],
-                    strict=False,
-                )
-            }
-        )
+        unique_chunk_file_pairs = {
+            (chunk, file)
+            for chunk, file in zip(
+                src_meta.episodes[f"videos/{key}/chunk_index"],
+                src_meta.episodes[f"videos/{key}/file_index"],
+                strict=False,
+            )
+        }
+        unique_chunk_file_pairs = sorted(unique_chunk_file_pairs)

        chunk_idx = video_idx["chunk"]
        file_idx = video_idx["file"]
@@ -522,14 +489,7 @@ def aggregate_videos(
    return videos_idx


-def aggregate_data(
-    src_meta: LeRobotDatasetMetadata,
-    dst_meta: LeRobotDatasetMetadata,
-    data_idx: IndexState,
-    data_files_size_in_mb: float,
-    chunk_size: int,
-    concatenate_data: bool = True,
-) -> IndexState:
+def aggregate_data(src_meta, dst_meta, data_idx, data_files_size_in_mb, chunk_size, concatenate_data=True):
    """Aggregates data chunks from a source dataset into the destination dataset.

    Reads source data files, updates indices to match the aggregated dataset,
@@ -550,16 +510,14 @@ def aggregate_data(
    Returns:
        dict: Updated data_idx with current chunk and file indices.
    """
-    unique_chunk_file_ids: list[ChunkFile] = sorted(
-        {
-            (c, f)
-            for c, f in zip(
-                src_meta.episodes["data/chunk_index"],
-                src_meta.episodes["data/file_index"],
-                strict=False,
-            )
-        }
-    )
+    unique_chunk_file_ids = {
+        (c, f)
+        for c, f in zip(
+            src_meta.episodes["data/chunk_index"], src_meta.episodes["data/file_index"], strict=False
+        )
+    }
+
+    unique_chunk_file_ids = sorted(unique_chunk_file_ids)
    contains_images = len(dst_meta.image_keys) > 0

    # retrieve features schema for proper image typing in parquet
@@ -567,7 +525,7 @@ def aggregate_data(

    # Track source to destination file mapping for metadata update
    # This is critical for handling datasets that are already results of a merge
-    src_to_dst: dict[ChunkFile, ChunkFile] = {}
+    src_to_dst: dict[tuple[int, int], tuple[int, int]] = {}

    for src_chunk_idx, src_file_idx in unique_chunk_file_ids:
        src_path = src_meta.root / DEFAULT_DATA_PATH.format(
@@ -606,13 +564,7 @@ def aggregate_data(
    return data_idx


-def aggregate_metadata(
-    src_meta: LeRobotDatasetMetadata,
-    dst_meta: LeRobotDatasetMetadata,
-    meta_idx: IndexState,
-    data_idx: IndexState,
-    videos_idx: VideoIndexState,
-) -> IndexState:
+def aggregate_metadata(src_meta, dst_meta, meta_idx, data_idx, videos_idx):
    """Aggregates metadata from a source dataset into the destination dataset.

    Reads source metadata files, updates all indices and timestamps,
@@ -628,16 +580,16 @@ def aggregate_metadata(
    Returns:
        dict: Updated meta_idx with current chunk and file indices.
    """
-    chunk_file_ids: list[ChunkFile] = sorted(
-        {
-            (c, f)
-            for c, f in zip(
-                src_meta.episodes["meta/episodes/chunk_index"],
-                src_meta.episodes["meta/episodes/file_index"],
-                strict=False,
-            )
-        }
-    )
+    chunk_file_ids = {
+        (c, f)
+        for c, f in zip(
+            src_meta.episodes["meta/episodes/chunk_index"],
+            src_meta.episodes["meta/episodes/file_index"],
+            strict=False,
+        )
+    }
+
+    chunk_file_ids = sorted(chunk_file_ids)
    for chunk_idx, file_idx in chunk_file_ids:
        src_path = src_meta.root / DEFAULT_EPISODES_PATH.format(chunk_index=chunk_idx, file_index=file_idx)
        df = pd.read_parquet(src_path)
@@ -670,16 +622,16 @@ def aggregate_metadata(
 def append_or_create_parquet_file(
    df: pd.DataFrame,
    src_path: Path,
-    idx: IndexState,
+    idx: dict[str, int],
    max_mb: float,
    chunk_size: int,
    default_path: str,
    contains_images: bool = False,
-    aggr_root: Path | None = None,
+    aggr_root: Path = None,
    hf_features: datasets.Features | None = None,
    concatenate: bool = True,
    one_row_group_per_episode: bool = False,
-) -> tuple[IndexState, ChunkFile]:
+) -> tuple[dict[str, int], tuple[int, int]]:
    """Appends data to an existing parquet file or creates a new one based on size constraints.

    Manages file rotation when size limits are exceeded to prevent individual files
@@ -702,13 +654,7 @@ def append_or_create_parquet_file(
    Returns:
        tuple: (updated_idx, (dst_chunk, dst_file)) where updated_idx is the index dict
               and (dst_chunk, dst_file) is the actual destination file the data was written to.
-
-    Raises:
-        ValueError: If aggr_root is not provided.
    """
-    if aggr_root is None:
-        raise ValueError("aggr_root must be provided.")
-
    dst_chunk, dst_file = idx["chunk"], idx["file"]
    dst_path = aggr_root / default_path.format(chunk_index=dst_chunk, file_index=dst_file)

@@ -752,9 +698,7 @@ def append_or_create_parquet_file(
    return idx, (dst_chunk, dst_file)


-def finalize_aggregation(
-    aggr_meta: LeRobotDatasetMetadata, all_metadata: list[LeRobotDatasetMetadata]
-) -> None:
+def finalize_aggregation(aggr_meta, all_metadata):
    """Finalizes the dataset aggregation by writing summary files and statistics.

    Writes the tasks file, info file with total counts and splits, and
@@ -764,16 +708,16 @@ def finalize_aggregation(
        aggr_meta: Aggregated dataset metadata.
        all_metadata: List of all source dataset metadata objects.
    """
-    logger.info("write tasks")
+    logging.info("write tasks")
    write_tasks(aggr_meta.tasks, aggr_meta.root)

-    logger.info("write info")
+    logging.info("write info")
    aggr_meta.info.total_tasks = len(aggr_meta.tasks)
    aggr_meta.info.total_episodes = sum(m.total_episodes for m in all_metadata)
    aggr_meta.info.total_frames = sum(m.total_frames for m in all_metadata)
    aggr_meta.info.splits = {"train": f"0:{sum(m.total_episodes for m in all_metadata)}"}
    write_info(aggr_meta.info, aggr_meta.root)

-    logger.info("write stats")
+    logging.info("write stats")
    aggr_meta.stats = aggregate_stats([m.stats for m in all_metadata])
    write_stats(aggr_meta.stats, aggr_meta.root)
@@ -188,8 +188,8 @@ class LeRobotDatasetMetadata:
    def _load_metadata(self):
        self.info = load_info(self.root)
        check_version_compatibility(self.repo_id, self._version, CODEBASE_VERSION)
-        self.tasks = load_tasks(self.root) if self.total_tasks > 0 else None
-        self.episodes = load_episodes(self.root) if self.total_episodes > 0 else None
+        self.tasks = load_tasks(self.root)
+        self.episodes = load_episodes(self.root)
        self.stats = load_stats(self.root)

    def ensure_readable(self) -> None:
@@ -163,10 +163,40 @@ class DatasetReader:
    def _load_hf_dataset(self) -> datasets.Dataset:
        """hf_dataset contains all the observations, states, actions, rewards, etc."""
        features = get_hf_features_from_features(self._meta.features)
+        # Annotated datasets may have language columns absent from metadata.
+        # Extend the schema before the strict Parquet cast.
+        features = self._extend_features_with_language_columns(features)
        hf_dataset = load_nested_dataset(self.root / "data", features=features, episodes=self.episodes)
        hf_dataset.set_transform(hf_transform_to_torch)
        return hf_dataset

+    def _extend_features_with_language_columns(self, features: datasets.Features) -> datasets.Features:
+        """Register language columns found in Parquet but missing from metadata."""
+        # Leave empty datasets to fail through the normal loading path.
+        try:
+            sample = next((self.root / "data").glob("*/*.parquet"))
+        except StopIteration:
+            return features
+
+        from pyarrow import parquet as _pq  # noqa: PLC0415
+
+        schema_names = set(_pq.read_schema(sample).names)
+        from .language import (  # noqa: PLC0415
+            LANGUAGE_EVENTS,
+            LANGUAGE_PERSISTENT,
+            language_events_column_feature,
+            language_persistent_column_feature,
+        )
+
+        extra: dict[str, object] = {}
+        if LANGUAGE_PERSISTENT in schema_names and LANGUAGE_PERSISTENT not in features:
+            extra[LANGUAGE_PERSISTENT] = language_persistent_column_feature()
+        if LANGUAGE_EVENTS in schema_names and LANGUAGE_EVENTS not in features:
+            extra[LANGUAGE_EVENTS] = language_events_column_feature()
+        if not extra:
+            return features
+        return datasets.Features({**features, **extra})
+
    def _check_cached_episodes_sufficient(self) -> bool:
        """Check if the cached dataset contains all requested episodes and their video files."""
        if self.hf_dataset is None or len(self.hf_dataset) == 0:
@@ -66,6 +66,17 @@ def resolve_delta_timestamps(
    return delta_timestamps


+def _resolve_episodes(
+    episodes: list[int] | None, exclude_episodes: list[int] | None, total_episodes: int
+) -> list[int] | None:
+    """Apply an episode exclusion list on top of an optional allowlist."""
+    if not exclude_episodes:
+        return episodes
+    base = episodes if episodes is not None else list(range(total_episodes))
+    excluded = set(exclude_episodes)
+    return [episode for episode in base if episode not in excluded]
+
+
 def make_dataset(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDataset:
    """Handles the logic of setting up delta timestamps and image transforms before creating a dataset.

@@ -87,11 +98,14 @@ def make_dataset(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDatas
            cfg.dataset.repo_id, root=cfg.dataset.root, revision=cfg.dataset.revision
        )
        delta_timestamps = resolve_delta_timestamps(cfg.trainable_config, ds_meta)
+        episodes = _resolve_episodes(
+            cfg.dataset.episodes, cfg.dataset.exclude_episodes, ds_meta.total_episodes
+        )
        if not cfg.dataset.streaming:
            dataset = LeRobotDataset(
                cfg.dataset.repo_id,
                root=cfg.dataset.root,
-                episodes=cfg.dataset.episodes,
+                episodes=episodes,
                delta_timestamps=delta_timestamps,
                image_transforms=image_transforms,
                revision=cfg.dataset.revision,
@@ -104,7 +118,7 @@ def make_dataset(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDatas
            dataset = StreamingLeRobotDataset(
                cfg.dataset.repo_id,
                root=cfg.dataset.root,
-                episodes=cfg.dataset.episodes,
+                episodes=episodes,
                delta_timestamps=delta_timestamps,
                image_transforms=image_transforms,
                revision=cfg.dataset.revision,
@@ -162,14 +162,28 @@ def render_sample(
    task: str | None = None,
    dataset_ctx: Any | None = None,
 ) -> RenderedMessages | None:
-    """Render the chat-style messages for a single dataset sample.
+    """Resolve one sample's bindings and render its message recipe.

-    Resolves the recipe's bindings against ``persistent`` and ``events`` rows
-    at frame timestamp ``t``, then expands the recipe's message templates.
-    Returns ``None`` if the resolved sample contains no target message.
+    Returns ``None`` when no text or low-level action supervision applies.
    """
    persistent_rows = _normalize_rows(persistent or [])
    event_rows = _normalize_rows(events or [])
+
+    # Route sparse VQA frames to a matching view-specific component before weighted selection.
+    # This avoids dropping annotated frames or selecting VQA without annotations.
+    if recipe.blend is not None:
+        vqa_rendered = _render_vqa_if_present(
+            recipe,
+            persistent=persistent_rows,
+            events=event_rows,
+            t=t,
+            sample_idx=sample_idx,
+            task=task,
+            dataset_ctx=dataset_ctx,
+        )
+        if vqa_rendered is not None:
+            return vqa_rendered
+
    selected_recipe = _select_recipe(recipe, sample_idx)
    bindings = _resolve_bindings(
        selected_recipe,
@@ -183,6 +197,55 @@ def render_sample(
    return _render_message_recipe(selected_recipe, bindings)


+def _render_vqa_if_present(
+    recipe: TrainingRecipe,
+    *,
+    persistent: Sequence[LanguageRow],
+    events: Sequence[LanguageRow],
+    t: float,
+    sample_idx: int,
+    task: str | None,
+    dataset_ctx: Any | None,
+) -> RenderedMessages | None:
+    """Render a matching VQA component, or return ``None`` for normal selection.
+
+    Multiple matching views are selected deterministically by relative weight.
+    """
+    assert recipe.blend is not None
+    renderable: list[tuple[float, RenderedMessages]] = []
+    for name, component in recipe.blend.items():
+        if not name.startswith("ask_vqa"):
+            continue
+        bindings = _resolve_bindings(
+            component,
+            persistent=persistent,
+            events=events,
+            t=t,
+            sample_idx=sample_idx,
+            task=task,
+            dataset_ctx=dataset_ctx,
+        )
+        rendered = _render_message_recipe(component, bindings)
+        if rendered is not None:
+            renderable.append((float(component.weight or 0.0), rendered))
+
+    if not renderable:
+        return None
+    if len(renderable) == 1:
+        return renderable[0][1]
+
+    # Choose among matching cameras by relative weight, or uniformly when all weights are zero.
+    total = sum(w for w, _ in renderable) or float(len(renderable))
+    digest = hashlib.blake2b(f"vqa:{sample_idx}".encode(), digest_size=8).digest()
+    draw = int.from_bytes(digest, "big") / 2**64 * total
+    cumulative = 0.0
+    for w, rendered in renderable:
+        cumulative += w or (total / len(renderable))
+        if draw < cumulative:
+            return rendered
+    return renderable[-1][1]
+
+
 def _select_recipe(recipe: TrainingRecipe, sample_idx: int) -> TrainingRecipe:
    """Pick a deterministic blend component for ``sample_idx`` (or return ``recipe``)."""
    if recipe.blend is None:
@@ -346,7 +409,9 @@ def _render_message_recipe(
        if turn.target:
            target_indices.append(message_idx)

-    if not target_indices:
+    # Keep samples with either text targets or low-level action supervision.
+    has_low_level = any(stream == "low_level" for stream in streams)
+    if not target_indices and not has_low_level:
        return None

    rendered = {
@@ -403,14 +468,12 @@ def _validate_rendered(rendered: RenderedMessages) -> None:

    if len(streams) != len(messages):
        raise ValueError("message_streams must be aligned with messages.")
-    if not target_indices:
-        raise ValueError("Rendered samples must contain at least one target message.")
+    # Require text or low-level action supervision.
+    if not target_indices and not any(s == "low_level" for s in streams):
+        raise ValueError("Rendered samples must contain a target message or a low_level-stream message.")
    for idx in target_indices:
        if idx < 0 or idx >= len(messages):
            raise ValueError(f"Target message index {idx} is out of bounds.")
-    # ``stream`` is enforced non-None at MessageTurn construction time
-    # (see ``MessageTurn.__post_init__``), so a missing stream here would
-    # mean the dataclass invariant was bypassed; no need to re-check.


 def _nth_relative(
@@ -17,8 +17,8 @@ from collections.abc import Sequence
 from typing import Any

 from lerobot.configs import PipelineFeatureType
-from lerobot.lerobot_types import RobotAction, RobotObservation
 from lerobot.processor import DataProcessorPipeline
+from lerobot.types import RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE, OBS_STR
 from lerobot.utils.feature_utils import hw_to_dataset_features

@@ -58,10 +58,6 @@ class LookAheadError(Exception):
    pass


-class _ShardExhaustedError(Exception):
-    """Raised when a streaming dataset shard has no more items."""
-
-
 class Backtrackable[T]:
    """
    Wrap any iterator/iterable so you can step back up to `history` items
@@ -182,7 +178,7 @@ class Backtrackable[T]:
        """
        Check if we can go back `steps` items without raising an IndexError.
        """
-        return steps < len(self._back_buf) + self._cursor
+        return steps <= len(self._back_buf) + self._cursor

    def can_peek_ahead(self, steps: int = 1) -> bool:
        """
@@ -426,7 +422,10 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):
                    else:
                        frames_buffer.append(frame)
                    break  # random shard sampled, switch shard
-            except _ShardExhaustedError:
+            except (
+                RuntimeError,
+                StopIteration,
+            ):  # NOTE: StopIteration inside a generator throws a RuntimeError since python 3.7
                del idx_to_backtrack_dataset[shard_key]  # Remove exhausted shard, onto another shard

        # Once shards are all exhausted, shuffle the buffer and yield the remaining frames
@@ -504,11 +503,7 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset):

    def make_frame(self, dataset_iterator: Backtrackable) -> Generator:
        """Makes a frame starting from a dataset iterator"""
-        try:
-            item = next(dataset_iterator)
-        except StopIteration as e:
-            # Translate exhaustion here, before PEP 479 turns it into an indistinguishable RuntimeError.
-            raise _ShardExhaustedError from e
+        item = next(dataset_iterator)
        item = item_to_torch(item)

        updates = []  # list of "updates" to apply to the item retrieved from hf_dataset (w/o camera features)
@@ -507,7 +507,7 @@ class MetaworldEnv(EnvConfig):
 class RoboCasaEnv(EnvConfig):
    task: str = "CloseFridge"
    fps: int = 20
-    episode_length: int | None = None
+    episode_length: int = 1000
    obs_type: str = "pixels_agent_pos"
    render_mode: str = "rgb_array"
    camera_name: str = "robot0_agentview_left,robot0_eye_in_hand,robot0_agentview_right"
@@ -30,7 +30,7 @@ from gymnasium import spaces
 from libero.libero import benchmark, get_libero_path
 from libero.libero.envs import OffScreenRenderEnv

-from lerobot.lerobot_types import RobotObservation
+from lerobot.types import RobotObservation

 from .utils import _LazyAsyncVectorEnv, parse_camera_names

@@ -384,12 +384,7 @@ class LiberoEnv(gym.Env):

    def close(self):
        if self._env is not None:
-            try:
-                self._env.close()
-            finally:
-                # LIBERO deletes its inner env on close, so this wrapper must
-                # be recreated before the next reset.
-                self._env = None
+            self._env.close()


 def _make_env_fns(
@@ -25,7 +25,7 @@ import metaworld.policies as policies
 import numpy as np
 from gymnasium import spaces

-from lerobot.lerobot_types import RobotObservation
+from lerobot.types import RobotObservation

 from .utils import _LazyAsyncVectorEnv

@@ -25,7 +25,7 @@ import gymnasium as gym
 import numpy as np
 from gymnasium import spaces

-from lerobot.lerobot_types import RobotObservation
+from lerobot.types import RobotObservation

 from .utils import _LazyAsyncVectorEnv, parse_camera_names

@@ -98,19 +98,6 @@ def _resolve_tasks(task: str) -> tuple[list[str], str | None]:
    return names, None


-def _get_task_horizon(task: str) -> int:
-    """Return the rollout horizon registered by RoboCasa for a task."""
-    from robocasa.utils.dataset_registry_utils import get_task_horizon
-
-    try:
-        return int(get_task_horizon(task))
-    except ValueError as exc:
-        raise ValueError(
-            f"No RoboCasa horizon is registered for task '{task}'. "
-            "Set `--env.episode_length=<steps>` explicitly."
-        ) from exc
-
-
 def convert_action(flat_action: np.ndarray) -> dict[str, Any]:
    """Split a flat (12,) action vector into a RoboCasa action dict.

@@ -167,7 +154,7 @@ class RoboCasaEnv(gym.Env):

        self.camera_name = parse_camera_names(camera_name)

-        self._max_episode_steps = episode_length if episode_length is not None else _get_task_horizon(task)
+        self._max_episode_steps = episode_length if episode_length is not None else 1000

        # Deferred — created on first reset() inside the worker subprocess
        # to avoid inheriting stale GPU/EGL contexts across fork().
@@ -28,7 +28,7 @@ import numpy as np
 import torch
 from gymnasium import spaces

-from lerobot.lerobot_types import RobotObservation
+from lerobot.types import RobotObservation
 from lerobot.utils.import_utils import _scipy_available

 from .utils import _LazyAsyncVectorEnv
@@ -384,9 +384,7 @@ class RoboTwinEnv(gym.Env):

        self._env: Any | None = None  # deferred — created on first reset() inside worker
        self._step_count: int = 0
-        self._black_frame: np.ndarray = np.zeros(
-            (self.observation_height, self.observation_width, 3), dtype=np.uint8
-        )
+        self._black_frame = np.zeros((self.observation_height, self.observation_width, 3), dtype=np.uint8)

        image_spaces = {
            cam: spaces.Box(
@@ -37,7 +37,7 @@ import numpy as np
 from gymnasium import spaces
 from scipy.spatial.transform import Rotation

-from lerobot.lerobot_types import RobotObservation
+from lerobot.types import RobotObservation

 from .utils import _LazyAsyncVectorEnv

@@ -373,7 +373,7 @@ class VLABenchEnv(gym.Env):

        if action.shape[0] != 7:
            # Unknown layout — fall back to zero-pad so the sim doesn't crash.
-            padded: np.ndarray = np.zeros(ctrl_dim, dtype=np.float64)
+            padded = np.zeros(ctrl_dim, dtype=np.float64)
            padded[: min(action.shape[0], ctrl_dim)] = action[:ctrl_dim]
            return padded

@@ -122,9 +122,6 @@ MODEL_ENCODING_TABLE = {
    "xm430-w350": X_SERIES_ENCODINGS_TABLE,
    "xm540-w270": X_SERIES_ENCODINGS_TABLE,
    "xc430-w150": X_SERIES_ENCODINGS_TABLE,
-    "xh540-w150": X_SERIES_ENCODINGS_TABLE,
-    "xc330-t288": X_SERIES_ENCODINGS_TABLE,
-    "xc330-t181": X_SERIES_ENCODINGS_TABLE,
 }

 # {model: model_resolution}
@@ -137,9 +134,6 @@ MODEL_RESOLUTION = {
    "xm430-w350": 4096,
    "xm540-w270": 4096,
    "xc430-w150": 4096,
-    "xh540-w150": 4096,
-    "xc330-t288": 4096,
-    "xc330-t181": 4096,
 }

 # {model: model_number}
@@ -151,9 +145,6 @@ MODEL_NUMBER_TABLE = {
    "xm430-w350": 1020,
    "xm540-w270": 1120,
    "xc430-w150": 1070,
-    "xh540-w150": 1110,
-    "xc330-t288": 1220,
-    "xc330-t181": 1210,
 }

 # {model: available_operating_modes}
@@ -165,9 +156,6 @@ MODEL_OPERATING_MODES = {
    "xm430-w350": [0, 1, 3, 4, 5, 16],
    "xm540-w270": [0, 1, 3, 4, 5, 16],
    "xc430-w150": [1, 3, 4, 16],
-    "xh540-w150": [0, 1, 3, 4, 5, 16],
-    "xc330-t288": [0, 1, 3, 4, 5, 16],
-    "xc330-t181": [0, 1, 3, 4, 5, 16],
 }

 MODEL_CONTROL_TABLE = {
@@ -178,9 +166,6 @@ MODEL_CONTROL_TABLE = {
    "xm430-w350": X_SERIES_CONTROL_TABLE,
    "xm540-w270": X_SERIES_CONTROL_TABLE,
    "xc430-w150": X_SERIES_CONTROL_TABLE,
-    "xh540-w150": X_SERIES_CONTROL_TABLE,
-    "xc330-t288": X_SERIES_CONTROL_TABLE,
-    "xc330-t181": X_SERIES_CONTROL_TABLE,
 }

 MODEL_BAUDRATE_TABLE = {
@@ -191,9 +176,6 @@ MODEL_BAUDRATE_TABLE = {
    "xm430-w350": X_SERIES_BAUDRATE_TABLE,
    "xm540-w270": X_SERIES_BAUDRATE_TABLE,
    "xc430-w150": X_SERIES_BAUDRATE_TABLE,
-    "xh540-w150": X_SERIES_BAUDRATE_TABLE,
-    "xc330-t288": X_SERIES_BAUDRATE_TABLE,
-    "xc330-t181": X_SERIES_BAUDRATE_TABLE,
 }

 AVAILABLE_BAUDRATES = [
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Any
 import torch

 from lerobot.configs.types import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import TransitionKey
 from lerobot.processor import (
    ComplementaryDataProcessorStep,
    PolicyAction,
@@ -32,6 +31,7 @@ from lerobot.processor import (
    make_default_policy_processor_steps,
    make_policy_processor_pipelines,
 )
+from lerobot.types import TransitionKey
 from lerobot.utils.constants import OBS_STATE
 from lerobot.utils.import_utils import _transformers_available, require_package

@@ -42,9 +42,6 @@ class Evo1Policy(PreTrainedPolicy):
    config_class = Evo1Config
    name = "evo1"

-    def supports_rtc(self) -> bool:
-        return True
-
    def __init__(self, config: Evo1Config, *, vlm_hub_kwargs: dict | None = None, **kwargs):
        super().__init__(config)
        config.validate_features()
@@ -21,7 +21,6 @@ from typing import Any
 import torch

 from lerobot.configs import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    AddBatchDimensionProcessorStep,
    DeviceProcessorStep,
@@ -41,6 +40,7 @@ from lerobot.processor.converters import (
    policy_action_to_transition,
    transition_to_policy_action,
 )
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import (
    ACTION,
    DONE,
@@ -302,33 +302,6 @@ def _pad_evo1_stats(
    return padded_stats


-def _refresh_evo1_normalization_steps(
-    config: Evo1Config,
-    preprocessor: PolicyProcessorPipeline,
-    postprocessor: PolicyProcessorPipeline,
-) -> None:
-    """Re-pad checkpoint-loaded (un)normalizer stats/features to EVO1's fixed widths.
-
-    Loading a checkpoint injects the raw dataset stats (unpadded to max_state_dim/max_action_dim)
-    into the (un)normalizer via the generic override path in make_pre_post_processors. Those stats
-    and their declared features must be re-padded/reshaped to EVO1's fixed widths, otherwise
-    normalization fails against the padded state/action tensors (e.g. state padded to 24 vs. 8-dim
-    LIBERO stats). Padding is a no-op when stats are already at the target width.
-    """
-    normalization_features = _evo1_normalization_features(config)
-    action_features = _evo1_action_features(config)
-    for step in preprocessor.steps:
-        if isinstance(step, NormalizerProcessorStep):
-            step.features = normalization_features
-            step.stats = _pad_evo1_stats(config, step.stats)
-            step.to(device=step.device, dtype=step.dtype)
-    for step in postprocessor.steps:
-        if isinstance(step, UnnormalizerProcessorStep):
-            step.features = action_features
-            step.stats = _pad_evo1_stats(config, step.stats)
-            step.to(device=step.device, dtype=step.dtype)
-
-
 def reconcile_evo1_processors(
    config: Evo1Config,
    preprocessor: PolicyProcessorPipeline,
@@ -336,19 +309,16 @@ def reconcile_evo1_processors(
 ) -> tuple[PolicyProcessorPipeline, PolicyProcessorPipeline]:
    """Reconcile checkpoint-loaded pipelines with the current EVO1 config.

-    Three things cannot be restored from a serialized pipeline alone: the EVO1 batch converter
-    (converters are plain functions and are never serialized), eval-time CLI overrides of the
-    action postprocessing flags (`postprocess_action_dim`, `binarize_gripper`, `gripper_*`), and the
-    (un)normalizer stats/features when the generic override path injects raw, unpadded dataset
-    stats. This restores the converter, re-pads the normalization stats to EVO1's fixed widths, and
-    rebuilds the action step from the current config so those overrides take effect.
+    Two things cannot be restored from a serialized pipeline alone: the EVO1 batch converter
+    (converters are plain functions and are never serialized), and eval-time CLI overrides of the
+    action postprocessing flags (`postprocess_action_dim`, `binarize_gripper`, `gripper_*`). This
+    restores the converter and rebuilds the action step from the current config so those overrides
+    take effect.
    """
    # Pipelines reloaded from a checkpoint come back with the default batch converter, which drops
    # non-observation extras (embodiment_id, state_mask, custom task fields) needed by EVO1.
    preprocessor.to_transition = evo1_batch_to_transition

-    _refresh_evo1_normalization_steps(config, preprocessor, postprocessor)
-
    action_step = Evo1ActionProcessorStep(
        action_dim=_evo1_action_dim(config),
        binarize_gripper=config.binarize_gripper,
@@ -28,7 +28,6 @@ if TYPE_CHECKING:

 from lerobot.configs import FeatureType, PreTrainedConfig
 from lerobot.envs import EnvConfig, env_to_policy_features
-from lerobot.lerobot_types import PolicyAction
 from lerobot.processor import (
    AbsoluteActionsProcessorStep,
    PolicyProcessorPipeline,
@@ -38,25 +37,19 @@ from lerobot.processor import (
    transition_to_batch,
    transition_to_policy_action,
 )
+from lerobot.types import PolicyAction
 from lerobot.utils.constants import (
    ACTION,
    POLICY_POSTPROCESSOR_DEFAULT_NAME,
    POLICY_PREPROCESSOR_DEFAULT_NAME,
 )
 from lerobot.utils.feature_utils import dataset_to_policy_features
-from lerobot.utils.import_utils import _peft_available, require_package

 from .evo1.configuration_evo1 import Evo1Config
 from .groot.configuration_groot import GrootConfig
 from .pretrained import PreTrainedPolicy
 from .utils import validate_visual_features_consistency

-if TYPE_CHECKING or _peft_available:
-    from peft import PeftConfig, PeftModel
-else:
-    PeftConfig = None
-    PeftModel = None
-

 def _reconnect_relative_absolute_steps(
    preprocessor: PolicyProcessorPipeline, postprocessor: PolicyProcessorPipeline
@@ -341,15 +334,12 @@ def make_policy(
        # Load a pretrained PEFT model on top of the policy. The pretrained path points to the folder/repo
        # of the adapter and the adapter's config contains the path to the base policy. So we need the
        # adapter config first, then load the correct policy and then apply PEFT.
-        require_package("peft", extra="peft")
+        from peft import PeftConfig, PeftModel

        logging.info("Loading policy's PEFT adapter.")

        peft_pretrained_path = str(cfg.pretrained_path)
-        peft_config = PeftConfig.from_pretrained(
-            peft_pretrained_path,
-            revision=cfg.pretrained_revision,
-        )
+        peft_config = PeftConfig.from_pretrained(peft_pretrained_path)

        kwargs["pretrained_name_or_path"] = peft_config.base_model_name_or_path
        if not kwargs["pretrained_name_or_path"]:
@@ -360,14 +350,9 @@ def make_policy(
                "the adapter was trained."
            )

-        kwargs["revision"] = peft_config.revision
        policy = policy_cls.from_pretrained(**kwargs)
        policy = PeftModel.from_pretrained(
-            policy,
-            peft_pretrained_path,
-            config=peft_config,
-            revision=cfg.pretrained_revision,
-            is_trainable=True,
+            policy, peft_pretrained_path, config=peft_config, is_trainable=True
        )

    else:
@@ -68,9 +68,6 @@ class GrootPolicy(PreTrainedPolicy):
    name = "groot"
    config_class = GrootConfig

-    def supports_rtc(self) -> bool:
-        return True
-
    def __init__(self, config: GrootConfig, **kwargs):
        """Initialize Groot policy wrapper."""
        require_package("transformers", extra="groot")
@@ -50,7 +50,6 @@ if TYPE_CHECKING or _datasets_available:
 else:
    LeRobotDataset = None

-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    AbsoluteActionsProcessorStep,
    AddBatchDimensionProcessorStep,
@@ -67,6 +66,7 @@ from lerobot.processor import (
    transition_to_batch,
    transition_to_policy_action,
 )
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import (
    ACTION,
    OBS_IMAGE,
@@ -43,22 +43,11 @@ from torch.distributions import Beta

 from lerobot.policies.pretrained import PreTrainedPolicy
 from lerobot.utils.constants import ACTION
-from lerobot.utils.import_utils import (
-    _peft_available,
-    _scipy_available,
-    _transformers_available,
-    require_package,
-)
+from lerobot.utils.import_utils import _scipy_available, _transformers_available, require_package

 from ..rtc.modeling_rtc import RTCProcessor
 from .configuration_molmoact2 import MolmoAct2Config

-if TYPE_CHECKING or _peft_available:
-    from peft import LoraConfig, get_peft_model
-else:
-    LoraConfig = None
-    get_peft_model = None
-
 logger = logging.getLogger(__name__)


@@ -520,9 +509,6 @@ class MolmoAct2Policy(PreTrainedPolicy):
    config_class = MolmoAct2Config
    name = "molmoact2"

-    def supports_rtc(self) -> bool:
-        return self.config.inference_action_mode == "continuous"
-
    def __init__(
        self,
        config: MolmoAct2Config,
@@ -1745,11 +1731,13 @@ class MolmoAct2Policy(PreTrainedPolicy):

    def _build_inner_lora_config(self):
        require_package("peft", extra="molmoact2")
+        from peft import LoraConfig

        return LoraConfig(**self._get_inner_peft_targets())

    def _apply_lora_adapters(self) -> None:
        require_package("peft", extra="molmoact2")
+        from peft import get_peft_model

        peft_config = self._build_inner_lora_config()
        self._validate_peft_config(peft_config)
@@ -36,7 +36,6 @@ import torch
 from torch import Tensor

 from lerobot.configs import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    AddBatchDimensionProcessorStep,
    DeviceProcessorStep,
@@ -50,6 +49,7 @@ from lerobot.processor import (
    policy_action_to_transition,
    transition_to_policy_action,
 )
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import (
    ACTION,
    OBS_IMAGES,
@@ -749,9 +749,6 @@ class PI0Policy(PreTrainedPolicy):
    config_class = PI0Config
    name = "pi0"

-    def supports_rtc(self) -> bool:
-        return True
-
    def __init__(
        self,
        config: PI0Config,
@@ -714,9 +714,6 @@ class PI05Policy(PreTrainedPolicy):
    config_class = PI05Config
    name = "pi05"

-    def supports_rtc(self) -> bool:
-        return True
-
    def __init__(
        self,
        config: PI05Config,
@@ -22,7 +22,6 @@ import numpy as np
 import torch

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    AbsoluteActionsProcessorStep,
    PolicyAction,
@@ -34,6 +33,7 @@ from lerobot.processor import (
    make_default_policy_processor_steps,
    make_policy_processor_pipelines,
 )
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import OBS_STATE

 from .configuration_pi05 import PI05Config
@@ -22,7 +22,6 @@ import numpy as np
 import torch

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    AbsoluteActionsProcessorStep,
    ActionTokenizerProcessorStep,
@@ -35,6 +34,7 @@ from lerobot.processor import (
    make_default_policy_processor_steps,
    make_policy_processor_pipelines,
 )
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import OBS_STATE

 from .configuration_pi0_fast import PI0FastConfig
@@ -34,22 +34,14 @@ from lerobot.configs import PreTrainedConfig
 from lerobot.configs.train import TrainPipelineConfig
 from lerobot.utils.device_utils import resolve_safetensors_device
 from lerobot.utils.hub import HubMixin
-from lerobot.utils.import_utils import _peft_available, require_package

 from .utils import log_model_loading_keys

-if TYPE_CHECKING or _peft_available:
-    from peft import PEFT_TYPE_TO_CONFIG_MAPPING, PeftType, get_peft_model
-else:
-    PEFT_TYPE_TO_CONFIG_MAPPING = None
-    PeftType = None
-    get_peft_model = None
+T = TypeVar("T", bound="PreTrainedPolicy")

 if TYPE_CHECKING:
    from lerobot.datasets.dataset_metadata import LeRobotDatasetMetadata

-T = TypeVar("T", bound="PreTrainedPolicy")
-

 def _build_card_context(
    cfg: TrainPipelineConfig | None,
@@ -249,10 +241,6 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):
        """
        raise NotImplementedError

-    def supports_rtc(self) -> bool:
-        """Whether this policy implements Real-Time Chunking inference semantics."""
-        return False
-
    # TODO(aliberts, rcadene): split into 'forward' and 'compute_loss'?
    @abc.abstractmethod
    def forward(self, batch: dict[str, Tensor]) -> tuple[Tensor, dict | None]:
@@ -396,7 +384,7 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):
            peft_cli_overrides: Optional dict of CLI overrides (method_type, target_modules, r, etc.)
                These are merged with policy defaults to build the final config.
        """
-        require_package("peft", extra="peft")
+        from peft import get_peft_model

        # If user provided a complete config, use it directly (with overrides)
        if peft_config is not None:
@@ -467,7 +455,7 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):
        Returns:
            Preprocessed dict with renamed keys and init_type mapped to method-specific key.
        """
-        require_package("peft", extra="peft")
+        from peft import PeftType

        cli_overrides = cli_overrides.copy()

@@ -492,7 +480,7 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):

    def _build_peft_config(self, cli_overrides: dict):
        """Build a PEFT config from policy defaults and CLI overrides."""
-        require_package("peft", extra="peft")
+        from peft import PEFT_TYPE_TO_CONFIG_MAPPING, PeftType

        # Determine PEFT method type (default to LORA)
        method_type_str = cli_overrides.get("method_type") or "lora"
@@ -519,7 +507,7 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC):

    def _apply_peft_cli_overrides(self, peft_config, cli_overrides: dict):
        """Apply CLI overrides to an existing PEFT config."""
-        require_package("peft", extra="peft")
+        from peft import PEFT_TYPE_TO_CONFIG_MAPPING, PeftType

        # Get method type from existing config or CLI override
        method_type_str = cli_overrides.get("method_type")
@@ -145,9 +145,6 @@ class SmolVLAPolicy(PreTrainedPolicy):
    config_class = SmolVLAConfig
    name = "smolvla"

-    def supports_rtc(self) -> bool:
-        return True
-
    def __init__(
        self,
        config: SmolVLAConfig,
@@ -168,23 +168,14 @@ class SmolVLMWithExpertModel(nn.Module):
                last_layers.append(self.num_vlm_layers - 2)
            frozen_layers = [
                "lm_head",
-                "text_model.norm.weight",
+                "text_model.model.norm.weight",
            ]
            for layer in last_layers:
-                frozen_layers.append(f"text_model.layers.{layer}.")
+                frozen_layers.append(f"text_model.model.layers.{layer}.")

-            unmatched_patterns = set(frozen_layers)
            for name, params in self.vlm.named_parameters():
-                matched_patterns = [k for k in frozen_layers if k in name]
-                if matched_patterns:
+                if any(k in name for k in frozen_layers):
                    params.requires_grad = False
-                    unmatched_patterns.difference_update(matched_patterns)
-            if unmatched_patterns:
-                raise RuntimeError(
-                    "Some frozen layer patterns matched no VLM parameters, so the corresponding layers "
-                    "would silently remain trainable (parameter naming may have changed in transformers): "
-                    f"{sorted(unmatched_patterns)}"
-                )
        # To avoid unused params issue with distributed training
        for name, params in self.lm_expert.named_parameters():
            if "lm_head" in name:
@@ -22,7 +22,7 @@ import torch
 from torch import nn

 from lerobot.configs import FeatureType, PolicyFeature, PreTrainedConfig
-from lerobot.lerobot_types import PolicyAction, RobotAction, RobotObservation
+from lerobot.types import PolicyAction, RobotAction, RobotObservation
 from lerobot.utils.constants import ACTION, OBS_STR
 from lerobot.utils.feature_utils import build_dataset_frame

@@ -15,13 +15,11 @@
 from __future__ import annotations

 from dataclasses import dataclass, field
-from typing import Any

 from lerobot.configs.policies import PreTrainedConfig
-from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature
+from lerobot.configs.types import NormalizationMode
 from lerobot.optim.optimizers import AdamWConfig
 from lerobot.optim.schedulers import CosineDecayWithWarmupSchedulerConfig
-from lerobot.utils.constants import OBS_STATE


@PreTrainedConfig.register_subclass("vla_jepa")
@@ -124,13 +122,6 @@ class VLAJEPAConfig(PreTrainedConfig):
        if self.robot_state_feature is not None:
            self.state_dim = self.robot_state_feature.shape[0]

-    def set_dataset_feature_metadata(self, dataset_features: dict[str, Any]) -> None:
-        """Add `observation.state` to `input_features` if missing, so it gets normalized."""
-        if OBS_STATE in self.input_features or OBS_STATE not in dataset_features:
-            return
-        shape = tuple(dataset_features[OBS_STATE]["shape"])
-        self.input_features[OBS_STATE] = PolicyFeature(type=FeatureType.STATE, shape=shape)
-
    def get_optimizer_preset(self) -> AdamWConfig:
        return AdamWConfig(
            lr=self.optimizer_lr,
@@ -399,8 +399,7 @@ class VLAJEPAPolicy(PreTrainedPolicy):
        state = batch.get(OBS_STATE)
        if state is not None:
            if state.ndim > 2:
-                # deltas are forward-looking here, so index 0 is the current observation, not -1.
-                state = state[:, 0, :]
+                state = state[:, -1, :]
            inputs["state"] = (state.unsqueeze(1) if state.ndim == 2 else state).float()  # [B, 1, dim]

        return inputs
@@ -150,7 +150,7 @@ class XVLAModel(nn.Module):
        # Freeze or unfreeze policy transformer
        if not self.config.train_policy_transformer:
            for name, param in self.transformer.named_parameters():
-                if "soft_prompt" not in name:
+                if "soft_prompts" not in name:
                    param.requires_grad = False

        # Freeze or unfreeze soft prompts
@@ -21,7 +21,6 @@ import numpy as np
 import torch

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    ObservationProcessorStep,
    PolicyAction,
@@ -32,6 +31,7 @@ from lerobot.processor import (
    make_default_policy_processor_steps,
    make_policy_processor_pipelines,
 )
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import (
    IMAGENET_STATS,
    OBS_IMAGES,
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-from lerobot.lerobot_types import (
+from lerobot.types import (
    EnvAction,
    EnvTransition,
    PolicyAction,
@@ -25,7 +25,7 @@ from dataclasses import dataclass, field
 from torch import Tensor

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, PolicyAction
+from lerobot.types import EnvTransition, PolicyAction
 from lerobot.utils.constants import OBS_ENV_STATE, OBS_IMAGE, OBS_IMAGES, OBS_STATE

 from .pipeline import (
@@ -175,9 +175,6 @@ class AddBatchDimensionComplementaryDataStep(ComplementaryDataProcessorStep):
            if isinstance(task_index_value, Tensor) and task_index_value.dim() == 0:
                complementary_data["task_index"] = task_index_value.unsqueeze(0)

-        complementary_data.pop("language_persistent", None)
-        complementary_data.pop("language_events", None)
-
        if "messages" in complementary_data:
            messages = complementary_data["messages"]
            if isinstance(messages, list) and (not messages or isinstance(messages[0], dict)):
@@ -23,7 +23,7 @@ from typing import Any
 import numpy as np
 import torch

-from lerobot.lerobot_types import EnvTransition, PolicyAction, RobotAction, RobotObservation, TransitionKey
+from lerobot.types import EnvTransition, PolicyAction, RobotAction, RobotObservation, TransitionKey
 from lerobot.utils.constants import ACTION, DONE, INFO, OBS_PREFIX, REWARD, TRUNCATED


@@ -17,7 +17,7 @@
 from dataclasses import dataclass

 from lerobot.configs import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import PolicyAction, RobotAction
+from lerobot.types import PolicyAction, RobotAction

 from .pipeline import ActionProcessorStep, ProcessorStepRegistry, RobotActionProcessorStep

@@ -25,7 +25,7 @@ from typing import Any
 import torch

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, PolicyAction, TransitionKey
+from lerobot.types import EnvTransition, PolicyAction, TransitionKey
 from lerobot.utils.device_utils import get_safe_torch_device

 from .pipeline import ProcessorStep, ProcessorStepRegistry
@@ -20,7 +20,7 @@ from typing import Any
 import torch

 from lerobot.configs.policies import PreTrainedConfig
-from lerobot.lerobot_types import PolicyAction, RobotAction, RobotObservation
+from lerobot.types import PolicyAction, RobotAction, RobotObservation
 from lerobot.utils.constants import POLICY_POSTPROCESSOR_DEFAULT_NAME, POLICY_PREPROCESSOR_DEFAULT_NAME

 from .batch_processor import AddBatchDimensionProcessorStep
@@ -17,7 +17,7 @@
 from dataclasses import dataclass

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvAction, EnvTransition, PolicyAction, TransitionKey
+from lerobot.types import EnvAction, EnvTransition, PolicyAction, TransitionKey

 from .converters import to_tensor
 from .hil_processor import TELEOP_ACTION_KEY
@@ -29,7 +29,7 @@ from lerobot.teleoperators.utils import TeleopEvents
 if TYPE_CHECKING:
    from lerobot.teleoperators.teleoperator import Teleoperator

-from lerobot.lerobot_types import EnvTransition, PolicyAction, TransitionKey
+from lerobot.types import EnvTransition, PolicyAction, TransitionKey

 from .pipeline import (
    ComplementaryDataProcessorStep,
@@ -25,7 +25,7 @@ import torch
 from torch import Tensor

 from lerobot.configs import FeatureType, NormalizationMode, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, PolicyAction, TransitionKey
+from lerobot.types import EnvTransition, PolicyAction, TransitionKey

 if TYPE_CHECKING:
    from lerobot.datasets import LeRobotDataset
@@ -41,18 +41,11 @@ from pathlib import Path
 from typing import Any, TypedDict, TypeVar, cast

 import torch
-from huggingface_hub import hf_hub_download
+from huggingface_hub import hf_hub_download, snapshot_download
 from safetensors.torch import load_file, save_file

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import (
-    EnvAction,
-    EnvTransition,
-    PolicyAction,
-    RobotAction,
-    RobotObservation,
-    TransitionKey,
-)
+from lerobot.types import EnvAction, EnvTransition, PolicyAction, RobotAction, RobotObservation, TransitionKey
 from lerobot.utils.constants import HF_LEROBOT_HOME
 from lerobot.utils.hub import HubMixin

@@ -212,6 +205,10 @@ class ProcessorStep(ABC):
        """
        return None

+    def save_artifacts(self, save_directory: Path) -> dict[str, str]:
+        """Save non-tensor assets and map constructor arguments to relative paths."""
+        return {}
+
    def reset(self) -> None:
        """Resets the internal state of the processor step, if any."""
        return None
@@ -556,6 +553,22 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):
        pipeline_config = self.get_config()
        pipeline_state_dict = self.state_dict()

+        for processor_step, step_entry in zip(self.steps, pipeline_config["steps"], strict=True):
+            artifacts = processor_step.save_artifacts(save_directory)
+            if artifacts:
+                for config_key, relative_path in artifacts.items():
+                    artifact_path = Path(relative_path)
+                    if artifact_path.is_absolute() or ".." in artifact_path.parts:
+                        raise ValueError(
+                            f"Processor artifact path must be relative to the checkpoint: {relative_path!r}"
+                        )
+                    if not (save_directory / artifact_path).exists():
+                        raise FileNotFoundError(
+                            f"Processor step did not save declared artifact '{relative_path}'"
+                        )
+                    step_entry["config"][config_key] = artifact_path.as_posix()
+                step_entry["artifacts"] = artifacts
+
        for state_key, step_state_dict in pipeline_state_dict.items():
            state_filename = f"{state_key}.safetensors"
            save_file(step_state_dict, save_directory / state_filename)
@@ -740,7 +753,13 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):

        # 3. Build steps with overrides
        steps, validated_overrides = cls._build_steps_with_overrides(
-            loaded_config, overrides or {}, model_id, base_path, hub_download_kwargs, is_local_source
+            loaded_config,
+            overrides or {},
+            model_id,
+            base_path,
+            config_filename,
+            hub_download_kwargs,
+            is_local_source,
        )

        # 4. Validate that all overrides were used
@@ -929,6 +948,7 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):
        overrides: dict[str, Any],
        model_id: str,
        base_path: Path | None,
+        config_filename: str,
        hub_download_kwargs: dict[str, Any],
        is_local_source: bool = False,
    ) -> tuple[list[ProcessorStep], set[str]]:
@@ -983,15 +1003,68 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):
            ImportError: If a step class cannot be imported or found in registry
            ValueError: If a step cannot be instantiated with its configuration
        """
+        loaded_config = deepcopy(loaded_config)
+        cls._resolve_artifact_paths(
+            loaded_config,
+            model_id,
+            base_path,
+            config_filename,
+            hub_download_kwargs,
+        )
        steps, remaining_override_keys = cls._build_steps_from_config(loaded_config, overrides)

        for step_instance, step_entry in zip(steps, loaded_config["steps"], strict=True):
            cls._load_step_state(
-                step_instance, step_entry, model_id, base_path, hub_download_kwargs, is_local_source
+                step_instance,
+                step_entry,
+                model_id,
+                base_path,
+                config_filename,
+                hub_download_kwargs,
+                is_local_source,
            )

        return steps, remaining_override_keys

+    @classmethod
+    def _resolve_artifact_paths(
+        cls,
+        loaded_config: dict[str, Any],
+        model_id: str,
+        base_path: Path | None,
+        config_filename: str,
+        hub_download_kwargs: dict[str, Any],
+    ) -> None:
+        """Resolve declared relative processor artifacts before step construction."""
+        is_local = Path(model_id).is_dir() or Path(model_id).is_file()
+
+        for step_entry in loaded_config["steps"]:
+            artifacts = step_entry.get("artifacts", {})
+            for config_key, relative_path in artifacts.items():
+                artifact_path = Path(relative_path)
+                if artifact_path.is_absolute() or ".." in artifact_path.parts:
+                    raise ValueError(
+                        f"Processor artifact path must be relative to the checkpoint: {relative_path!r}"
+                    )
+
+                resolved_path = base_path / artifact_path if base_path is not None else artifact_path
+                if not resolved_path.exists() and not is_local:
+                    repository_path = Path(config_filename).parent / artifact_path
+                    snapshot_download(
+                        repo_id=model_id,
+                        repo_type="model",
+                        allow_patterns=f"{repository_path.as_posix()}/**",
+                        **hub_download_kwargs,
+                    )
+
+                if not resolved_path.exists():
+                    step_name = step_entry.get("registry_name", step_entry.get("class", "unknown"))
+                    raise FileNotFoundError(
+                        f"Missing processor artifact '{relative_path}' for step '{step_name}' "
+                        f"next to '{config_filename}'. Checkpoint artifacts are incomplete."
+                    )
+                step_entry["config"][config_key] = str(resolved_path)
+
    @classmethod
    def _build_steps_from_config(
        cls,
@@ -1151,6 +1224,7 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):
        step_entry: dict[str, Any],
        model_id: str,
        base_path: Path | None,
+        config_filename: str,
        hub_download_kwargs: dict[str, Any],
        is_local_source: bool = False,
    ) -> None:
@@ -1216,7 +1290,7 @@ class DataProcessorPipeline[TInput, TOutput](HubMixin):
            # Download from Hub
            state_path = hf_hub_download(
                repo_id=model_id,
-                filename=state_filename,
+                filename=(Path(config_filename).parent / state_filename).as_posix(),
                repo_type="model",
                **hub_download_kwargs,
            )
@@ -20,7 +20,7 @@ from typing import Any
 import torch

 from lerobot.configs import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import PolicyAction, RobotAction
+from lerobot.types import PolicyAction, RobotAction
 from lerobot.utils.constants import ACTION

 from .pipeline import ActionProcessorStep, ProcessorStepRegistry
@@ -20,7 +20,7 @@ import torch
 from torch import Tensor

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import OBS_STATE

 from .delta_action_processor import MapDeltaActionToRobotActionStep, MapTensorToDeltaActionDictStep
@@ -16,14 +16,14 @@

 from __future__ import annotations

-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from typing import Any

 from lerobot.configs import PipelineFeatureType, PolicyFeature
 from lerobot.configs.recipe import TrainingRecipe
 from lerobot.datasets.language import LANGUAGE_EVENTS, LANGUAGE_PERSISTENT
 from lerobot.datasets.language_render import render_sample
-from lerobot.lerobot_types import EnvTransition, TransitionKey
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.utils import unwrap_scalar

 from .pipeline import ProcessorStep, ProcessorStepRegistry
@@ -32,17 +32,18 @@ from .pipeline import ProcessorStep, ProcessorStepRegistry
@dataclass
@ProcessorStepRegistry.register(name="render_messages_processor")
 class RenderMessagesStep(ProcessorStep):
-    """Processor step that turns raw language columns into rendered chat messages.
-
-    Reads ``language_persistent`` and ``language_events`` from the transition's
-    complementary data, renders them through ``recipe`` at the sample timestamp,
-    and replaces the raw columns with the resulting ``messages`` /
-    ``message_streams`` / ``target_message_indices`` keys.
-    """
+    """Render language columns into recipe-defined messages and supervision metadata."""

    recipe: TrainingRecipe
    dataset_ctx: Any | None = None

+    def __post_init__(self) -> None:
+        if isinstance(self.recipe, dict):
+            self.recipe = TrainingRecipe.from_dict(self.recipe)
+
+    def get_config(self) -> dict[str, Any]:
+        return {"recipe": asdict(self.recipe)}
+
    def __call__(self, transition: EnvTransition) -> EnvTransition | None:
        """Render messages for a single transition; return ``None`` to drop it."""
        complementary_data = transition.get(TransitionKey.COMPLEMENTARY_DATA) or {}
@@ -50,7 +51,17 @@ class RenderMessagesStep(ProcessorStep):
        events = complementary_data.get(LANGUAGE_EVENTS) or []

        if not persistent and not events:
-            return transition
+            rendered = _fallback_low_level_render(complementary_data.get("task"))
+            if rendered is None:
+                return transition
+            new_transition = transition.copy()
+            new_complementary_data = dict(new_transition.get(TransitionKey.COMPLEMENTARY_DATA) or {})
+            new_complementary_data.update(rendered)
+            new_transition[TransitionKey.COMPLEMENTARY_DATA] = new_complementary_data
+            return new_transition
+
+        if _is_batched_language(persistent) or _is_batched_language(events):
+            return self._call_batch(transition, complementary_data, persistent, events)

        timestamp = complementary_data.get("timestamp")
        if timestamp is None:
@@ -67,18 +78,147 @@ class RenderMessagesStep(ProcessorStep):
            dataset_ctx=self.dataset_ctx,
        )
        if rendered is None:
-            return None
+            rendered = _fallback_low_level_render(complementary_data.get("task"))
+            if rendered is None:
+                return None

        new_transition = transition.copy()
-        new_complementary_data = dict(complementary_data)
+        new_complementary_data = dict(new_transition.get(TransitionKey.COMPLEMENTARY_DATA) or {})
        new_complementary_data.pop(LANGUAGE_PERSISTENT, None)
        new_complementary_data.pop(LANGUAGE_EVENTS, None)
        new_complementary_data.update(rendered)
        new_transition[TransitionKey.COMPLEMENTARY_DATA] = new_complementary_data
        return new_transition

+    def _call_batch(
+        self,
+        transition: EnvTransition,
+        complementary_data: dict[str, Any],
+        persistent_batch: list,
+        events_batch: list,
+    ) -> EnvTransition | None:
+        timestamp = complementary_data.get("timestamp")
+        if timestamp is None:
+            raise KeyError("RenderMessagesStep requires sample timestamp in complementary data.")
+
+        batch_size = max(len(persistent_batch), len(events_batch))
+        messages: list[list[dict[str, Any]]] = []
+        message_streams: list[list[str | None]] = []
+        target_message_indices: list[list[int]] = []
+        keep_indices: list[int] = []
+
+        for i in range(batch_size):
+            rendered = render_sample(
+                recipe=self.recipe,
+                persistent=persistent_batch[i] if i < len(persistent_batch) else [],
+                events=events_batch[i] if i < len(events_batch) else [],
+                t=_batch_value(timestamp, i),
+                sample_idx=int(_batch_value(complementary_data.get("index", 0), i)),
+                task=_batch_value(complementary_data.get("task"), i),
+                dataset_ctx=self.dataset_ctx,
+            )
+            if rendered is None:
+                rendered = _fallback_low_level_render(_batch_value(complementary_data.get("task"), i))
+                if rendered is None:
+                    continue
+            keep_indices.append(i)
+            messages.append(rendered["messages"])
+            message_streams.append(rendered["message_streams"])
+            target_message_indices.append(rendered["target_message_indices"])
+
+        if not messages:
+            return None
+
+        new_transition = (
+            _select_batch_indices(transition, keep_indices)
+            if len(keep_indices) != batch_size
+            else transition.copy()
+        )
+        new_complementary_data = dict(new_transition.get(TransitionKey.COMPLEMENTARY_DATA) or {})
+        new_complementary_data.pop(LANGUAGE_PERSISTENT, None)
+        new_complementary_data.pop(LANGUAGE_EVENTS, None)
+        new_complementary_data["messages"] = messages
+        new_complementary_data["message_streams"] = message_streams
+        new_complementary_data["target_message_indices"] = target_message_indices
+        new_transition[TransitionKey.COMPLEMENTARY_DATA] = new_complementary_data
+        return new_transition
+
    def transform_features(
        self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
    ) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
        """Pass features through unchanged; rendering only touches complementary data."""
        return features
+
+
+def _scalar(value: Any) -> float | int:
+    """Unwrap a tensor/array/single-element list into a Python scalar."""
+    if hasattr(value, "item"):
+        return value.item()
+    if isinstance(value, list):
+        if len(value) != 1:
+            raise ValueError(f"Expected a scalar, got list of length {len(value)}: {value!r}")
+        return _scalar(value[0])
+    return value
+
+
+def _is_batched_language(value: Any) -> bool:
+    return isinstance(value, list) and bool(value) and isinstance(value[0], list)
+
+
+def _batch_value(value: Any, index: int) -> Any:
+    if value is None:
+        return None
+    if isinstance(value, list):
+        return value[index]
+    if hasattr(value, "ndim") and value.ndim > 0:
+        return _scalar(value[index])
+    return _scalar(value)
+
+
+def _select_batch_indices(transition: EnvTransition, indices: list[int]) -> EnvTransition:
+    selected = transition.copy()
+    for key in (TransitionKey.OBSERVATION, TransitionKey.COMPLEMENTARY_DATA):
+        data = selected.get(key)
+        if isinstance(data, dict):
+            selected[key] = {k: _select_value(v, indices) for k, v in data.items()}
+    action = selected.get(TransitionKey.ACTION)
+    if action is not None:
+        selected[TransitionKey.ACTION] = _select_value(action, indices)
+    return selected
+
+
+def _select_value(value: Any, indices: list[int]) -> Any:
+    if isinstance(value, list) and len(value) >= len(indices):
+        return [value[i] for i in indices]
+    if hasattr(value, "index_select") and hasattr(value, "new_tensor") and getattr(value, "ndim", 0) > 0:
+        return value.index_select(0, value.new_tensor(indices).long())
+    return value
+
+
+def _fallback_low_level_render(task: Any) -> dict[str, Any] | None:
+    """Keep action-only samples trainable when no recipe branch matches."""
+    if hasattr(task, "item"):
+        task = task.item()
+    if isinstance(task, list):
+        messages = []
+        message_streams = []
+        target_message_indices = []
+        for t in task:
+            rendered = _fallback_low_level_render(t)
+            if rendered is None:
+                return None
+            messages.append(rendered["messages"])
+            message_streams.append(rendered["message_streams"])
+            target_message_indices.append(rendered["target_message_indices"])
+        return {
+            "messages": messages,
+            "message_streams": message_streams,
+            "target_message_indices": target_message_indices,
+        }
+    if not isinstance(task, str) or not task:
+        return None
+    return {
+        "messages": [{"role": "user", "content": task}],
+        "message_streams": ["low_level"],
+        "target_message_indices": [],
+    }
@@ -25,13 +25,15 @@ from __future__ import annotations

 import logging
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TYPE_CHECKING, Any

 import torch

 from lerobot.configs import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, RobotObservation, TransitionKey
+from lerobot.types import EnvTransition, RobotObservation, TransitionKey
 from lerobot.utils.constants import (
+    ACTION_CODE_TOKEN_MASK,
    ACTION_TOKEN_MASK,
    ACTION_TOKENS,
    OBS_LANGUAGE_ATTENTION_MASK,
@@ -136,7 +138,7 @@ class TokenizerProcessorStep(ObservationProcessorStep):
        # Standardize to a list of strings for the tokenizer
        if isinstance(task, str):
            return [task]
-        elif isinstance(task, (list, tuple)) and all(isinstance(t, str) for t in task):
+        elif isinstance(task, list | tuple) and all(isinstance(t, str) for t in task):
            return list(task)

        return None
@@ -349,6 +351,7 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
    max_action_tokens: int = 256
    fast_skip_tokens: int = 128
    paligemma_tokenizer_name: str = "google/paligemma-3b-pt-224"
+    allow_truncation: bool = True
    # Internal tokenizer instance (not part of the config)
    action_tokenizer: Any = field(default=None, init=False, repr=False)
    _paligemma_tokenizer: Any = field(default=None, init=False, repr=False)
@@ -412,14 +415,15 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
            # During inference, no action is available, skip tokenization
            return new_transition

-        # Tokenize and get both tokens and mask
-        tokens, mask = self._tokenize_action(action)
+        # Tokenize and get masks for the full formatted sequence and the discrete action codes.
+        tokens, mask, code_mask = self._tokenize_action(action)

        # Store mask in complementary data
        complementary_data = new_transition.get(TransitionKey.COMPLEMENTARY_DATA, {})
        if complementary_data is None:
            complementary_data = {}
        complementary_data[ACTION_TOKEN_MASK] = mask
+        complementary_data[ACTION_CODE_TOKEN_MASK] = code_mask
        complementary_data[ACTION_TOKENS] = tokens
        new_transition[TransitionKey.COMPLEMENTARY_DATA] = complementary_data
        return new_transition
@@ -430,7 +434,7 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
        """
        return self._paligemma_tokenizer.vocab_size - 1 - self.fast_skip_tokens - tokens

-    def _tokenize_action(self, action: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+    def _tokenize_action(self, action: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """
        Tokenizes the action tensor and creates a mask.

@@ -459,6 +463,7 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
        # The fast tokenizer expects action data and returns token IDs
        tokens_list = []
        masks_list = []
+        code_masks_list = []

        for i in range(batch_size):
            # Tokenize single action (move to CPU first as tokenizer uses scipy which requires numpy)
@@ -476,65 +481,82 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
            if tokens.dim() > 1:
                tokens = tokens.flatten()

+            action_code_tokens = self._act_tokens_to_paligemma_tokens(tokens)
            bos_id = self._paligemma_tokenizer.bos_token_id
-            # add bos
+            prompt_tokens = torch.tensor(
+                self._paligemma_tokenizer.encode("Action: ", add_special_tokens=False),
+                device=action.device,
+            )
+            end_tokens = torch.tensor(self._paligemma_tokenizer.encode("|"), device=action.device)
+
+            code_start = 1 + len(prompt_tokens)
+            code_end = code_start + len(action_code_tokens)
            tokens = torch.cat(
                [
                    torch.tensor([bos_id], device=action.device),
-                    torch.tensor(
-                        self._paligemma_tokenizer.encode("Action: ", add_special_tokens=False),
-                        device=action.device,
-                    ),
-                    self._act_tokens_to_paligemma_tokens(tokens),
-                    torch.tensor(self._paligemma_tokenizer.encode("|"), device=action.device),
+                    prompt_tokens,
+                    action_code_tokens,
+                    end_tokens,
                ]
            )
+            code_mask = torch.zeros(len(tokens), dtype=torch.bool, device=action.device)
+            code_mask[code_start:code_end] = True

            # Truncate or pad to max_action_tokens
            if len(tokens) > self.max_action_tokens:
+                if not self.allow_truncation:
+                    raise ValueError(
+                        f"FAST action sequence has {len(tokens)} tokens, exceeding "
+                        f"max_action_tokens={self.max_action_tokens}."
+                    )
                logging.warning(
                    f"Token length ({len(tokens)}) exceeds max length ({self.max_action_tokens}), truncating. "
                    "Consider increasing the `max_action_tokens` in your model config if this happens frequently."
                )
                tokens = tokens[: self.max_action_tokens]
+                code_mask = code_mask[: self.max_action_tokens]
                mask = torch.ones(self.max_action_tokens, dtype=torch.bool, device=action.device)
            else:
+                pad_len = self.max_action_tokens - len(tokens)
                mask = torch.cat(
                    [
                        torch.ones(len(tokens), dtype=torch.bool, device=action.device),
-                        torch.zeros(
-                            self.max_action_tokens - len(tokens), dtype=torch.bool, device=action.device
-                        ),
+                        torch.zeros(pad_len, dtype=torch.bool, device=action.device),
                    ]
                )
+                code_mask = torch.nn.functional.pad(code_mask, (0, pad_len), value=False)
                # Pad tokens with zeros
-                tokens = torch.nn.functional.pad(tokens, (0, self.max_action_tokens - len(tokens)), value=0)
+                tokens = torch.nn.functional.pad(tokens, (0, pad_len), value=0)

            tokens_list.append(tokens)
            masks_list.append(mask)
+            code_masks_list.append(code_mask)

        # Stack into batched tensors
        tokens_batch = torch.stack(tokens_list, dim=0)  # (B, max_action_tokens)
        masks_batch = torch.stack(masks_list, dim=0)  # (B, max_action_tokens)
+        code_masks_batch = torch.stack(code_masks_list, dim=0)  # (B, max_action_tokens)

        # Remove batch dimension if input was single sample
        if single_sample:
            tokens_batch = tokens_batch.squeeze(0)
            masks_batch = masks_batch.squeeze(0)
+            code_masks_batch = code_masks_batch.squeeze(0)

        # Move to the same device as the input
        if device is not None:
            tokens_batch = tokens_batch.to(device)
            masks_batch = masks_batch.to(device)
+            code_masks_batch = code_masks_batch.to(device)

-        return tokens_batch, masks_batch
+        return tokens_batch, masks_batch, code_masks_batch

    def action(self, action: torch.Tensor) -> torch.Tensor:
        """
        This method is not used since we override __call__.
        Required by ActionProcessorStep ABC.
        """
-        tokens, _ = self._tokenize_action(action)
+        tokens, _, _ = self._tokenize_action(action)
        return tokens

    def get_config(self) -> dict[str, Any]:
@@ -550,6 +572,9 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):
        config = {
            "trust_remote_code": self.trust_remote_code,
            "max_action_tokens": self.max_action_tokens,
+            "fast_skip_tokens": self.fast_skip_tokens,
+            "paligemma_tokenizer_name": self.paligemma_tokenizer_name,
+            "allow_truncation": self.allow_truncation,
        }

        # Only save tokenizer_name if it was used to create the tokenizer
@@ -558,6 +583,14 @@ class ActionTokenizerProcessorStep(ActionProcessorStep):

        return config

+    def save_artifacts(self, save_directory: Path) -> dict[str, str]:
+        artifact_path = Path("action_tokenizer")
+        save_pretrained = getattr(self.action_tokenizer, "save_pretrained", None)
+        if save_pretrained is None:
+            raise TypeError("Action tokenizer must implement save_pretrained() to save a portable pipeline.")
+        save_pretrained(save_directory / artifact_path)
+        return {"action_tokenizer_name": artifact_path.as_posix()}
+
    def transform_features(
        self, features: dict[PipelineFeatureType, dict[str, PolicyFeature]]
    ) -> dict[PipelineFeatureType, dict[str, PolicyFeature]]:
@@ -57,10 +57,10 @@ import torch
 from tqdm import tqdm

 from lerobot.datasets import LeRobotDataset
-from lerobot.lerobot_types import TransitionKey
 from lerobot.rewards.robometer.configuration_robometer import RobometerConfig
 from lerobot.rewards.robometer.modeling_robometer import RobometerRewardModel
 from lerobot.rewards.robometer.processor_robometer import RobometerEncoderProcessorStep
+from lerobot.types import TransitionKey

 DEFAULT_OUTPUT_FILENAME = "robometer_progress.parquet"

@@ -25,7 +25,6 @@ from PIL import Image
 from torch import Tensor

 from lerobot.configs import PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, TransitionKey
 from lerobot.processor import (
    AddBatchDimensionProcessorStep,
    DeviceProcessorStep,
@@ -40,6 +39,7 @@ from lerobot.rewards.robometer.configuration_robometer import (
    RobometerConfig,
 )
 from lerobot.rewards.robometer.modeling_robometer import ROBOMETER_FEATURE_PREFIX
+from lerobot.types import EnvTransition, TransitionKey
 from lerobot.utils.constants import (
    OBS_IMAGES,
    POLICY_POSTPROCESSOR_DEFAULT_NAME,
@@ -47,7 +47,6 @@ else:
    Faker = None  # type: ignore[assignment, misc]

 from lerobot.configs import FeatureType, PipelineFeatureType, PolicyFeature
-from lerobot.lerobot_types import EnvTransition, PolicyAction, TransitionKey
 from lerobot.processor import (
    AddBatchDimensionProcessorStep,
    DeviceProcessorStep,
@@ -59,6 +58,7 @@ from lerobot.processor import (
    policy_action_to_transition,
    transition_to_policy_action,
 )
+from lerobot.types import EnvTransition, PolicyAction, TransitionKey
 from lerobot.utils.constants import POLICY_POSTPROCESSOR_DEFAULT_NAME, POLICY_PREPROCESSOR_DEFAULT_NAME

 from .configuration_sarm import SARMConfig
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Pepijn	c266fe4f1a	Merge branch 'main' into codex/language-recipes	2026-07-28 12:50:11 +02:00
Pepijn	913664c320	test(collate): expect preserved language columns	2026-07-28 12:39:15 +02:00
Pepijn	3f093d8927	feat(data): add recipe-driven language supervision	2026-07-28 10:04:28 +02:00