Merge branch 'feat/language-columns' into feat/language-annotation-pipeline

Resolve conflicts and pull in the latest PR 1 fixes.

Conflicts:
- pyproject.toml: PR 1 added `lerobot-rollout` and PR 2 added
  `lerobot-annotate` to the same `[project.scripts]` block. Kept both.
- uv.lock: dropped both sides and regenerated against the merged
  `pyproject.toml` (PR 2 dropped the `datatrove` dep when distribution
  moved to HF Jobs; PR 1's lock didn't have it).

Test follow-up:
- `tests/annotations/test_pipeline_recipe_render.py` — PR 1 deleted
  `src/lerobot/configs/recipes/pi05_hirobot.yaml` (review feedback:
  remove the canonical-recipe file; recipes are user-supplied). The
  cross-PR contract this test guards is "the recipe DSL renders
  non-empty messages from pipeline output", which doesn't depend on
  any specific YAML, so the test now builds an inline blend recipe
  with the same coverage. Passes.

Sweep: 82 passed, 2 failed (pre-existing module-impl bugs:
`test_module1_attaches_video_block_to_subtask_prompt`,
`test_module2_mid_episode_emits_paired_interjection_and_speech`).
The PR 1 carryover (`test_emitted_at_raises_on_ambiguous_per_camera_vqa`)
is now passing — the merge brought in PR 1's tightened `_select_one`
ambiguity check.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Pepijn
2026-05-08 11:13:11 +02:00
179 changed files with 11667 additions and 4571 deletions
@@ -35,14 +35,60 @@ from lerobot.annotations.steerable_pipeline.modules import (
)
from lerobot.annotations.steerable_pipeline.validator import StagingValidator
from lerobot.annotations.steerable_pipeline.writer import LanguageColumnsWriter
from lerobot.configs.recipe import TrainingRecipe
from lerobot.configs.recipe import MessageTurn, TrainingRecipe
from lerobot.datasets.language_render import render_sample
from ._helpers import make_canned_responder
_RECIPE_PATH = (
Path(__file__).resolve().parents[2] / "src" / "lerobot" / "configs" / "recipes" / "pi05_hirobot.yaml"
)
def _build_pr1_style_blend_recipe() -> TrainingRecipe:
"""Inline blend recipe that consumes every style this pipeline produces.
PR 1 used to ship ``src/lerobot/configs/recipes/pi05_hirobot.yaml`` as
a canonical example, but that file was dropped during PR 1 review. The
cross-PR contract this test guards is "the recipe DSL can render
non-empty messages from pipeline output", which doesn't require a
specific YAML — so we build the equivalent blend in code.
"""
return TrainingRecipe(
blend={
"low_level_execution": TrainingRecipe(
weight=0.35,
messages=[
MessageTurn(
role="user",
content="${task}\nPlan: ${plan}\nMemory: ${memory}",
stream="high_level",
),
MessageTurn(role="assistant", content="${subtask}", stream="low_level", target=True),
],
),
"user_interjection_response": TrainingRecipe(
weight=0.16,
bindings={
"speech": "emitted_at(t, role=assistant, tool_name=say)",
"interjection": "emitted_at(t, style=interjection)",
},
messages=[
MessageTurn(role="user", content="${task}", stream="high_level"),
MessageTurn(
role="user",
content="${interjection}",
stream="high_level",
if_present="interjection",
),
MessageTurn(
role="assistant",
content="${plan}",
stream="high_level",
target=True,
if_present="plan",
tool_calls_from="speech",
),
],
),
}
)
def _build_executor() -> Executor:
@@ -96,13 +142,7 @@ def test_pr1_canonical_recipe_renders_nonempty_from_pipeline_output(
events_lists = table.column("language_events").to_pylist()
timestamps = table.column("timestamp").to_pylist()
recipe = TrainingRecipe.from_yaml(_RECIPE_PATH) if hasattr(TrainingRecipe, "from_yaml") else None
if recipe is None:
# PR 1 may not expose from_yaml; load via PyYAML and TrainingRecipe(**...)
import yaml
loaded = yaml.safe_load(_RECIPE_PATH.read_text(encoding="utf-8"))
recipe = TrainingRecipe(**loaded)
recipe = _build_pr1_style_blend_recipe()
rendered_any = False
for ts, persistent, events in zip(timestamps, persistent_lists, events_lists, strict=True):