mirror of
https://github.com/huggingface/lerobot.git
synced 2026-08-08 17:39:44 +00:00
refactor(g05): keep recipe runtime unchanged
This commit is contained in:
+3
-4
@@ -149,10 +149,9 @@ lerobot-train \
|
||||
|
||||
The bundled `g05_bbox_subtask.yaml` recipe resolves the active
|
||||
`language_persistent` `subtask` and camera-scoped grounded `vqa` event at each
|
||||
sample timestamp. It applies independent `0.5` dropout to the optional BBox and
|
||||
Subtask targets, producing Action-only, Subtask+Action, BBox+Action, and joint
|
||||
BBox+Subtask+Action samples. An unavailable annotation is skipped before
|
||||
dropout through the recipe's `if_present` guard.
|
||||
sample timestamp. It emits each optional BBox or Subtask target when that
|
||||
annotation is present; unavailable annotations are skipped through the recipe's
|
||||
existing `if_present` guard.
|
||||
|
||||
Grounded VQA boxes are converted from pixel-space `xyxy` JSON using the source
|
||||
camera dimensions captured before image resizing, then serialized as G0.5
|
||||
|
||||
@@ -146,24 +146,6 @@ The renderer does not apply a tokenizer chat template. Policy processors decide
|
||||
Blend recipes select one weighted sub-recipe deterministically from the sample index.
|
||||
`recipes/subtask_mem.yaml` trains the compact core blend — high-level subtask prediction, low-level execution, and memory. `recipes/subtask_mem_vqa_speech.yaml` is the fuller variant that also adds VQA and spoken interjection responses.
|
||||
|
||||
Message turns can set `dropout` to a probability between `0` and `1`. Dropout is
|
||||
deterministic for a given sample index and independent between turns. Combine it
|
||||
with `if_present` to sample optional supervision formats while gracefully
|
||||
skipping annotations that are unavailable:
|
||||
|
||||
```yaml
|
||||
messages:
|
||||
- { role: user, content: "${task}", stream: low_level }
|
||||
- {
|
||||
role: assistant,
|
||||
content: "${subtask}",
|
||||
stream: low_level,
|
||||
target: true,
|
||||
if_present: subtask,
|
||||
dropout: 0.5,
|
||||
}
|
||||
```
|
||||
|
||||
A message recipe with a supervised assistant turn on the `low_level` stream trains
|
||||
the π0.5 paper's joint sequence instead of a blend: the target span gets text CE
|
||||
while also conditioning the action losses in the same forward.
|
||||
|
||||
@@ -18,7 +18,6 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, get_args
|
||||
|
||||
@@ -50,9 +49,8 @@ class MessageTurn:
|
||||
``content`` may be a plain string, a list of HF-style multimodal blocks, or
|
||||
``None`` when ``tool_calls_from`` supplies tool-call payloads instead.
|
||||
``stream`` tags the turn for downstream filtering, ``target`` flags it as a
|
||||
training target, ``if_present`` skips the turn when the named binding
|
||||
resolves to ``None``, and ``dropout`` independently omits it for a
|
||||
deterministic fraction of samples.
|
||||
training target, and ``if_present`` skips the turn when the named binding
|
||||
resolves to ``None``.
|
||||
"""
|
||||
|
||||
role: MessageRole
|
||||
@@ -61,7 +59,6 @@ class MessageTurn:
|
||||
target: bool = False
|
||||
if_present: str | None = None
|
||||
tool_calls_from: str | None = None
|
||||
dropout: float = 0.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate role, stream, and content after dataclass construction."""
|
||||
@@ -89,11 +86,6 @@ class MessageTurn:
|
||||
raise ValueError(
|
||||
"Multimodal content blocks must be HF-style dictionaries with a type key."
|
||||
)
|
||||
if isinstance(self.dropout, bool) or not isinstance(self.dropout, int | float):
|
||||
raise TypeError("MessageTurn.dropout must be a probability.")
|
||||
self.dropout = float(self.dropout)
|
||||
if not isfinite(self.dropout) or not 0.0 <= self.dropout <= 1.0:
|
||||
raise ValueError("MessageTurn.dropout must be between 0 and 1.")
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> MessageTurn:
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
# G0.5 CoT supervision from the signals LeRobot can currently provide:
|
||||
# active subtasks and camera-grounded bounding boxes.
|
||||
#
|
||||
# BBox and Subtask turns are dropped independently and deterministically. This
|
||||
# produces Action-only, BBox+Action, Subtask+Action, and
|
||||
# BBox+Subtask+Action samples without a policy-specific recipe selector.
|
||||
# Each target is emitted when its corresponding annotation is present.
|
||||
|
||||
bindings:
|
||||
bbox: "emitted_at(t, style=vqa, role=assistant, camera=observation.images.exterior)"
|
||||
@@ -16,7 +13,6 @@ messages:
|
||||
stream: low_level,
|
||||
target: true,
|
||||
if_present: bbox,
|
||||
dropout: 0.5,
|
||||
}
|
||||
- {
|
||||
role: assistant,
|
||||
@@ -24,5 +20,4 @@ messages:
|
||||
stream: low_level,
|
||||
target: true,
|
||||
if_present: subtask,
|
||||
dropout: 0.5,
|
||||
}
|
||||
|
||||
@@ -194,7 +194,7 @@ def render_sample(
|
||||
task=task,
|
||||
dataset_ctx=dataset_ctx,
|
||||
)
|
||||
return _render_message_recipe(selected_recipe, bindings, sample_idx=sample_idx)
|
||||
return _render_message_recipe(selected_recipe, bindings)
|
||||
|
||||
|
||||
def _render_vqa_if_present(
|
||||
@@ -225,7 +225,7 @@ def _render_vqa_if_present(
|
||||
task=task,
|
||||
dataset_ctx=dataset_ctx,
|
||||
)
|
||||
rendered = _render_message_recipe(component, bindings, sample_idx=sample_idx)
|
||||
rendered = _render_message_recipe(component, bindings)
|
||||
if rendered is not None:
|
||||
renderable.append((float(component.weight or 0.0), rendered))
|
||||
|
||||
@@ -382,8 +382,6 @@ def _parse_resolver_args(args: str) -> dict[str, Any]:
|
||||
def _render_message_recipe(
|
||||
recipe: TrainingRecipe,
|
||||
bindings: dict[str, LanguageRow | str | None],
|
||||
*,
|
||||
sample_idx: int,
|
||||
) -> RenderedMessages | None:
|
||||
"""Expand ``recipe.messages`` into rendered chat messages using ``bindings``."""
|
||||
assert recipe.messages is not None
|
||||
@@ -393,11 +391,9 @@ def _render_message_recipe(
|
||||
streams: list[str | None] = []
|
||||
target_indices: list[int] = []
|
||||
|
||||
for turn_idx, turn in enumerate(recipe.messages):
|
||||
for turn in recipe.messages:
|
||||
if turn.if_present is not None and bindings.get(turn.if_present) is None:
|
||||
continue
|
||||
if _drop_message_turn(turn.dropout, sample_idx=sample_idx, turn_idx=turn_idx):
|
||||
continue
|
||||
|
||||
message = {"role": turn.role}
|
||||
if turn.content is not None:
|
||||
@@ -429,21 +425,6 @@ def _render_message_recipe(
|
||||
return rendered
|
||||
|
||||
|
||||
def _drop_message_turn(dropout: float, *, sample_idx: int, turn_idx: int) -> bool:
|
||||
"""Apply reproducible, independent dropout to one recipe message."""
|
||||
|
||||
if dropout <= 0.0:
|
||||
return False
|
||||
if dropout >= 1.0:
|
||||
return True
|
||||
digest = hashlib.blake2b(
|
||||
f"message_dropout:{sample_idx}:{turn_idx}".encode(),
|
||||
digest_size=8,
|
||||
).digest()
|
||||
draw = int.from_bytes(digest, "big") / 2**64
|
||||
return draw < dropout
|
||||
|
||||
|
||||
def _render_content(
|
||||
content: str | list[dict[str, Any]],
|
||||
bindings: dict[str, LanguageRow | str | None],
|
||||
|
||||
@@ -47,17 +47,6 @@ def test_message_turn_requires_a_stream():
|
||||
MessageTurn(role="user", content="${task}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dropout", [-0.1, 1.1, float("inf"), float("nan")])
|
||||
def test_message_turn_rejects_invalid_dropout(dropout):
|
||||
with pytest.raises(ValueError, match="between 0 and 1"):
|
||||
MessageTurn(role="user", content="${task}", stream="high_level", dropout=dropout)
|
||||
|
||||
|
||||
def test_message_turn_rejects_non_numeric_dropout():
|
||||
with pytest.raises(TypeError, match="probability"):
|
||||
MessageTurn(role="user", content="${task}", stream="high_level", dropout="half")
|
||||
|
||||
|
||||
def test_message_recipe_requires_at_least_one_target():
|
||||
with pytest.raises(ValueError, match="target"):
|
||||
TrainingRecipe(
|
||||
@@ -163,7 +152,7 @@ def test_from_dict_with_nested_blend():
|
||||
assert isinstance(recipe.blend["a"].messages[0], MessageTurn)
|
||||
|
||||
|
||||
def test_message_dropout_round_trips_from_dict():
|
||||
def test_message_if_present_round_trips_from_dict():
|
||||
recipe = TrainingRecipe.from_dict(
|
||||
{
|
||||
"messages": [
|
||||
@@ -174,13 +163,11 @@ def test_message_dropout_round_trips_from_dict():
|
||||
"stream": "low_level",
|
||||
"target": True,
|
||||
"if_present": "subtask",
|
||||
"dropout": 0.5,
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert recipe.messages[1].dropout == 0.5
|
||||
assert recipe.messages[1].if_present == "subtask"
|
||||
|
||||
|
||||
|
||||
@@ -176,35 +176,7 @@ def test_deterministic_blend_sampling():
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_message_dropout_skips_missing_optional_bindings():
|
||||
recipe = TrainingRecipe(
|
||||
messages=[
|
||||
MessageTurn(role="user", content="${task}", stream="low_level"),
|
||||
MessageTurn(
|
||||
role="assistant",
|
||||
content="${subtask}",
|
||||
stream="low_level",
|
||||
target=True,
|
||||
if_present="subtask",
|
||||
dropout=0.5,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
rendered = render_sample(
|
||||
recipe=recipe,
|
||||
persistent=[],
|
||||
events=[],
|
||||
t=0.0,
|
||||
sample_idx=0,
|
||||
task="pick the cup",
|
||||
)
|
||||
|
||||
assert rendered["messages"] == [{"role": "user", "content": "pick the cup"}]
|
||||
assert rendered["target_message_indices"] == []
|
||||
|
||||
|
||||
def test_g05_dropout_produces_all_bbox_subtask_combinations():
|
||||
def test_g05_recipe_emits_available_bbox_and_subtask():
|
||||
recipe = TrainingRecipe.from_yaml("src/lerobot/configs/recipes/g05_bbox_subtask.yaml")
|
||||
persistent = [persistent_row("assistant", "grasp the cup", "subtask", 0.0)]
|
||||
events = [
|
||||
@@ -216,64 +188,18 @@ def test_g05_dropout_produces_all_bbox_subtask_combinations():
|
||||
}
|
||||
]
|
||||
|
||||
rendered_by_targets = {}
|
||||
for sample_idx in range(100):
|
||||
rendered = render_sample(
|
||||
recipe=recipe,
|
||||
persistent=persistent,
|
||||
events=events,
|
||||
t=0.0,
|
||||
sample_idx=sample_idx,
|
||||
sample_idx=0,
|
||||
task="pick the cup",
|
||||
)
|
||||
targets = tuple(
|
||||
rendered["messages"][idx]["content"].split(":", 1)[0]
|
||||
for idx in rendered["target_message_indices"]
|
||||
)
|
||||
rendered_by_targets.setdefault(targets, rendered)
|
||||
|
||||
assert set(rendered_by_targets) == {
|
||||
(),
|
||||
("BBoxJSON",),
|
||||
("Subtask",),
|
||||
("BBoxJSON", "Subtask"),
|
||||
}
|
||||
joint = rendered_by_targets[("BBoxJSON", "Subtask")]
|
||||
assert joint["messages"][1]["content"].startswith("BBoxJSON:")
|
||||
assert joint["messages"][2]["content"] == "Subtask: grasp the cup"
|
||||
|
||||
|
||||
def test_message_dropout_is_deterministic_for_sample_index():
|
||||
recipe = TrainingRecipe(
|
||||
messages=[
|
||||
MessageTurn(role="user", content="${task}", stream="low_level"),
|
||||
MessageTurn(
|
||||
role="assistant",
|
||||
content="${subtask}",
|
||||
stream="low_level",
|
||||
target=True,
|
||||
dropout=0.5,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
first = render_sample(
|
||||
recipe=recipe,
|
||||
persistent=PERSISTENT,
|
||||
events=[],
|
||||
t=0.0,
|
||||
sample_idx=42,
|
||||
task="pick the cup",
|
||||
)
|
||||
second = render_sample(
|
||||
recipe=recipe,
|
||||
persistent=PERSISTENT,
|
||||
events=[],
|
||||
t=0.0,
|
||||
sample_idx=42,
|
||||
task="pick the cup",
|
||||
)
|
||||
assert first == second
|
||||
assert rendered["target_message_indices"] == [1, 2]
|
||||
assert rendered["messages"][1]["content"].startswith("BBoxJSON:")
|
||||
assert rendered["messages"][2]["content"] == "Subtask: grasp the cup"
|
||||
|
||||
|
||||
def test_emitted_at_filters_vqa_by_camera():
|
||||
|
||||
Reference in New Issue
Block a user