diff --git a/src/lerobot/configs/recipes/subtask.yaml b/src/lerobot/configs/recipes/subtask.yaml index b84a03965..c90ca8f78 100644 --- a/src/lerobot/configs/recipes/subtask.yaml +++ b/src/lerobot/configs/recipes/subtask.yaml @@ -1,15 +1,5 @@ -# subtask — subtask + action blend, no memory. -# -# Same as subtask_mem but with the memory_update sub-recipe dropped, so -# the model only learns: -# -# high_level_subtask — predict the subtask from the task. -# low_level_execution — flow loss with [images, subtask, state]. -# -# No persistent high-level state (no memory note, no plan): the prompt is -# just the task / subtask. Requires the dataset to carry `subtask` -# annotations; samples whose `if_present` bindings are missing don't -# render. +# Predicts subtasks from tasks and trains subtask-conditioned action flow without memory or plans. +# Requires `subtask` annotations; samples with missing `if_present` bindings do not render. blend: @@ -22,9 +12,5 @@ blend: low_level_execution: weight: 0.70 messages: - # The action expert is conditioned on the SUBTASK — at inference - # `HighLevelSubtaskFwd` generates it via the LM head and feeds it - # here. `stream: low_level` flips `predict_actions=True` so the - # flow loss fires; no text-CE target (subtask prediction is owned - # by `high_level_subtask`). + # The low-level stream trains action flow on the generated or annotated subtask. - {role: user, content: "${subtask}", stream: low_level, if_present: subtask} diff --git a/src/lerobot/configs/recipes/subtask_mem.yaml b/src/lerobot/configs/recipes/subtask_mem.yaml index e47d2c9f5..d12fe5009 100644 --- a/src/lerobot/configs/recipes/subtask_mem.yaml +++ b/src/lerobot/configs/recipes/subtask_mem.yaml @@ -1,17 +1,5 @@ -# subtask_mem — compact Hi-Robot blend with memory. -# -# Trains the core subtask + action objectives and memory updates: -# -# high_level_subtask — predict the subtask from the task. -# low_level_execution — flow loss with [images, subtask, state]. -# memory_update — compress progress into a memory note. -# -# Plan is intentionally left out — memory is the only persistent -# high-level state here, keeping the prompt short. -# -# Requires the dataset to carry `subtask` and `memory` annotations. -# Sub-recipes whose `if_present` bindings are missing simply don't -# render for that sample. +# Trains subtask prediction, subtask-conditioned action flow, and memory updates without plans. +# Requires `subtask` and `memory`; missing `if_present` bindings skip the affected sub-recipe. blend: @@ -24,26 +12,12 @@ blend: low_level_execution: weight: 0.60 messages: - # The action expert is conditioned on the SUBTASK — at inference - # `HighLevelSubtaskFwd` generates it via the LM head and feeds it - # here. `stream: low_level` flips `predict_actions=True` so the - # flow loss fires; no text-CE target (subtask prediction is owned - # by `high_level_subtask`). + # The low-level stream trains action flow on the generated or annotated subtask. - {role: user, content: "${subtask}", stream: low_level, if_present: subtask} memory_update: - # At inference, `MemoryUpdateFwd` is triggered only on - # `subtask_change` events (sparse). Training densely with - # `active_at` — i.e. on every frame inside a subtask interval, - # not just the boundary frame — supervises the same - # (prior_memory, completed_subtask) → current_memory mapping - # against varied observations within the interval. The model - # learns a stateless transformation; the *when* to emit lives in - # the inference trigger, not the model. Annotations only exist - # for ~1% of frames as boundary events, so `emitted_at` would - # waste 99% of the blend draws (and silently leak them into a - # task-conditioned fallback); `active_at` lifts the renderable - # rate to ~87% on this dataset. + # `active_at` densifies sparse boundaries while preserving the prior-memory/subtask mapping. + # Inference controls update timing through `subtask_change` events. weight: 0.15 bindings: prior_memory: "nth_prev(style=memory, offset=1)" diff --git a/src/lerobot/configs/recipes/subtask_mem_vqa_speech.yaml b/src/lerobot/configs/recipes/subtask_mem_vqa_speech.yaml index b6424bdf5..1a3d39952 100644 --- a/src/lerobot/configs/recipes/subtask_mem_vqa_speech.yaml +++ b/src/lerobot/configs/recipes/subtask_mem_vqa_speech.yaml @@ -1,28 +1,5 @@ -# subtask_mem_vqa_speech — Hi-Robot blend + memory + spoken responses. -# -# Extends the compact subtask_mem recipe with VQA and spoken interjection responses: -# -# high_level_subtask — predict the subtask from the task. -# low_level_execution — flow loss with [images, subtask, state]. -# memory_update — compress progress into a memory note. -# user_interjection_response — reply to a user interjection with a -# spoken `say` tool call (no plan, no -# subtask text — just the spoken reply). -# ask_vqa_{top,wrist} — camera-grounded VQA. -# -# Plan is intentionally left out — memory is the only persistent -# high-level state here, keeping the prompt short. -# -# Requires the dataset to carry `memory`, `interjection` and `say`-tool -# annotations (the annotation pipeline's memory + interjection modules) -# in addition to `subtask` and `vqa`. Sub-recipes whose `if_present` -# bindings are missing simply don't render for that sample, so a -# dataset without interjections still trains the rest of the blend. -# -# Tool-call note: the `say` tool call on the interjection-response turn -# is flattened to a `...` text marker by the tokenizer step -# (`_flatten_say_tool_calls`) so the LM head learns to emit exactly the -# marker the runtime parses back (`_split_plan_and_say`). +# Adds memory, spoken interjection responses, and camera-grounded VQA to subtask/action training. +# Missing optional annotations skip only their sub-recipe; `say` tool calls tokenize as `...`. blend: @@ -35,26 +12,12 @@ blend: low_level_execution: weight: 0.40 messages: - # The action expert is conditioned on the SUBTASK — at inference - # `HighLevelSubtaskFwd` generates it via the LM head and feeds it - # here. `stream: low_level` flips `predict_actions=True` so the - # flow loss fires; no text-CE target (subtask prediction is owned - # by `high_level_subtask`). + # The low-level stream trains action flow on the generated or annotated subtask. - {role: user, content: "${subtask}", stream: low_level, if_present: subtask} memory_update: - # At inference, `MemoryUpdateFwd` is triggered only on - # `subtask_change` events (sparse). Training densely with - # `active_at` — i.e. on every frame inside a subtask interval, - # not just the boundary frame — supervises the same - # (prior_memory, completed_subtask) → current_memory mapping - # against varied observations within the interval. The model - # learns a stateless transformation; the *when* to emit lives in - # the inference trigger, not the model. Annotations only exist - # for ~1% of frames as boundary events, so `emitted_at` would - # waste 99% of the blend draws (and silently leak them into the - # task-conditioned fallback); `active_at` lifts the renderable - # rate to ~87% on Hi-Robot-style datasets. + # `active_at` densifies sparse boundaries while preserving the prior-memory/subtask mapping. + # Inference controls update timing through `subtask_change` events. weight: 0.10 bindings: prior_memory: "nth_prev(style=memory, offset=1)" @@ -74,15 +37,10 @@ blend: messages: - {role: user, content: "${task}", stream: high_level} - {role: user, content: "${interjection}", stream: high_level, if_present: interjection} - # Spoken reply only: the assistant turn carries no text content, - # just a `say` tool call (`tool_calls_from: speech`). The chat - # tokenizer flattens it to a `...` marker, so the - # supervised target trains the model to respond to an - # interjection with a spoken acknowledgement. + # The assistant target is a `say` tool call flattened to a `...` marker. - {role: assistant, stream: high_level, target: true, if_present: speech, tool_calls_from: speech} - # VQA is view-dependent — each camera gets its own sub-recipe so the - # resolver disambiguates via `camera=...`. Adjust camera keys to your dataset. + # Each camera uses a separate VQA sub-recipe for view-specific binding. ask_vqa_top: weight: 0.075 bindings: