mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-26 22:20:06 +00:00
smolvla2(runtime): wire MemoryUpdateFwd into the inference pipeline
MemoryUpdateFwd was importable but never installed, so subtask_change events fired by HighLevelSubtaskFwd had no listener and current_memory stayed at its initial None value — the runtime panel always showed 'memory (not set)' even when the policy was trained with the memory_update recipe (e.g. subtask_mem_vqa_speech.yaml, weight 0.15). Insert MemoryUpdateFwd between HighLevelSubtaskFwd and AskVQAFwd so the event is visible the same tick it is emitted, and refresh the stale comment that claimed memory was not in scope. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -33,8 +33,9 @@ from .steps import (
|
|||||||
HighLevelSubtaskFwd,
|
HighLevelSubtaskFwd,
|
||||||
InferenceStep,
|
InferenceStep,
|
||||||
LowLevelForward,
|
LowLevelForward,
|
||||||
|
MemoryUpdateFwd,
|
||||||
)
|
)
|
||||||
from .triggers import HzTrigger, TickClock
|
from .triggers import EventTrigger, HzTrigger, TickClock
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -67,29 +68,40 @@ class SmolVLA2Runtime:
|
|||||||
_stop: bool = field(default=False, init=False)
|
_stop: bool = field(default=False, init=False)
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
def __post_init__(self) -> None:
|
||||||
# Subtask + VQA configuration (current scope — plan and memory
|
# Subtask + memory + VQA configuration. Pipeline:
|
||||||
# are not trained yet). Pipeline:
|
|
||||||
#
|
#
|
||||||
# HighLevelSubtaskFwd → generate the next subtask via the LM
|
# HighLevelSubtaskFwd → generate the next subtask via the LM
|
||||||
# head at ~``high_level_hz``; writes
|
# head at ~``high_level_hz``; writes
|
||||||
# ``current_subtask``
|
# ``current_subtask`` and emits
|
||||||
# AskVQAFwd → answer camera-grounded stdin questions
|
# ``subtask_change`` on a transition.
|
||||||
|
# MemoryUpdateFwd → on ``subtask_change``, refresh
|
||||||
|
# ``current_memory`` from the
|
||||||
|
# ``memory_update`` head.
|
||||||
|
# AskVQAFwd → answer camera-grounded stdin questions.
|
||||||
# LowLevelForward → action chunk conditioned on the
|
# LowLevelForward → action chunk conditioned on the
|
||||||
# generated ``current_subtask``
|
# generated ``current_subtask``.
|
||||||
# DispatchAction → drain the chunk to the robot
|
# DispatchAction → drain the chunk to the robot.
|
||||||
# DispatchToolCalls → fire any pending tool calls
|
# DispatchToolCalls → fire any pending tool calls.
|
||||||
#
|
#
|
||||||
# Order matters: ``HighLevelSubtaskFwd`` and ``LowLevelForward``
|
# Order matters: ``HighLevelSubtaskFwd`` must run before
|
||||||
# are both gated on "action queue empty", so the subtask must
|
# ``MemoryUpdateFwd`` so the event is visible the same tick, and
|
||||||
# refresh *before* the chunk that consumes it. ``MemoryUpdateFwd``
|
# both must run before ``LowLevelForward`` (which is gated on
|
||||||
# / ``UserInterjectionFwd`` are still importable from
|
# "action queue empty") so the chunk consumes the freshest
|
||||||
# ``inference.steps`` — re-add once plan / memory are in scope.
|
# subtask. ``UserInterjectionFwd`` is still importable but
|
||||||
|
# disabled until plan generation is wired in.
|
||||||
self.pipeline = [
|
self.pipeline = [
|
||||||
HighLevelSubtaskFwd(
|
HighLevelSubtaskFwd(
|
||||||
trigger=HzTrigger(self.high_level_hz),
|
trigger=HzTrigger(self.high_level_hz),
|
||||||
policy=self.policy,
|
policy=self.policy,
|
||||||
observation_provider=self.observation_provider,
|
observation_provider=self.observation_provider,
|
||||||
),
|
),
|
||||||
|
# Listens for the ``subtask_change`` event raised by
|
||||||
|
# ``HighLevelSubtaskFwd`` and refreshes ``current_memory``.
|
||||||
|
MemoryUpdateFwd(
|
||||||
|
trigger=EventTrigger("subtask_change"),
|
||||||
|
policy=self.policy,
|
||||||
|
observation_provider=self.observation_provider,
|
||||||
|
),
|
||||||
AskVQAFwd(
|
AskVQAFwd(
|
||||||
policy=self.policy,
|
policy=self.policy,
|
||||||
observation_provider=self.observation_provider,
|
observation_provider=self.observation_provider,
|
||||||
|
|||||||
Reference in New Issue
Block a user