From b9db4d21a2b852cbf2b28b1c67d959389f1d742f Mon Sep 17 00:00:00 2001
From: Pepijn <pepijn@huggingface.co>
Date: Tue, 12 May 2026 17:38:06 +0200
Subject: [PATCH] fix(smolvla2): high-level steps must run before
 LowLevelForward refills
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both HighLevelSubtaskFwd and LowLevelForward are gated on
'action queue is empty'. With LowLevelForward listed first, it refilled
the queue on the empty-queue tick before HighLevelSubtaskFwd got to
check — so the gate I added in the previous commit made the high-level
step a permanent no-op after the initial bootstrap. Visible symptom:
subtask string never advances past whatever bootstrap seeded, no
subtask_change events, memory stays unset, and the new overfit
diagnostics never appear on the panel because last_subtask_raw is
never written.

Move all high-level steps (subtask, memory, interjection, vqa) ahead
of LowLevelForward. On an empty-queue tick the subtask refreshes
first, the new string flows into the next chunk's prompt, then
LowLevelForward generates the chunk, then DispatchAction drains it.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .../policies/smolvla2/inference/runtime.py    | 29 +++++++++++++------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/lerobot/policies/smolvla2/inference/runtime.py b/src/lerobot/policies/smolvla2/inference/runtime.py
index f888b8bc3..3d76015ab 100644
--- a/src/lerobot/policies/smolvla2/inference/runtime.py
+++ b/src/lerobot/policies/smolvla2/inference/runtime.py
@@ -69,16 +69,18 @@ class SmolVLA2Runtime:
     _stop: bool = field(default=False, init=False)
 
     def __post_init__(self) -> None:
+        # Pipeline order matters. Both ``HighLevelSubtaskFwd`` and
+        # ``LowLevelForward`` are gated on "action queue is empty" so
+        # the slow LLM call (select_message) doesn't starve dispatch.
+        # If LowLevelForward runs first, it refills the queue and the
+        # high-level step never sees ``queue == 0`` afterwards.
+        #
+        # Order is therefore: high-level steps that read state (subtask,
+        # memory, interjection, vqa) → low-level chunk refresh → action
+        # dispatch → tool dispatch. So on an empty-queue tick the
+        # subtask refreshes first, the new subtask string flows into
+        # the next chunk's prompt, and DispatchAction drains.
         self.pipeline = [
-            LowLevelForward(
-                trigger=HzTrigger(self.chunk_hz),
-                policy=self.policy,
-                observation_provider=self.observation_provider,
-            ),
-            DispatchAction(
-                trigger=HzTrigger(self.ctrl_hz),
-                robot_executor=self.robot_executor,
-            ),
             HighLevelSubtaskFwd(
                 trigger=HzTrigger(self.high_level_hz),
                 policy=self.policy,
@@ -96,6 +98,15 @@ class SmolVLA2Runtime:
                 policy=self.policy,
                 observation_provider=self.observation_provider,
             ),
+            LowLevelForward(
+                trigger=HzTrigger(self.chunk_hz),
+                policy=self.policy,
+                observation_provider=self.observation_provider,
+            ),
+            DispatchAction(
+                trigger=HzTrigger(self.ctrl_hz),
+                robot_executor=self.robot_executor,
+            ),
             DispatchToolCalls(tools=self.tools),
         ]
         self.state = initial_runtime_state()