mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-06 09:37:06 +00:00
Pepijn
4fa9578e3d
Cleanup pass over the language-support PR to cut LOC and scope creep. Removals: - SayTool + tools/ package (registry, Tool protocol, [tools] extra) and the runtime's tool-dispatch path. Kept <say> training supervision and inference stripping so speech-annotated datasets still train. - WeightedEpisodeAwareSampler + VQA oversampling wiring (_build_vqa_oversample_weights, vqa_target_fraction) — training uses plain EpisodeAwareSampler again. - Debug env-gates PI052_DEBUG_TENSORS, PI052_SUBTASK_USE_TASK, EVAL_TASK_OVERRIDE. - Dead code: broken _tp._DUMP_BUDGET block, unused imports (copy/Tensor, RevisionNotFoundError, LeRobotDataset, os), messages_for_vqa, steps.py shim (modeling imports pi052_adapter directly), duplicated _emit, builtins.type[T]. Moves: - Policy-agnostic runtime -> src/lerobot/runtime/ (LanguageConditionedRuntime + adapter Protocol + state); pi052 keeps only its adapter + CLI. Tests -> tests/runtime/. Other: - Compacted verbose AI-authored comments/docstrings across pi052 (kept the hard-won DDP / barrier-timeout / reduce-max / VQA-routing notes). - Relocated LM-head prediction debug helper to pi052/debug_utils.py. - Fixed test_render_messages: assert task-fallback render (current behavior) instead of the stale no-op expectation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
72 lines
2.2 KiB
Python
72 lines
2.2 KiB
Python
from lerobot.runtime import (
|
|
LanguageConditionedRuntime,
|
|
RuntimeState,
|
|
VQAResult,
|
|
)
|
|
|
|
|
|
class FakeAdapter:
|
|
def __init__(self):
|
|
self.updated = False
|
|
self.text_calls = []
|
|
|
|
def select_action(self, observation, state):
|
|
assert observation == {"observation.state": 1}
|
|
assert state.task == "clean"
|
|
return ["a0", "a1"]
|
|
|
|
def select_text(self, kind, observation, state, user_text=None):
|
|
self.text_calls.append((kind, user_text))
|
|
return "new plan"
|
|
|
|
def answer_vqa(self, question, camera, observation, state):
|
|
return VQAResult(answer=f"answer: {question}")
|
|
|
|
def update_language_state(self, observation, state):
|
|
self.updated = True
|
|
state.set_context("subtask", "pick cup", label="subtask")
|
|
|
|
|
|
def test_runtime_tick_updates_language_enqueues_and_dispatches_action():
|
|
adapter = FakeAdapter()
|
|
executed = []
|
|
runtime = LanguageConditionedRuntime(
|
|
policy_adapter=adapter,
|
|
observation_provider=lambda: {"observation.state": 1},
|
|
action_executor=executed.append,
|
|
)
|
|
runtime.set_task("clean")
|
|
|
|
logs = runtime.step_once()
|
|
|
|
assert adapter.updated
|
|
assert runtime.state.language_context["subtask"] == "pick cup"
|
|
assert executed == ["a0"]
|
|
assert list(runtime.state.action_queue) == ["a1"]
|
|
assert " subtask: pick cup" in logs
|
|
|
|
|
|
def test_runtime_handles_user_interjection():
|
|
adapter = FakeAdapter()
|
|
runtime = LanguageConditionedRuntime(
|
|
policy_adapter=adapter,
|
|
observation_provider=lambda: {"observation.state": 1},
|
|
)
|
|
runtime.set_task("clean")
|
|
runtime.state.extra["recent_interjection"] = "please say ok"
|
|
runtime.state.emit("user_interjection")
|
|
|
|
runtime.step_once()
|
|
|
|
assert ("interjection", "please say ok") in adapter.text_calls
|
|
assert runtime.state.language_context["plan"] == "new plan"
|
|
|
|
|
|
def test_runtime_state_aliases_legacy_keys_to_language_context():
|
|
state = RuntimeState()
|
|
state["current_subtask"] = "open drawer"
|
|
state["current_memory"] = "drawer open"
|
|
|
|
assert state.get("current_subtask") == "open drawer"
|
|
assert state.language_context == {"subtask": "open drawer", "memory": "drawer open"}
|