diff --git a/src/lerobot/policies/pi052/configuration_pi052.py b/src/lerobot/policies/pi052/configuration_pi052.py index f79534bf1..030f039b6 100644 --- a/src/lerobot/policies/pi052/configuration_pi052.py +++ b/src/lerobot/policies/pi052/configuration_pi052.py @@ -32,8 +32,8 @@ This is the dual-head co-training pattern from the paper: with α = 10.0 per § IV.D of arxiv:2504.16054. The π0.5 model splits inference into a text-prediction step followed by an action-prediction -step, which the multi-rate ``PI052Runtime`` (in -``lerobot.policies.pi052.inference``) drives at separate rates. +step, which the multi-rate runtime (``lerobot.runtime``, via the +``lerobot-pi052-runtime`` CLI) drives at separate rates. """ from dataclasses import dataclass diff --git a/src/lerobot/policies/pi052/inference/__init__.py b/src/lerobot/policies/pi052/inference/__init__.py index 6b613c3cd..21391157f 100644 --- a/src/lerobot/policies/pi052/inference/__init__.py +++ b/src/lerobot/policies/pi052/inference/__init__.py @@ -12,31 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""PI052 runtime adapter and CLI helpers.""" +"""PI052 bridge to the generic language-conditioned runtime. -from lerobot.runtime import ( - LanguageConditionedRuntime, - RuntimeState, - Tick, - TickClock, - VQAResult, -) +The runtime, REPL, and CLI are policy-agnostic and live in +:mod:`lerobot.runtime`. PI052 supplies only :class:`PI052PolicyAdapter`; +the ``lerobot-pi052-runtime`` entry point wires it into +:func:`lerobot.runtime.cli.run`. +""" from .pi052_adapter import PI052PolicyAdapter -from .repl import StdinReader -from .runtime import PI052Runtime -from .ui import make_state_panel, print_robot_lines, print_user_line -__all__ = [ - "LanguageConditionedRuntime", - "PI052PolicyAdapter", - "PI052Runtime", - "RuntimeState", - "StdinReader", - "Tick", - "TickClock", - "VQAResult", - "make_state_panel", - "print_robot_lines", - "print_user_line", -] +__all__ = ["PI052PolicyAdapter"] diff --git a/src/lerobot/policies/pi052/inference/pi052_adapter.py b/src/lerobot/policies/pi052/inference/pi052_adapter.py index 2575d210f..6b263c327 100644 --- a/src/lerobot/policies/pi052/inference/pi052_adapter.py +++ b/src/lerobot/policies/pi052/inference/pi052_adapter.py @@ -21,7 +21,7 @@ import re from dataclasses import dataclass from typing import Any -from lerobot.runtime import RuntimeState, VQAResult +from lerobot.runtime import RuntimeState logger = logging.getLogger(__name__) @@ -73,18 +73,6 @@ class PI052PolicyAdapter: plan, _speech = split_plan_and_say(text) return "" if looks_like_gibberish(plan) else plan - def answer_vqa( - self, - question: str, - camera: str | None, - observation: dict[str, Any] | None, - state: RuntimeState, - ) -> VQAResult: - answer = self.select_text("vqa", observation, state, user_text=question) - from .vqa import parse_vqa_answer # noqa: PLC0415 - - return VQAResult(answer=answer, parsed=parse_vqa_answer(answer), camera=camera) - def update_language_state(self, observation: dict[str, Any] | None, state: RuntimeState) -> None: chunks_per_gen = max(1, int(state.extra.get("subtask_chunks_per_gen", 1) or 1)) if "_hl_chunks_until_gen" not in state.extra: @@ -171,8 +159,6 @@ class PI052PolicyAdapter: return messages if kind == "plan": return [{"role": "user", "content": state.task or ""}] - if kind == "vqa": - return [{"role": "user", "content": user_text or ""}] raise ValueError(f"Unknown PI052 text kind: {kind}") diff --git a/src/lerobot/policies/pi052/inference/runtime.py b/src/lerobot/policies/pi052/inference/runtime.py deleted file mode 100644 index 03765a138..000000000 --- a/src/lerobot/policies/pi052/inference/runtime.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""PI052 compatibility wrapper for the generic language-conditioned runtime.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from lerobot.runtime import ( - LanguageConditionedRuntime, - RuntimeState, - Tick, - TickClock, - VQAResult, -) - -from .pi052_adapter import PI052PolicyAdapter - - -class PI052Runtime(LanguageConditionedRuntime): - """Backwards-compatible PI052 runtime constructor.""" - - def __init__( - self, - policy: Any, - *, - observation_provider: Callable[[], dict | None] | None = None, - robot_executor: Callable[[Any], None] | None = None, - event_collector: Callable[[RuntimeState], None] | None = None, - chunk_hz: float = 4.0, - ctrl_hz: float = 50.0, - high_level_hz: float = 1.0, - max_rate_hz: float = 50.0, - ) -> None: - super().__init__( - policy_adapter=policy if isinstance(policy, PI052PolicyAdapter) else PI052PolicyAdapter(policy), - observation_provider=observation_provider, - action_executor=robot_executor, - event_collector=event_collector, - chunk_hz=chunk_hz, - ctrl_hz=ctrl_hz, - high_level_hz=high_level_hz, - max_rate_hz=max_rate_hz, - ) - - -__all__ = [ - "LanguageConditionedRuntime", - "PI052PolicyAdapter", - "PI052Runtime", - "RuntimeState", - "Tick", - "TickClock", - "VQAResult", -] diff --git a/src/lerobot/policies/pi052/inference/vqa.py b/src/lerobot/policies/pi052/inference/vqa.py deleted file mode 100644 index 6b690abc8..000000000 --- a/src/lerobot/policies/pi052/inference/vqa.py +++ /dev/null @@ -1,406 +0,0 @@ -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Interactive VQA for the PI052 runtime. - -In ``/vlm`` mode a typed line is treated as a VQA question. This module -runs the full interactive flow: - - 1. pull the current observation and list available cameras, - 2. ask the operator which camera to ground the question on, - 3. generate the answer with the VLM conditioned on that one camera, - 4. parse the JSON answer; if it carries a bounding box (``bbox``) or a - point (``keypoint``), draw the overlay on the camera frame, save a - PNG to ``./vqa_overlays/`` and auto-open it. - -VQA answer schemas mirror the annotation pipeline's ``VQA_ANSWER_SHAPES`` -(see ``lerobot.annotations.steerable_pipeline.validator``): - - * ``bbox`` — ``{"detections": [{"label", "bbox_format": "xyxy", - "bbox": [x1, y1, x2, y2]}, ...]}`` - * ``keypoint`` — ``{"label", "point_format": "xy", "point": [x, y]}`` - * ``count`` / ``attribute`` / ``spatial`` — text-only, no overlay. -""" - -from __future__ import annotations - -import json -import logging -import os -import re -import subprocess -import sys -import time -import webbrowser -from contextlib import suppress -from pathlib import Path -from typing import Any - -logger = logging.getLogger(__name__) - -_IMAGE_PREFIX = "observation.images." - -# PaliGemma detection / pointing vocabulary. PI052 trains spatial VQA -# answers in this native ```` format (index in [0, 1023], -# normalized to the image axis) instead of pixel-coordinate JSON, so the -# answer string the runtime parses can be e.g. -# `` blue cube`` (point) or -# `` blue cube`` (box). -_LOC_RE = re.compile(r"") - -# Iteration order for shape matching — most specific keys first so an -# answer is classified deterministically. -_SHAPE_ORDER = ("bbox", "keypoint", "count", "attribute", "spatial") - -_BBOX_COLOR = (255, 64, 64) -_POINT_COLOR = (64, 220, 64) - - -# --------------------------------------------------------------------------- -# Camera selection -# --------------------------------------------------------------------------- - - -def available_cameras(observation: dict | None) -> list[str]: - """Return the sorted ``observation.images.*`` keys present in ``observation``.""" - if not observation: - return [] - return sorted(k for k in observation if isinstance(k, str) and k.startswith(_IMAGE_PREFIX)) - - -def camera_short_name(camera_key: str) -> str: - """Strip the ``observation.images.`` prefix for display.""" - return camera_key[len(_IMAGE_PREFIX) :] if camera_key.startswith(_IMAGE_PREFIX) else camera_key - - -def prompt_camera_choice( - cameras: list[str], - *, - input_fn: Any = input, - print_fn: Any = print, -) -> str | None: - """Ask the operator which camera frame to draw a VQA overlay on. - - Accepts either the menu number or the (short or full) camera name. - A single-camera setup auto-selects without prompting. Returns the - chosen ``observation.images.*`` key, or ``None`` if the operator - cancels / gives an invalid answer. - """ - if not cameras: - return None - if len(cameras) == 1: - return cameras[0] - print_fn("Draw the result on which camera?") - for i, cam in enumerate(cameras, 1): - print_fn(f" [{i}] {camera_short_name(cam)}") - try: - raw = str(input_fn("camera> ")).strip() - except (EOFError, KeyboardInterrupt): - return None - if not raw: - return cameras[0] - if raw.isdigit(): - idx = int(raw) - 1 - return cameras[idx] if 0 <= idx < len(cameras) else None - for cam in cameras: - if raw == cam or raw == camera_short_name(cam): - return cam - return None - - -# --------------------------------------------------------------------------- -# Answer parsing -# --------------------------------------------------------------------------- - - -def _loc_to_norm(idx: int) -> float: - """PaliGemma ```` index → normalized [0, 1] axis coordinate.""" - return max(0.0, min(1023.0, float(idx))) / 1023.0 - - -def parse_loc_answer(answer: str) -> dict | None: - """Parse a PaliGemma ````-format spatial VQA answer. - - Point: ``