diff --git a/src/lerobot/policies/evo1/modeling_evo1.py b/src/lerobot/policies/evo1/modeling_evo1.py index a81a0705a..b9369f4b0 100644 --- a/src/lerobot/policies/evo1/modeling_evo1.py +++ b/src/lerobot/policies/evo1/modeling_evo1.py @@ -42,6 +42,9 @@ class Evo1Policy(PreTrainedPolicy): config_class = Evo1Config name = "evo1" + def supports_rtc(self) -> bool: + return True + def __init__(self, config: Evo1Config, *, vlm_hub_kwargs: dict | None = None, **kwargs): super().__init__(config) config.validate_features() diff --git a/src/lerobot/policies/groot/modeling_groot.py b/src/lerobot/policies/groot/modeling_groot.py index 415af9309..8c5dfba34 100644 --- a/src/lerobot/policies/groot/modeling_groot.py +++ b/src/lerobot/policies/groot/modeling_groot.py @@ -68,6 +68,9 @@ class GrootPolicy(PreTrainedPolicy): name = "groot" config_class = GrootConfig + def supports_rtc(self) -> bool: + return True + def __init__(self, config: GrootConfig, **kwargs): """Initialize Groot policy wrapper.""" require_package("transformers", extra="groot") diff --git a/src/lerobot/policies/molmoact2/modeling_molmoact2.py b/src/lerobot/policies/molmoact2/modeling_molmoact2.py index 14d9e19e2..b05091900 100644 --- a/src/lerobot/policies/molmoact2/modeling_molmoact2.py +++ b/src/lerobot/policies/molmoact2/modeling_molmoact2.py @@ -520,6 +520,9 @@ class MolmoAct2Policy(PreTrainedPolicy): config_class = MolmoAct2Config name = "molmoact2" + def supports_rtc(self) -> bool: + return self.config.inference_action_mode == "continuous" + def __init__( self, config: MolmoAct2Config, diff --git a/src/lerobot/policies/pi0/modeling_pi0.py b/src/lerobot/policies/pi0/modeling_pi0.py index 7a444f97f..aca9112b2 100644 --- a/src/lerobot/policies/pi0/modeling_pi0.py +++ b/src/lerobot/policies/pi0/modeling_pi0.py @@ -749,6 +749,9 @@ class PI0Policy(PreTrainedPolicy): config_class = PI0Config name = "pi0" + def supports_rtc(self) -> bool: + return True + def __init__( self, config: PI0Config, diff --git a/src/lerobot/policies/pi05/modeling_pi05.py b/src/lerobot/policies/pi05/modeling_pi05.py index d45f5a5c2..ee19fd5ab 100644 --- a/src/lerobot/policies/pi05/modeling_pi05.py +++ b/src/lerobot/policies/pi05/modeling_pi05.py @@ -714,6 +714,9 @@ class PI05Policy(PreTrainedPolicy): config_class = PI05Config name = "pi05" + def supports_rtc(self) -> bool: + return True + def __init__( self, config: PI05Config, diff --git a/src/lerobot/policies/pretrained.py b/src/lerobot/policies/pretrained.py index 0283f389a..8823e5808 100644 --- a/src/lerobot/policies/pretrained.py +++ b/src/lerobot/policies/pretrained.py @@ -249,6 +249,10 @@ class PreTrainedPolicy(nn.Module, HubMixin, abc.ABC): """ raise NotImplementedError + def supports_rtc(self) -> bool: + """Whether this policy implements Real-Time Chunking inference semantics.""" + return False + # TODO(aliberts, rcadene): split into 'forward' and 'compute_loss'? @abc.abstractmethod def forward(self, batch: dict[str, Tensor]) -> tuple[Tensor, dict | None]: diff --git a/src/lerobot/policies/smolvla/modeling_smolvla.py b/src/lerobot/policies/smolvla/modeling_smolvla.py index 7f54d03f0..2ac63b994 100644 --- a/src/lerobot/policies/smolvla/modeling_smolvla.py +++ b/src/lerobot/policies/smolvla/modeling_smolvla.py @@ -145,6 +145,9 @@ class SmolVLAPolicy(PreTrainedPolicy): config_class = SmolVLAConfig name = "smolvla" + def supports_rtc(self) -> bool: + return True + def __init__( self, config: SmolVLAConfig, diff --git a/src/lerobot/rollout/context.py b/src/lerobot/rollout/context.py index cf69d57b5..e4974561b 100644 --- a/src/lerobot/rollout/context.py +++ b/src/lerobot/rollout/context.py @@ -57,6 +57,7 @@ from .inference import ( SyncInferenceConfig, create_inference_engine, ) +from .inference.rtc import supports_rtc_inference from .robot_wrapper import ThreadSafeRobot if TYPE_CHECKING or _peft_available: @@ -226,6 +227,12 @@ def build_rollout_context( policy = _load_pretrained_policy(policy_config) if is_rtc: + if not supports_rtc_inference(policy): + raise ValueError( + f"RTC inference is not supported by policy type '{policy_config.type}': " + "the policy must implement RTC semantics and predict_action_chunk must accept " + "inference_delay and prev_chunk_left_over. Use '--inference.type=sync' instead." + ) policy.config.rtc_config = cfg.inference.rtc if hasattr(policy, "init_rtc_processor"): policy.init_rtc_processor() diff --git a/src/lerobot/rollout/inference/rtc.py b/src/lerobot/rollout/inference/rtc.py index 0eef62cef..8ff44998a 100644 --- a/src/lerobot/rollout/inference/rtc.py +++ b/src/lerobot/rollout/inference/rtc.py @@ -22,6 +22,7 @@ way via ``notify_observation``. from __future__ import annotations +import inspect import logging import math import time @@ -62,6 +63,23 @@ _RTC_JOIN_TIMEOUT_S: float = 3.0 # --------------------------------------------------------------------------- +def supports_rtc_inference(policy: PreTrainedPolicy) -> bool: + """Whether a policy declares RTC support and accepts the RTC call shape.""" + supports_rtc = getattr(policy, "supports_rtc", None) + if not callable(supports_rtc) or not supports_rtc(): + return False + + try: + inspect.signature(policy.predict_action_chunk).bind( + object(), + inference_delay=0, + prev_chunk_left_over=None, + ) + except (TypeError, ValueError): + return False + return True + + def _normalize_prev_actions_length(prev_actions: torch.Tensor, target_steps: int) -> torch.Tensor: """Pad or truncate RTC prefix actions to a fixed length for stable compiled inference.""" if prev_actions.ndim != 2: