diff --git a/examples/rtc/eval_with_real_robot.py b/examples/rtc/eval_with_real_robot.py index 1f449f9c8..58a0e6027 100644 --- a/examples/rtc/eval_with_real_robot.py +++ b/examples/rtc/eval_with_real_robot.py @@ -10,25 +10,30 @@ This script demonstrates: 4. Managing action buffers and timing for real-time operation Usage: - # With real robot - python rtc_demo.py --policy.path=lerobot/smolvla_base --robot.type=so100 + # Run RTC with Real robot with RTC + uv run examples/rtc/eval_with_real_robot.py \ + --policy.path=helper2424/smolvla_check_rtc_last3 \ + --policy.device=mps \ + --rtc.enabled=true \ + --rtc.execution_horizon=20 \ + --robot.type=so100_follower \ + --robot.port=/dev/tty.usbmodem58FA0834591 \ + --robot.id=so100_follower \ + --robot.cameras="{ gripper: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30}, front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \ + --task="Move green small object into the purple platform" \ + --duration=120 - # With simulation environment - python rtc_demo.py --policy.path=lerobot/smolvla_base --env.type=pusht - - # With config file - python rtc_demo.py --config_path=path/to/config.json - - # With policy compilation for faster inference (recommended for production) - python rtc_demo.py --policy.path=lerobot/smolvla_base --robot.type=so100 --compile_policy=true - - # With aggressive compilation for maximum speed - python rtc_demo.py --policy.path=lerobot/smolvla_base --robot.type=so100 --compile_policy=true --compile_mode=max-autotune - -Performance Notes: - - torch.compile() is NOT supported on MPS (Apple Silicon) due to attention operation limitations - - For MPS optimization, reduce num_steps in the policy config (biggest speedup) - - CUDA devices will see 2-5x speedup with compilation enabled + # Run RTC with Real robot without RTC + uv run examples/rtc/eval_with_real_robot.py \ + --policy.path=helper2424/smolvla_check_rtc_last3 \ + --policy.device=mps \ + --rtc.enabled=false \ + --robot.type=so100_follower \ + --robot.port=/dev/tty.usbmodem58FA0834591 \ + --robot.id=so100_follower \ + --robot.cameras="{ gripper: {type: opencv, index_or_path: 1, width: 640, height: 480, fps: 30}, front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \ + --task="Move green small object into the purple platform" \ + --duration=120 """ import logging @@ -44,6 +49,7 @@ import torch from torch import Tensor from lerobot.cameras.opencv.configuration_opencv import OpenCVCameraConfig # noqa: F401 +from lerobot.cameras.realsense.configuration_realsense import RealSenseCameraConfig # noqa: F401 from lerobot.configs import parser from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.types import RTCAttentionSchedule @@ -340,14 +346,21 @@ def get_actions( dataset_features = hw_to_dataset_features(robot.observation_features(), "observation") policy_device = policy.config.device + # Load preprocessor and postprocessor from pretrained files + # The stats are embedded in the processor .safetensors files + logger.info(f"[GET_ACTIONS] Loading preprocessor/postprocessor from {cfg.policy.pretrained_path}") + preprocessor, postprocessor = make_pre_post_processors( policy_cfg=cfg.policy, pretrained_path=cfg.policy.pretrained_path, + dataset_stats=None, # Will load from pretrained processor files preprocessor_overrides={ "device_processor": {"device": cfg.policy.device}, }, ) + logger.info("[GET_ACTIONS] Preprocessor/postprocessor loaded successfully with embedded stats") + get_actions_threshold = cfg.action_queue_size_to_get_new_actions if not cfg.rtc.enabled: @@ -383,17 +396,10 @@ def get_actions( obs_with_policy_features[name] = obs_with_policy_features[name].unsqueeze(0) obs_with_policy_features[name] = obs_with_policy_features[name].to(policy_device) - # for k, v in obs_with_policy_features.items(): - # if isinstance(v, np.ndarray): - # obs_with_policy_features[k] = torch.from_numpy(v).to(policy_device) - - # if is_image_key(k): - # obs_with_policy_features[k] = obs_with_policy_features[k].type(torch.float32) / 255 - # obs_with_policy_features[k] = obs_with_policy_features[k].permute(2, 0, 1).unsqueeze(0) - # elif isinstance(obs_with_policy_features[k], torch.Tensor): - # obs_with_policy_features[k] = obs_with_policy_features[k].unsqueeze(0) - - obs_with_policy_features["task"] = cfg.task + obs_with_policy_features["task"] = [cfg.task] # Task should be a list, not a string! + obs_with_policy_features["robot_type"] = ( + robot.robot.name if hasattr(robot.robot, "name") else "" + ) preproceseded_obs = preprocessor(obs_with_policy_features) @@ -463,9 +469,9 @@ def actor_control( if action is not None: action = action.cpu() - action = {key: action[i].item() for i, key in enumerate(robot.action_features())} - action = robot_action_processor((action, None)) - robot.send_action(action) + action_dict = {key: action[i].item() for i, key in enumerate(robot.action_features())} + action_processed = robot_action_processor((action_dict, None)) + robot.send_action(action_processed) action_count += 1