mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-15 16:49:55 +00:00
Eugene Mironov
0acdde4ae2
Implement Real-Time Chunking (RTC) for action chunking policies using flow matching denoising. RTC enables smooth action transitions between consecutive chunks by using prefix guidance during denoising. Key features: - RTCProcessor class with denoise_step method for RTC guidance - Tracker system for debug tracking using time-based dictionary storage - RTCDebugVisualizer with comprehensive visualization utilities - Integration with SmolVLA policy for flow matching models - Support for multiple prefix attention schedules (ZEROS, ONES, LINEAR, EXP) - Configurable execution horizon and max guidance weight - Example scripts for dataset evaluation and real-time control Technical details: - Uses autograd-based gradient computation for RTC corrections - Time-based tracking eliminates duplicate step issues - Proxy methods in RTCProcessor for cleaner API - Full integration with LeRobot's policy and dataset systems Files added/modified: - src/lerobot/configs/types.py: Add RTCAttentionSchedule enum - src/lerobot/policies/rtc/: Core RTC implementation - configuration_rtc.py: RTC configuration - modeling_rtc.py: RTCProcessor with denoise_step - debug_handler.py: Tracker for debug information - debug_visualizer.py: Visualization utilities - src/lerobot/policies/smolvla/modeling_smolvla.py: RTC integration - examples/rtc/: Example scripts and evaluation tools 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Alexander Soare <alexander.soare159@gmail.com> Co-Authored-By: Claude <noreply@anthropic.com>
76 lines
2.6 KiB
Bash
Executable File
76 lines
2.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Example script to run RTC evaluation on dataset
|
|
# This shows different usage scenarios
|
|
|
|
set -e # Exit on error
|
|
|
|
POLICY_PATH="lerobot/smolvla_base"
|
|
DATASET="lerobot/pusht"
|
|
DEVICE="cuda" # Change to "cpu" or "mps" if needed
|
|
|
|
echo "========================================"
|
|
echo "RTC Dataset Evaluation Examples"
|
|
echo "========================================"
|
|
|
|
# Example 1: Quick evaluation (100 samples, every step)
|
|
echo -e "\n[Example 1] Quick evaluation - 100 samples, every step"
|
|
python examples/rtc/evaluate_rtc_on_dataset.py \
|
|
--policy.path="${POLICY_PATH}" \
|
|
--dataset.repo_id="${DATASET}" \
|
|
--num_iterations=100 \
|
|
--skip_steps=1 \
|
|
--device="${DEVICE}" \
|
|
--output_path="results/rtc_eval_quick.json"
|
|
|
|
# Example 2: Simulating realistic inference delay (every 3rd step)
|
|
echo -e "\n[Example 2] Realistic inference delay - 200 samples, every 3rd step"
|
|
python examples/rtc/evaluate_rtc_on_dataset.py \
|
|
--policy.path="${POLICY_PATH}" \
|
|
--dataset.repo_id="${DATASET}" \
|
|
--num_iterations=200 \
|
|
--skip_steps=3 \
|
|
--rtc.execution_horizon=10 \
|
|
--device="${DEVICE}" \
|
|
--output_path="results/rtc_eval_delay3.json"
|
|
|
|
# Example 3: Higher inference delay (every 5th step)
|
|
echo -e "\n[Example 3] High inference delay - 200 samples, every 5th step"
|
|
python examples/rtc/evaluate_rtc_on_dataset.py \
|
|
--policy.path="${POLICY_PATH}" \
|
|
--dataset.repo_id="${DATASET}" \
|
|
--num_iterations=200 \
|
|
--skip_steps=5 \
|
|
--rtc.execution_horizon=12 \
|
|
--device="${DEVICE}" \
|
|
--output_path="results/rtc_eval_delay5.json"
|
|
|
|
# Example 4: Testing different RTC configurations
|
|
echo -e "\n[Example 4] Different RTC config - LINEAR schedule"
|
|
python examples/rtc/evaluate_rtc_on_dataset.py \
|
|
--policy.path="${POLICY_PATH}" \
|
|
--dataset.repo_id="${DATASET}" \
|
|
--num_iterations=100 \
|
|
--skip_steps=3 \
|
|
--rtc.execution_horizon=8 \
|
|
--rtc.prefix_attention_schedule=LINEAR \
|
|
--rtc.max_guidance_weight=5.0 \
|
|
--device="${DEVICE}" \
|
|
--output_path="results/rtc_eval_linear.json"
|
|
|
|
# Example 5: Verbose mode for debugging
|
|
echo -e "\n[Example 5] Verbose mode - 20 samples with detailed output"
|
|
python examples/rtc/evaluate_rtc_on_dataset.py \
|
|
--policy.path="${POLICY_PATH}" \
|
|
--dataset.repo_id="${DATASET}" \
|
|
--num_iterations=20 \
|
|
--skip_steps=3 \
|
|
--device="${DEVICE}" \
|
|
--verbose=true \
|
|
--output_path="results/rtc_eval_verbose.json"
|
|
|
|
echo -e "\n========================================"
|
|
echo "All evaluations completed!"
|
|
echo "Results saved in results/ directory"
|
|
echo "========================================"
|