From a130a9db393187676e30a6394b6b44253c1b5eff Mon Sep 17 00:00:00 2001 From: Khalil Meftah Date: Wed, 17 Jun 2026 11:41:50 +0200 Subject: [PATCH] feat(eval): optionally push recorded eval datasets to the Hub --- src/lerobot/configs/default.py | 9 +++++++- src/lerobot/scripts/lerobot_eval.py | 32 +++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/lerobot/configs/default.py b/src/lerobot/configs/default.py index 30f021ba3..648e03f33 100644 --- a/src/lerobot/configs/default.py +++ b/src/lerobot/configs/default.py @@ -73,10 +73,17 @@ class EvalConfig: # `use_async_envs` specifies whether to use asynchronous environments (multiprocessing). # Defaults to True; automatically downgraded to SyncVectorEnv when batch_size=1. use_async_envs: bool = True - # Whether to record eval rollouts as a LeRobot v3.0 dataset on disk. + # Whether to record eval rollouts as a LeRobot dataset on disk. recording: bool = False + # If set, push recorded eval datasets to the Hub under this repo id (one repo per task, + # suffixed by task and env index). Requires recording=true. + recording_repo_id: str | None = None + # Whether the pushed recording repositories should be private. + recording_private: bool = False def __post_init__(self) -> None: + if self.recording_repo_id is not None and not self.recording: + raise ValueError("eval.recording_repo_id requires eval.recording=true.") if self.batch_size == 0: self.batch_size = self._auto_batch_size() if self.batch_size > self.n_episodes: diff --git a/src/lerobot/scripts/lerobot_eval.py b/src/lerobot/scripts/lerobot_eval.py index 6ec262a30..c965573c1 100644 --- a/src/lerobot/scripts/lerobot_eval.py +++ b/src/lerobot/scripts/lerobot_eval.py @@ -167,6 +167,8 @@ def rollout( render_callback: Callable[[gym.vector.VectorEnv], None] | None = None, recording_dir: Path | None = None, env_features: dict | None = None, + recording_repo_id: str | None = None, + recording_private: bool = False, ) -> dict: """Run a batched policy rollout once through a batch of environments. @@ -215,10 +217,13 @@ def rollout( fps = env.unwrapped.metadata.get("render_fps", 30) recording_datasets = [] for i in range(env.num_envs): - root = str(recording_dir / f"env_{i}") if env.num_envs > 1 else str(recording_dir) + multi_env = env.num_envs > 1 + root = str(recording_dir / f"env_{i}") if multi_env else str(recording_dir) + base_repo_id = recording_repo_id or "eval_recording" + repo_id = f"{base_repo_id}_env_{i}" if multi_env else base_repo_id recording_datasets.append( LeRobotDataset.create( - repo_id="eval_recording", + repo_id=repo_id, fps=fps, features=features, root=root, @@ -364,6 +369,8 @@ def rollout( if recording_datasets is not None: for ds in recording_datasets: ds.finalize() + if recording_repo_id is not None: + ds.push_to_hub(private=recording_private) if hasattr(policy, "use_original_modules"): policy.use_original_modules() @@ -385,6 +392,8 @@ def eval_policy( start_seed: int | None = None, recording_dir: Path | None = None, env_features: dict | None = None, + recording_repo_id: str | None = None, + recording_private: bool = False, ) -> dict: """ Args: @@ -475,6 +484,8 @@ def eval_policy( render_callback=render_frame if max_episodes_rendered > 0 else None, recording_dir=recording_dir, env_features=env_features, + recording_repo_id=recording_repo_id, + recording_private=recording_private, ) # Figure out where in each rollout sequence the first done condition was encountered (results after @@ -697,6 +708,8 @@ def eval_main(cfg: EvalPipelineConfig): max_parallel_tasks=cfg.env.max_parallel_tasks, recording_dir=recording_dir, env_features=cfg.env.features if cfg.eval.recording else None, + recording_repo_id=cfg.eval.recording_repo_id, + recording_private=cfg.eval.recording_private, ) print("Overall Aggregated Metrics:") print(info["overall"]) @@ -741,6 +754,8 @@ def eval_one( start_seed: int | None, recording_dir: Path | None = None, env_features: dict | None = None, + recording_repo_id: str | None = None, + recording_private: bool = False, ) -> TaskMetrics: """Evaluates one task_id of one suite using the provided vec env.""" @@ -760,6 +775,8 @@ def eval_one( start_seed=start_seed, recording_dir=recording_dir, env_features=env_features, + recording_repo_id=recording_repo_id, + recording_private=recording_private, ) per_episode = task_result["per_episode"] @@ -788,6 +805,8 @@ def run_one( start_seed: int | None, recording_dir: Path | None = None, env_features: dict | None = None, + recording_repo_id: str | None = None, + recording_private: bool = False, ): """ Run eval_one for a single (task_group, task_id, env). @@ -800,8 +819,11 @@ def run_one( task_videos_dir.mkdir(parents=True, exist_ok=True) task_recording_dir = None + task_repo_id = None if recording_dir is not None and env_features is not None: task_recording_dir = recording_dir / f"{task_group}_{task_id}" + if recording_repo_id is not None: + task_repo_id = f"{recording_repo_id}_{task_group}_{task_id}" metrics = eval_one( env, @@ -817,6 +839,8 @@ def run_one( start_seed=start_seed, recording_dir=task_recording_dir, env_features=env_features, + recording_repo_id=task_repo_id, + recording_private=recording_private, ) if max_episodes_rendered > 0: @@ -836,6 +860,8 @@ def eval_policy_all( max_episodes_rendered: int = 0, recording_dir: Path | None = None, env_features: dict | None = None, + recording_repo_id: str | None = None, + recording_private: bool = False, videos_dir: Path | None = None, return_episode_data: bool = False, start_seed: int | None = None, @@ -897,6 +923,8 @@ def eval_policy_all( start_seed=start_seed, recording_dir=recording_dir, env_features=env_features, + recording_repo_id=recording_repo_id, + recording_private=recording_private, ) if max_parallel_tasks <= 1: