diff --git a/docs/source/envhub.mdx b/docs/source/envhub.mdx index 9fc6c6220..a94f49a4f 100644 --- a/docs/source/envhub.mdx +++ b/docs/source/envhub.mdx @@ -284,6 +284,37 @@ envs_dict = make_env( Any keyword arguments you pass will be forwarded to the hub environment's `make_env` function. Check the environment's documentation for supported configuration options. +### Using Custom kwargs with lerobot-eval + +When evaluating policies using the `lerobot-eval` CLI, you can pass custom kwargs to hub environments using the `--env_kwargs.` prefix: + +```bash +lerobot-eval \ + --policy.path=user123/example-policy-checkpoint \ + --env=user123/example-sim-backend \ + --eval.batch_size=1 \ + --eval.n_episodes=10 \ + --env_kwargs.task_id=demo_task_alpha \ + --env_kwargs.agent_profile=arm_v2 \ + --env_kwargs.target_item=object_red \ + --env_kwargs.run_mode=offscreen \ + --env_kwargs.enable_sensors=true \ + --env_kwargs.record_output=true \ + --env_kwargs.output_horizon=10 \ + --env_kwargs.output_stride=15 \ + --env_kwargs.state_features=joint_angles \ + --env_kwargs.visual_streams=front_camera +``` + +All `--env_kwargs.*` arguments will be collected into a dictionary and passed as keyword arguments to the hub environment's `make_env` function. This allows you to: +- Pass configuration file paths +- Override default settings +- Specify custom task parameters +- Control simulation options (headless mode, camera settings, etc.) +- Select different embodiments or objects + +The hub environment's `make_env` function receives these as regular keyword arguments, so check the environment's documentation for the available options. + ## URL Format Reference The hub URL format supports several patterns: diff --git a/src/lerobot/configs/eval.py b/src/lerobot/configs/eval.py index 2f085da56..a4969ae6e 100644 --- a/src/lerobot/configs/eval.py +++ b/src/lerobot/configs/eval.py @@ -38,6 +38,8 @@ class EvalPipelineConfig: seed: int | None = 1000 # Rename map for the observation to override the image and state keys rename_map: dict[str, str] = field(default_factory=dict) + # Additional kwargs to pass to hub environments (e.g., config_path, config_overrides, custom params) + env_kwargs: dict = field(default_factory=dict) def __post_init__(self) -> None: # HACK: We parse again the cli args here to get the pretrained path if there was one. diff --git a/src/lerobot/scripts/lerobot_eval.py b/src/lerobot/scripts/lerobot_eval.py index d23b9d083..78722987d 100644 --- a/src/lerobot/scripts/lerobot_eval.py +++ b/src/lerobot/scripts/lerobot_eval.py @@ -43,6 +43,17 @@ lerobot-eval \ Note that in both examples, the repo/folder should contain at least `config.json` and `model.safetensors` files. +You can also evaluate a model on a Hub environment with custom kwargs: +``` +lerobot-eval \ + --policy.path=HF_USER/HF_REPO \ + --env=HF_USER/HF_REPO \ + --eval.batch_size=1 \ + --eval.n_episodes=10 \ + --env_kwargs.environment=env_A \ + --env_kwargs.embodiment=emb_B \ +``` + You can learn about the CLI options for this script in the `EvalPipelineConfig` in lerobot/configs/eval.py """ @@ -509,7 +520,12 @@ def eval_main(cfg: EvalPipelineConfig): logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") logging.info("Making environment.") - envs = make_env(cfg.env, n_envs=cfg.eval.batch_size, use_async_envs=cfg.eval.use_async_envs) + envs = make_env( + cfg.env, + n_envs=cfg.eval.batch_size, + use_async_envs=cfg.eval.use_async_envs, + **cfg.env_kwargs, + ) logging.info("Making policy.")