mirror of
https://github.com/huggingface/lerobot.git
synced 2026-06-27 05:07:15 +00:00
Khalil Meftah
6a788fbdb0
* refactor(training): rename eval_freq to env_eval_freq - Rename eval_freq to env_eval_freq to distinguish sim environment evaluation from offline loss evaluation. * feat(training): add inline offline validation with train/eval split - Add eval_split config for balanced per-task holdout - Add eval_steps for periodic inline eval loss computation - Add max_eval_samples to cap eval cost * fix(datasets): remap absolute indices in __getitem__ for filtered datasets * fix(train): vectorize eval subset selection for max_eval_samples * fix(datasets): Move the remapping into EpisodeAwareSampler via absolute_to_relative_idx * fix(validation): add eval_split range check and eval_steps warning Validate eval_split is in [0.0, 1.0) to prevent garbage splits from out-of-range values. Raise when eval_steps > 0 but eval_split is 0.0 since no offline eval will run. * fix(train): prepare eval dataloader with accelerator for multi-GPU Prepare eval_dataloader through accelerator.prepare() so eval data is sharded across ranks instead of duplicated. Reduce eval_loss across ranks with mean reduction for consistent logging. * fix(test): rename eval_freq to env_eval_freq for multi-GPU training
187 lines
5.2 KiB
Makefile
187 lines
5.2 KiB
Makefile
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
.PHONY: tests
|
|
|
|
PYTHON_PATH := $(shell which python)
|
|
|
|
# If uv is installed and a virtual environment exists, use it
|
|
UV_CHECK := $(shell command -v uv)
|
|
ifneq ($(UV_CHECK),)
|
|
PYTHON_PATH := $(shell .venv/bin/python)
|
|
endif
|
|
|
|
export PATH := $(dir $(PYTHON_PATH)):$(PATH)
|
|
|
|
DEVICE ?= cpu
|
|
|
|
build-user:
|
|
docker build -f docker/Dockerfile.user -t lerobot-user .
|
|
|
|
build-internal:
|
|
docker build -f docker/Dockerfile.internal -t lerobot-internal .
|
|
|
|
test-end-to-end:
|
|
${MAKE} DEVICE=$(DEVICE) test-act-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-act-ete-train-resume
|
|
${MAKE} DEVICE=$(DEVICE) test-act-ete-eval
|
|
${MAKE} DEVICE=$(DEVICE) test-diffusion-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-diffusion-ete-eval
|
|
${MAKE} DEVICE=$(DEVICE) test-tdmpc-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-tdmpc-ete-eval
|
|
${MAKE} DEVICE=$(DEVICE) test-smolvla-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-smolvla-ete-eval
|
|
|
|
test-act-ete-train:
|
|
lerobot-train \
|
|
--policy.type=act \
|
|
--policy.dim_model=64 \
|
|
--policy.n_action_steps=20 \
|
|
--policy.chunk_size=20 \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/aloha_sim_transfer_cube_human \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=4 \
|
|
--env_eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_freq=2 \
|
|
--save_checkpoint=true \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/act/
|
|
|
|
test-act-ete-train-resume:
|
|
lerobot-train \
|
|
--config_path=tests/outputs/act/checkpoints/000002/pretrained_model/train_config.json \
|
|
--resume=true
|
|
|
|
test-act-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/act/checkpoints/000004/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
test-diffusion-ete-train:
|
|
lerobot-train \
|
|
--policy.type=diffusion \
|
|
--policy.down_dims='[64,128,256]' \
|
|
--policy.diffusion_step_embed_dim=32 \
|
|
--policy.num_inference_steps=10 \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/pusht \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=2 \
|
|
--env_eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_checkpoint=true \
|
|
--save_freq=2 \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/diffusion/
|
|
|
|
test-diffusion-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/diffusion/checkpoints/000002/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
test-tdmpc-ete-train:
|
|
lerobot-train \
|
|
--policy.type=tdmpc \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/pusht_image \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=2 \
|
|
--env_eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_checkpoint=true \
|
|
--save_freq=2 \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/tdmpc/
|
|
|
|
test-tdmpc-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/tdmpc/checkpoints/000002/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--env.observation_height=96 \
|
|
--env.observation_width=96 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
|
|
test-smolvla-ete-train:
|
|
lerobot-train \
|
|
--policy.type=smolvla \
|
|
--policy.n_action_steps=20 \
|
|
--policy.chunk_size=20 \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/aloha_sim_transfer_cube_human \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=4 \
|
|
--env_eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_freq=2 \
|
|
--save_checkpoint=true \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/smolvla/
|
|
|
|
test-smolvla-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/smolvla/checkpoints/000004/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
# E2E annotation pipeline smoke test against a tiny in-memory fixture
|
|
# dataset. Opt-in (not part of `make test-end-to-end`) and uses a stub VLM
|
|
# backend, so it does not require a real model checkpoint or GPU.
|
|
annotation-e2e:
|
|
uv run python -m tests.annotations.run_e2e_smoke
|