mirror of
https://github.com/huggingface/lerobot.git
synced 2026-06-16 07:49:48 +00:00
Pepijn
cec8ee0be6
Steerable annotation pipeline (lerobot-annotate) that populates the language_persistent and language_events columns introduced in PR 1 (#3467) directly into data/chunk-*/file-*.parquet. This is PR 2 of the three-PR plan: PR 1 (Add extensive language support #3467): schema + DSL + rendering, base of this PR PR 2 (this PR): annotation pipeline writing into PR 1's columns PR 3: model with language prediction and runtime A VLM (Qwen-VL family, served on vLLM) watches each episode's video and emits grounded language annotations: subtasks, plans, memory, task rephrasings, interjections + speech, and per-camera VQA. The pipeline is built for production annotation at scale — single-camera grounding, embedded-frame inputs, a describe-then-segment grounding flow, and a deterministic full-episode coverage guarantee — informed by Scale's dense-captioning findings (representation > sampling, rules > reasoning, model capacity is the biggest lever, two-pass systems compound errors)
187 lines
5.2 KiB
Makefile
187 lines
5.2 KiB
Makefile
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
.PHONY: tests
|
|
|
|
PYTHON_PATH := $(shell which python)
|
|
|
|
# If uv is installed and a virtual environment exists, use it
|
|
UV_CHECK := $(shell command -v uv)
|
|
ifneq ($(UV_CHECK),)
|
|
PYTHON_PATH := $(shell .venv/bin/python)
|
|
endif
|
|
|
|
export PATH := $(dir $(PYTHON_PATH)):$(PATH)
|
|
|
|
DEVICE ?= cpu
|
|
|
|
build-user:
|
|
docker build -f docker/Dockerfile.user -t lerobot-user .
|
|
|
|
build-internal:
|
|
docker build -f docker/Dockerfile.internal -t lerobot-internal .
|
|
|
|
test-end-to-end:
|
|
${MAKE} DEVICE=$(DEVICE) test-act-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-act-ete-train-resume
|
|
${MAKE} DEVICE=$(DEVICE) test-act-ete-eval
|
|
${MAKE} DEVICE=$(DEVICE) test-diffusion-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-diffusion-ete-eval
|
|
${MAKE} DEVICE=$(DEVICE) test-tdmpc-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-tdmpc-ete-eval
|
|
${MAKE} DEVICE=$(DEVICE) test-smolvla-ete-train
|
|
${MAKE} DEVICE=$(DEVICE) test-smolvla-ete-eval
|
|
|
|
test-act-ete-train:
|
|
lerobot-train \
|
|
--policy.type=act \
|
|
--policy.dim_model=64 \
|
|
--policy.n_action_steps=20 \
|
|
--policy.chunk_size=20 \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/aloha_sim_transfer_cube_human \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=4 \
|
|
--eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_freq=2 \
|
|
--save_checkpoint=true \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/act/
|
|
|
|
test-act-ete-train-resume:
|
|
lerobot-train \
|
|
--config_path=tests/outputs/act/checkpoints/000002/pretrained_model/train_config.json \
|
|
--resume=true
|
|
|
|
test-act-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/act/checkpoints/000004/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
test-diffusion-ete-train:
|
|
lerobot-train \
|
|
--policy.type=diffusion \
|
|
--policy.down_dims='[64,128,256]' \
|
|
--policy.diffusion_step_embed_dim=32 \
|
|
--policy.num_inference_steps=10 \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/pusht \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=2 \
|
|
--eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_checkpoint=true \
|
|
--save_freq=2 \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/diffusion/
|
|
|
|
test-diffusion-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/diffusion/checkpoints/000002/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
test-tdmpc-ete-train:
|
|
lerobot-train \
|
|
--policy.type=tdmpc \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/pusht_image \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=2 \
|
|
--eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_checkpoint=true \
|
|
--save_freq=2 \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/tdmpc/
|
|
|
|
test-tdmpc-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/tdmpc/checkpoints/000002/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=pusht \
|
|
--env.episode_length=5 \
|
|
--env.observation_height=96 \
|
|
--env.observation_width=96 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
|
|
test-smolvla-ete-train:
|
|
lerobot-train \
|
|
--policy.type=smolvla \
|
|
--policy.n_action_steps=20 \
|
|
--policy.chunk_size=20 \
|
|
--policy.device=$(DEVICE) \
|
|
--policy.push_to_hub=false \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--dataset.repo_id=lerobot/aloha_sim_transfer_cube_human \
|
|
--dataset.image_transforms.enable=true \
|
|
--dataset.episodes="[0]" \
|
|
--batch_size=2 \
|
|
--steps=4 \
|
|
--eval_freq=2 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1 \
|
|
--save_freq=2 \
|
|
--save_checkpoint=true \
|
|
--log_freq=1 \
|
|
--wandb.enable=false \
|
|
--output_dir=tests/outputs/smolvla/
|
|
|
|
test-smolvla-ete-eval:
|
|
lerobot-eval \
|
|
--policy.path=tests/outputs/smolvla/checkpoints/000004/pretrained_model \
|
|
--policy.device=$(DEVICE) \
|
|
--env.type=aloha \
|
|
--env.episode_length=5 \
|
|
--eval.n_episodes=1 \
|
|
--eval.batch_size=1
|
|
|
|
# E2E annotation pipeline smoke test against a tiny in-memory fixture
|
|
# dataset. Opt-in (not part of `make test-end-to-end`) and uses a stub VLM
|
|
# backend, so it does not require a real model checkpoint or GPU.
|
|
annotation-e2e:
|
|
uv run python -m tests.annotations.run_e2e_smoke
|