diff --git a/.github/workflows/benchmark_tests.yml b/.github/workflows/benchmark_tests.yml new file mode 100644 index 000000000..2425835af --- /dev/null +++ b/.github/workflows/benchmark_tests.yml @@ -0,0 +1,167 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Integration tests: build an isolated Docker image per benchmark and run a +# 1-episode smoke eval. Each benchmark gets its own image so incompatible +# dependency trees (e.g. hf-libero vs metaworld==3.0.0) can never collide. +# +# To add a new benchmark: +# 1. Add docker/Dockerfile.benchmark. (install only lerobot[]) +# 2. Copy one of the jobs below and adjust the image name and eval command. +name: Benchmark Integration Tests + +on: + # Run manually from the Actions tab + workflow_dispatch: + + push: + branches: + - feat/async-vector-env + paths: + - "src/lerobot/envs/**" + - "docker/Dockerfile.benchmark.*" + - ".github/workflows/benchmark_tests.yml" + + pull_request: + branches: + - main + paths: + - "src/lerobot/envs/**" + - "docker/Dockerfile.benchmark.*" + - ".github/workflows/benchmark_tests.yml" + +permissions: + contents: read + +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.12" + +# Cancel in-flight runs for the same branch/PR. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # ── LIBERO ──────────────────────────────────────────────────────────────── + # Isolated image: lerobot[libero] only (hf-libero, dm-control, mujoco chain) + libero-integration-test: + name: Libero — build image + 1-episode eval + runs-on: + group: aws-g6-4xlarge-plus + env: + HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }} + + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + lfs: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses] + with: + cache-binary: false + + # Build the benchmark-specific image; layer cache lives in the runner's + # local Docker daemon — reused across re-runs on the same machine. + - name: Build Libero benchmark image + uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] + with: + context: . + file: docker/Dockerfile.benchmark.libero + push: false + load: true + tags: lerobot-benchmark-libero:ci + cache-from: type=local,src=/tmp/.buildx-cache-libero + cache-to: type=local,dest=/tmp/.buildx-cache-libero,mode=max + + - name: Login to Hugging Face + if: env.HF_USER_TOKEN != '' + run: | + docker run --rm \ + -e HF_HOME=/tmp/hf \ + lerobot-benchmark-libero:ci \ + bash -c "hf auth login --token '$HF_USER_TOKEN' --add-to-git-credential && hf auth whoami" + + - name: Run Libero smoke eval (1 episode) + run: | + docker run --rm --gpus all \ + --shm-size=4g \ + -e HF_HOME=/tmp/hf \ + -e HF_USER_TOKEN="${HF_USER_TOKEN}" \ + lerobot-benchmark-libero:ci \ + bash -c " + hf auth login --token \"\$HF_USER_TOKEN\" --add-to-git-credential 2>/dev/null || true + lerobot-eval \ + --policy.path=pepijn223/smolvla_libero \ + --env.type=libero \ + --env.task=libero_spatial \ + --eval.batch_size=1 \ + --eval.n_episodes=1 \ + --eval.use_async_envs=false \ + --policy.device=cuda \ + '--env.camera_name_mapping={\"agentview_image\": \"camera1\", \"robot0_eye_in_hand_image\": \"camera2\"}' \ + --policy.empty_cameras=1 + " + + # ── METAWORLD ───────────────────────────────────────────────────────────── + # Isolated image: lerobot[metaworld] only (metaworld==3.0.0, mujoco>=3 chain) + metaworld-integration-test: + name: MetaWorld — build image + 1-episode eval + runs-on: + group: aws-g6-4xlarge-plus + env: + HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }} + + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + lfs: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 # zizmor: ignore[unpinned-uses] + with: + cache-binary: false + + - name: Build MetaWorld benchmark image + uses: docker/build-push-action@v6 # zizmor: ignore[unpinned-uses] + with: + context: . + file: docker/Dockerfile.benchmark.metaworld + push: false + load: true + tags: lerobot-benchmark-metaworld:ci + cache-from: type=local,src=/tmp/.buildx-cache-metaworld + cache-to: type=local,dest=/tmp/.buildx-cache-metaworld,mode=max + + - name: Run MetaWorld smoke eval (1 episode) + run: | + docker run --rm --gpus all \ + --shm-size=4g \ + -e HF_HOME=/tmp/hf \ + -e HF_USER_TOKEN="${HF_USER_TOKEN}" \ + lerobot-benchmark-metaworld:ci \ + bash -c " + hf auth login --token \"\$HF_USER_TOKEN\" --add-to-git-credential 2>/dev/null || true + lerobot-eval \ + --policy.path=pepijn223/smolvla_metaworld \ + --env.type=metaworld \ + --env.task=metaworld-push-v2 \ + --eval.batch_size=1 \ + --eval.n_episodes=1 \ + --eval.use_async_envs=false \ + --policy.device=cuda + " diff --git a/docker/Dockerfile.benchmark.libero b/docker/Dockerfile.benchmark.libero new file mode 100644 index 000000000..de06911c0 --- /dev/null +++ b/docker/Dockerfile.benchmark.libero @@ -0,0 +1,74 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Isolated benchmark image for LIBERO integration tests. +# Installs only lerobot[libero] so its dep tree (hf-libero, dm-control, mujoco) +# cannot conflict with other benchmarks. +# +# Build: docker build -f docker/Dockerfile.benchmark.libero -t lerobot-benchmark-libero . +# Run: docker run --gpus all --rm lerobot-benchmark-libero lerobot-eval ... + +ARG CUDA_VERSION=12.4.1 +ARG OS_VERSION=22.04 +FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION} + +ARG PYTHON_VERSION=3.12 + +ENV DEBIAN_FRONTEND=noninteractive \ + MUJOCO_GL=egl \ + PATH=/lerobot/.venv/bin:$PATH \ + CUDA_VISIBLE_DEVICES=0 \ + DEVICE=cuda + +# System deps — same set as Dockerfile.internal +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common build-essential git curl \ + libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ + libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \ + cmake pkg-config ninja-build \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-venv \ + python${PYTHON_VERSION}-dev \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && mv /root/.local/bin/uv /usr/local/bin/uv \ + && useradd --create-home --shell /bin/bash user_lerobot \ + && usermod -aG sudo user_lerobot \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /lerobot +RUN chown -R user_lerobot:user_lerobot /lerobot +USER user_lerobot + +ENV HOME=/home/user_lerobot \ + HF_HOME=/home/user_lerobot/.cache/huggingface \ + HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \ + TORCH_HOME=/home/user_lerobot/.cache/torch \ + TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton + +RUN uv venv --python python${PYTHON_VERSION} + +# Install only lerobot[libero] — completely isolated from metaworld's dep tree +COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml README.md MANIFEST.in ./ +COPY --chown=user_lerobot:user_lerobot src/ src/ + +RUN uv pip install --no-cache ".[libero]" + +RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas + +COPY --chown=user_lerobot:user_lerobot . . + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.benchmark.metaworld b/docker/Dockerfile.benchmark.metaworld new file mode 100644 index 000000000..a9934c284 --- /dev/null +++ b/docker/Dockerfile.benchmark.metaworld @@ -0,0 +1,74 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Isolated benchmark image for MetaWorld integration tests. +# Installs only lerobot[metaworld] so its dep tree (metaworld==3.0.0, mujoco>=3) +# cannot conflict with other benchmarks. +# +# Build: docker build -f docker/Dockerfile.benchmark.metaworld -t lerobot-benchmark-metaworld . +# Run: docker run --gpus all --rm lerobot-benchmark-metaworld lerobot-eval ... + +ARG CUDA_VERSION=12.4.1 +ARG OS_VERSION=22.04 +FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${OS_VERSION} + +ARG PYTHON_VERSION=3.12 + +ENV DEBIAN_FRONTEND=noninteractive \ + MUJOCO_GL=egl \ + PATH=/lerobot/.venv/bin:$PATH \ + CUDA_VISIBLE_DEVICES=0 \ + DEVICE=cuda + +# System deps — same set as Dockerfile.internal +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common build-essential git curl \ + libglib2.0-0 libgl1-mesa-glx libegl1-mesa ffmpeg \ + libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \ + cmake pkg-config ninja-build \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-venv \ + python${PYTHON_VERSION}-dev \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && mv /root/.local/bin/uv /usr/local/bin/uv \ + && useradd --create-home --shell /bin/bash user_lerobot \ + && usermod -aG sudo user_lerobot \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /lerobot +RUN chown -R user_lerobot:user_lerobot /lerobot +USER user_lerobot + +ENV HOME=/home/user_lerobot \ + HF_HOME=/home/user_lerobot/.cache/huggingface \ + HF_LEROBOT_HOME=/home/user_lerobot/.cache/huggingface/lerobot \ + TORCH_HOME=/home/user_lerobot/.cache/torch \ + TRITON_CACHE_DIR=/home/user_lerobot/.cache/triton + +RUN uv venv --python python${PYTHON_VERSION} + +# Install only lerobot[metaworld] — completely isolated from libero's dep tree +COPY --chown=user_lerobot:user_lerobot setup.py pyproject.toml README.md MANIFEST.in ./ +COPY --chown=user_lerobot:user_lerobot src/ src/ + +RUN uv pip install --no-cache ".[metaworld]" + +RUN chmod +x /lerobot/.venv/lib/python${PYTHON_VERSION}/site-packages/triton/backends/nvidia/bin/ptxas + +COPY --chown=user_lerobot:user_lerobot . . + +CMD ["/bin/bash"]