# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Benchmark image for VLABench integration tests. # Extends the nightly GPU image with the PR's source code and VLABench setup. # # Build: docker build -f docker/Dockerfile.benchmark.vlabench -t lerobot-benchmark-vlabench . # Run: docker run --gpus all --rm lerobot-benchmark-vlabench lerobot-eval ... FROM huggingface/lerobot-gpu:latest # Install VLABench from GitHub (not on PyPI) and pin MuJoCo/dm-control. # Shallow-clone without submodule recursion (nested SSH-only submodules fail in CI). # Editable install (-e) because VLABench/utils/ has no __init__.py, so # find_packages() omits it from wheels; editable mode uses the source tree directly. # rrt-algorithms has the same packaging issue (rrt/ dir missing __init__.py). # Patch: constant.py calls os.listdir on ~100 asset/obj/meshes/* dirs at import # time. Guard the call so missing dirs return [] instead of crashing (in case # the asset download is partial). RUN git clone --depth 1 https://github.com/OpenMOSS/VLABench.git ~/VLABench && \ git clone --depth 1 https://github.com/motion-planning/rrt-algorithms.git ~/rrt-algorithms && \ python3 -c "\ import pathlib; \ p = pathlib.Path.home() / 'VLABench/VLABench/configs/constant.py'; \ t = p.read_text(); \ p.write_text(t.replace( \ 'subdirs = os.listdir(xml_dir)', \ 'if not os.path.isdir(xml_dir): return []\n subdirs = os.listdir(xml_dir)'))" && \ uv pip install --no-cache -e ~/VLABench -e ~/rrt-algorithms \ mujoco==3.2.2 dm-control==1.0.22 \ open3d colorlog scikit-learn openai gdown # Download VLABench mesh assets. Task configs reference object meshes # (obj/meshes/fruit/, containers/basket/, etc.); without them the task builder # picks from an empty mesh list and crashes at task-build time. # # Preferred source: an HF Hub mirror. Set VLABENCH_ASSETS_REPO at build time # (e.g. --build-arg VLABENCH_ASSETS_REPO=lerobot/vlabench-assets) and we'll # snapshot_download the repo into VLABench's assets dir. This is the reliable # path for CI — Google Drive frequently returns HTTP 429 ("Too many users have # viewed or downloaded this file recently") on shared academic files. # # Fallback: VLABench's own gdown-based script. Best-effort only; the build # will NOT fail if gdown hits a Drive quota, letting us ship an image where # task-build may still IndexError but the rest of the pipeline is exercised. ARG VLABENCH_ASSETS_REPO="" RUN ASSETS_DIR="$HOME/VLABench/VLABench/assets" && \ if [ -n "${VLABENCH_ASSETS_REPO}" ]; then \ echo "Downloading VLABench assets from HF Hub: ${VLABENCH_ASSETS_REPO}" && \ python -c "from huggingface_hub import snapshot_download; \ snapshot_download(repo_id='${VLABENCH_ASSETS_REPO}', repo_type='dataset', local_dir='${ASSETS_DIR}')"; \ else \ echo "No VLABENCH_ASSETS_REPO set — falling back to gdown (best-effort)" && \ python ~/VLABench/scripts/download_assets.py --choice all || \ echo "WARN: VLABench asset download failed (likely Google Drive quota). Env task-build may crash."; \ fi # Overlay the PR's source code on top of the nightly image. COPY --chown=user_lerobot:user_lerobot . . # Re-install lerobot editably so the new source (with VLABenchEnv registration # and updated obs handling) replaces the stale package baked into the nightly image. RUN uv pip install --no-cache --no-deps -e . CMD ["/bin/bash"]