# Copyright 2026 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. name: Model Profiling on: schedule: - cron: "0 0 * * 0" pull_request: branches: - main - feat/libero-benchmark paths: - .github/workflows/model_profiling.yml - profiling/model_profiling_specs.json - scripts/ci/run_model_profiling.py - src/lerobot/configs/train.py - src/lerobot/scripts/lerobot_train.py - src/lerobot/utils/profiling_utils.py - tests/scripts/test_model_profiling.py workflow_dispatch: inputs: git_ref: description: Git ref to profile when no commit SHA is provided required: false type: string default: main git_commit: description: Optional exact commit SHA to profile required: false type: string default: "" policies: description: Optional comma-separated policy filter required: false type: string default: "" profile_mode: description: Torch profiler mode required: false type: choice options: - trace - summary default: trace publish_results: description: Publish results to the profiling dataset when a Hub token is available required: false type: boolean default: true results_repo: description: Dataset repo name or fully qualified repo id required: false type: string default: model-profiling-history permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.inputs.git_commit || github.event.inputs.git_ref || github.ref_name || github.run_id }} cancel-in-progress: true jobs: profile-models: name: Weekly Model Profiling runs-on: group: aws-g6-4xlarge-plus env: HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }} PROFILE_MODE: ${{ github.event_name == 'pull_request' && 'summary' || github.event.inputs.profile_mode || 'trace' }} POLICY_FILTER: ${{ github.event_name == 'pull_request' && 'act' || github.event.inputs.policies || '' }} RESULTS_REPO: ${{ github.event.inputs.results_repo || 'model-profiling-history' }} SHOULD_PUBLISH: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_results == 'true') }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false lfs: true ref: ${{ github.event.pull_request.head.sha || github.event.inputs.git_commit || github.event.inputs.git_ref || 'main' }} - name: Pull GPU image run: docker pull huggingface/lerobot-gpu:latest - name: Run model profiling run: | set -eux mkdir -p profiling-results docker run --rm --gpus all \ --user "$(id -u):$(id -g)" \ --shm-size=16g \ -e HOME=/tmp/lerobot-home \ -e HF_HOME=/tmp/hf \ -e HF_LEROBOT_HOME=/tmp/hf-lerobot \ -e TORCH_HOME=/tmp/torch-home \ -e TORCHINDUCTOR_CACHE_DIR=/tmp/torchinductor-cache \ -e UV_PROJECT_ENVIRONMENT=/tmp/lerobot-venv \ -e UV_CACHE_DIR=/tmp/uv-cache \ -e UV_PYTHON_PREFERENCE=only-system \ -e XDG_DATA_HOME=/tmp/xdg-data \ -e XDG_CACHE_HOME=/tmp/xdg-cache \ -e HOST_GIT_COMMIT="${{ github.event.pull_request.head.sha || github.event.inputs.git_commit || github.sha }}" \ -e HF_USER_TOKEN="${HF_USER_TOKEN}" \ -e HF_TOKEN="${HF_USER_TOKEN}" \ -e PROFILE_MODE="${PROFILE_MODE}" \ -e POLICY_FILTER="${POLICY_FILTER}" \ -e RESULTS_REPO="${RESULTS_REPO}" \ -e SHOULD_PUBLISH="${SHOULD_PUBLISH}" \ -v "${GITHUB_WORKSPACE}:/workspace" \ -w /workspace \ huggingface/lerobot-gpu:latest \ bash -c ' set -euxo pipefail mkdir -p "${HOME}" "${HF_HOME}" "${HF_LEROBOT_HOME}" "${TORCH_HOME}" "${UV_CACHE_DIR}" "${XDG_CACHE_HOME}" "${XDG_DATA_HOME}" "${TORCHINDUCTOR_CACHE_DIR}" rm -rf /tmp/lerobot-src cp -a /workspace/. /tmp/lerobot-src cd /tmp/lerobot-src if [[ -n "${HF_USER_TOKEN:-}" ]]; then hf auth login --token "${HF_USER_TOKEN}" --add-to-git-credential 2>/dev/null || true fi policies_for_install=() if [[ -n "${POLICY_FILTER}" ]]; then IFS="," read -ra policies_for_install <<< "${POLICY_FILTER}" else policies_for_install=(act diffusion groot multi_task_dit pi0 pi0_fast pi05 smolvla wall_x xvla) fi extras=(training test) add_extra() { local extra="$1" for existing in "${extras[@]}"; do if [[ "${existing}" == "${extra}" ]]; then return fi done extras+=("${extra}") } for policy in "${policies_for_install[@]}"; do policy="$(echo "${policy}" | xargs)" case "${policy}" in act) ;; diffusion) add_extra diffusion ;; groot) add_extra groot ;; multi_task_dit) add_extra multi_task_dit ;; pi0|pi0_fast|pi05) add_extra pi ;; smolvla) add_extra smolvla ;; wall_x) add_extra wallx ;; xvla) add_extra xvla ;; *) echo "Unknown profiling policy ${policy}" >&2 exit 1 ;; esac done sync_cmd=(uv sync --locked) for extra in "${extras[@]}"; do sync_cmd+=(--extra "${extra}") done "${sync_cmd[@]}" cmd=( uv run python scripts/ci/run_model_profiling.py --output_dir=/workspace/profiling-results --hub_org=lerobot --results_repo="${RESULTS_REPO}" --profile_mode="${PROFILE_MODE}" --git_commit="${HOST_GIT_COMMIT}" --git_ref="${{ github.head_ref || github.ref_name || github.event.inputs.git_ref || 'main' }}" --pr_number="${{ github.event.pull_request.number || '' }}" ) if [[ -n "${POLICY_FILTER}" ]]; then IFS="," read -ra policies <<< "${POLICY_FILTER}" cmd+=(--policies) for policy in "${policies[@]}"; do policy="$(echo "${policy}" | xargs)" if [[ -n "${policy}" ]]; then cmd+=("${policy}") fi done fi if [[ "${SHOULD_PUBLISH}" == "true" && -n "${HF_USER_TOKEN:-}" ]]; then cmd+=(--publish) fi "${cmd[@]}" ' - name: Upload profiling artifacts if: always() uses: actions/upload-artifact@v4 # zizmor: ignore[unpinned-uses] with: name: model-profiling-results path: profiling-results if-no-files-found: warn