lerobot/.github/workflows/model_profiling.yml

# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Model Profiling

on:
  schedule:
    - cron: "0 0 * * 0"
  pull_request:
    branches:
      - main
      - feat/libero-benchmark
    paths:
      - .github/workflows/model_profiling.yml
      - profiling/model_profiling_specs.json
      - scripts/ci/run_model_profiling.py
      - src/lerobot/configs/train.py
      - src/lerobot/scripts/lerobot_train.py
      - src/lerobot/utils/profiling_utils.py
      - tests/scripts/test_model_profiling.py
  workflow_dispatch:
    inputs:
      git_ref:
        description: Git ref to profile when no commit SHA is provided
        required: false
        type: string
        default: main
      git_commit:
        description: Optional exact commit SHA to profile
        required: false
        type: string
        default: ""
      policies:
        description: Optional comma-separated policy filter
        required: false
        type: string
        default: ""
      profile_mode:
        description: Torch profiler mode
        required: false
        type: choice
        options:
          - trace
          - summary
        default: trace
      publish_results:
        description: Publish results to the profiling dataset when a Hub token is available
        required: false
        type: boolean
        default: true
      results_repo:
        description: Dataset repo name or fully qualified repo id
        required: false
        type: string
        default: model-profiling-history

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.inputs.git_commit || github.event.inputs.git_ref || github.ref_name || github.run_id }}
  cancel-in-progress: true

jobs:
  profile-models:
    name: Weekly Model Profiling
    runs-on:
      group: aws-g6-4xlarge-plus
    env:
      HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
      PROFILE_MODE: ${{ github.event_name == 'pull_request' && 'summary' || github.event.inputs.profile_mode || 'trace' }}
      POLICY_FILTER: ${{ github.event_name == 'pull_request' && 'act' || github.event.inputs.policies || '' }}
      RESULTS_REPO: ${{ github.event.inputs.results_repo || 'model-profiling-history' }}
      SHOULD_PUBLISH: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_results == 'true') }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          lfs: true
          ref: ${{ github.event.pull_request.head.sha || github.event.inputs.git_commit || github.event.inputs.git_ref || 'main' }}

      - name: Pull GPU image
        run: docker pull huggingface/lerobot-gpu:latest

      - name: Run model profiling
        run: |
          set -eux
          mkdir -p profiling-results
          docker run --rm --gpus all \
            --shm-size=16g \
            -e HF_HOME=/tmp/hf \
            -e UV_PROJECT_ENVIRONMENT=/tmp/lerobot-venv \
            -e UV_CACHE_DIR=/tmp/uv-cache \
            -e XDG_CACHE_HOME=/tmp/xdg-cache \
            -e HOST_GIT_COMMIT="${{ github.event.pull_request.head.sha || github.event.inputs.git_commit || github.sha }}" \
            -e HF_USER_TOKEN="${HF_USER_TOKEN}" \
            -e HF_TOKEN="${HF_USER_TOKEN}" \
            -e PROFILE_MODE="${PROFILE_MODE}" \
            -e POLICY_FILTER="${POLICY_FILTER}" \
            -e RESULTS_REPO="${RESULTS_REPO}" \
            -e SHOULD_PUBLISH="${SHOULD_PUBLISH}" \
            -v "${GITHUB_WORKSPACE}:/workspace" \
            -w /workspace \
            huggingface/lerobot-gpu:latest \
            bash -lc '
              set -euxo pipefail
              rm -rf /tmp/lerobot-src
              cp -a /workspace/. /tmp/lerobot-src
              cd /tmp/lerobot-src

              if [[ -n "${HF_USER_TOKEN:-}" ]]; then
                hf auth login --token "${HF_USER_TOKEN}" --add-to-git-credential 2>/dev/null || true
              fi

              uv sync --locked --extra all

              cmd=(
                uv run python scripts/ci/run_model_profiling.py
                --output_dir=/workspace/profiling-results
                --hub_org=lerobot
                --results_repo="${RESULTS_REPO}"
                --profile_mode="${PROFILE_MODE}"
                --git_commit="${HOST_GIT_COMMIT}"
              )

              if [[ -n "${POLICY_FILTER}" ]]; then
                IFS="," read -ra policies <<< "${POLICY_FILTER}"
                cmd+=(--policies)
                for policy in "${policies[@]}"; do
                  policy="$(echo "${policy}" | xargs)"
                  if [[ -n "${policy}" ]]; then
                    cmd+=("${policy}")
                  fi
                done
              fi

              if [[ "${SHOULD_PUBLISH}" == "true" && -n "${HF_USER_TOKEN:-}" ]]; then
                cmd+=(--publish)
              fi

              "${cmd[@]}"
            '

      - name: Upload profiling artifacts
        if: always()
        uses: actions/upload-artifact@v4 # zizmor: ignore[unpinned-uses]
        with:
          name: model-profiling-results
          path: profiling-results
          if-no-files-found: warn