mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-15 16:49:55 +00:00
feat(profiling): add weekly model profiling
This commit is contained in:
Pepijn
@@ -0,0 +1,161 @@
|
||||
# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Model Profiling
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * 0"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- feat/libero-benchmark
|
||||
paths:
|
||||
- .github/workflows/model_profiling.yml
|
||||
- profiling/model_profiling_specs.json
|
||||
- scripts/ci/run_model_profiling.py
|
||||
- src/lerobot/configs/train.py
|
||||
- src/lerobot/scripts/lerobot_train.py
|
||||
- src/lerobot/utils/profiling_utils.py
|
||||
- tests/scripts/test_model_profiling.py
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
git_ref:
|
||||
description: Git ref to profile when no commit SHA is provided
|
||||
required: false
|
||||
type: string
|
||||
default: main
|
||||
git_commit:
|
||||
description: Optional exact commit SHA to profile
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
policies:
|
||||
description: Optional comma-separated policy filter
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
profile_mode:
|
||||
description: Torch profiler mode
|
||||
required: false
|
||||
type: choice
|
||||
options:
|
||||
- trace
|
||||
- summary
|
||||
default: trace
|
||||
publish_results:
|
||||
description: Publish results to the profiling dataset when a Hub token is available
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
results_repo:
|
||||
description: Dataset repo name or fully qualified repo id
|
||||
required: false
|
||||
type: string
|
||||
default: model-profiling-history
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.inputs.git_commit || github.event.inputs.git_ref || github.ref_name || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
profile-models:
|
||||
name: Weekly Model Profiling
|
||||
runs-on:
|
||||
group: aws-g6-4xlarge-plus
|
||||
env:
|
||||
HF_USER_TOKEN: ${{ secrets.LEROBOT_HF_USER }}
|
||||
PROFILE_MODE: ${{ github.event_name == 'pull_request' && 'summary' || github.event.inputs.profile_mode || 'trace' }}
|
||||
POLICY_FILTER: ${{ github.event_name == 'pull_request' && 'act' || github.event.inputs.policies || '' }}
|
||||
RESULTS_REPO: ${{ github.event.inputs.results_repo || 'model-profiling-history' }}
|
||||
SHOULD_PUBLISH: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_results == 'true') }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
lfs: true
|
||||
ref: ${{ github.event.pull_request.head.sha || github.event.inputs.git_commit || github.event.inputs.git_ref || 'main' }}
|
||||
|
||||
- name: Pull GPU image
|
||||
run: docker pull huggingface/lerobot-gpu:latest
|
||||
|
||||
- name: Run model profiling
|
||||
run: |
|
||||
set -eux
|
||||
mkdir -p profiling-results
|
||||
docker run --rm --gpus all \
|
||||
--shm-size=16g \
|
||||
-e HF_HOME=/tmp/hf \
|
||||
-e UV_PROJECT_ENVIRONMENT=/tmp/lerobot-venv \
|
||||
-e UV_CACHE_DIR=/tmp/uv-cache \
|
||||
-e XDG_CACHE_HOME=/tmp/xdg-cache \
|
||||
-e HOST_GIT_COMMIT="${{ github.event.pull_request.head.sha || github.event.inputs.git_commit || github.sha }}"
|
||||
-e HF_USER_TOKEN="${HF_USER_TOKEN}" \
|
||||
-e HF_TOKEN="${HF_USER_TOKEN}" \
|
||||
-e PROFILE_MODE="${PROFILE_MODE}" \
|
||||
-e POLICY_FILTER="${POLICY_FILTER}" \
|
||||
-e RESULTS_REPO="${RESULTS_REPO}" \
|
||||
-e SHOULD_PUBLISH="${SHOULD_PUBLISH}" \
|
||||
-v "${GITHUB_WORKSPACE}:/workspace" \
|
||||
-w /workspace \
|
||||
huggingface/lerobot-gpu:latest \
|
||||
bash -lc '
|
||||
set -euxo pipefail
|
||||
rm -rf /tmp/lerobot-src
|
||||
cp -a /workspace/. /tmp/lerobot-src
|
||||
cd /tmp/lerobot-src
|
||||
|
||||
if [[ -n "${HF_USER_TOKEN:-}" ]]; then
|
||||
hf auth login --token "${HF_USER_TOKEN}" --add-to-git-credential 2>/dev/null || true
|
||||
fi
|
||||
|
||||
uv sync --locked --extra all
|
||||
|
||||
cmd=(
|
||||
uv run python scripts/ci/run_model_profiling.py
|
||||
--output_dir=/workspace/profiling-results
|
||||
--hub_org=lerobot
|
||||
--results_repo="${RESULTS_REPO}"
|
||||
--profile_mode="${PROFILE_MODE}"
|
||||
--git_commit="${HOST_GIT_COMMIT}"
|
||||
)
|
||||
|
||||
if [[ -n "${POLICY_FILTER}" ]]; then
|
||||
IFS="," read -ra policies <<< "${POLICY_FILTER}"
|
||||
cmd+=(--policies)
|
||||
for policy in "${policies[@]}"; do
|
||||
policy="$(echo "${policy}" | xargs)"
|
||||
if [[ -n "${policy}" ]]; then
|
||||
cmd+=("${policy}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "${SHOULD_PUBLISH}" == "true" && -n "${HF_USER_TOKEN:-}" ]]; then
|
||||
cmd+=(--publish)
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
'
|
||||
|
||||
- name: Upload profiling artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4 # zizmor: ignore[unpinned-uses]
|
||||
with:
|
||||
name: model-profiling-results
|
||||
path: profiling-results
|
||||
if-no-files-found: warn
|
||||
Reference in New Issue
Block a user