From 0f03129536853517a7f74210226fb7a73ef6cfa2 Mon Sep 17 00:00:00 2001 From: Pepijn Date: Fri, 17 Apr 2026 13:40:31 +0100 Subject: [PATCH] ci(robomme): smoke-eval 10 tasks instead of 5 Broader coverage on the RoboMME benchmark CI job: bump the smoke eval from 5 tasks to 10 (one episode each), all drawn from ROBOMME_TASKS. Tasks now run: PickXtimes, BinFill, StopCube, MoveCube, InsertPeg, SwingXtimes, VideoUnmask, ButtonUnmask, PickHighlight, PatternLock. Updated the parse_eval_metrics.py `--task` label from the single `PickXtimes` stub to the full comma list so the metrics artifact reflects what was actually run. `parse_eval_metrics.py` already reads `overall` for multi-task runs, so no parser change is needed. Made-with: Cursor --- .github/workflows/benchmark_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark_tests.yml b/.github/workflows/benchmark_tests.yml index a0a5c99a7..5221503e8 100644 --- a/.github/workflows/benchmark_tests.yml +++ b/.github/workflows/benchmark_tests.yml @@ -347,7 +347,7 @@ jobs: load: true tags: lerobot-benchmark-robomme:ci - - name: Run RoboMME smoke eval (1 episode) + - name: Run RoboMME smoke eval (10 tasks, 1 episode each) if: env.HF_USER_TOKEN != '' run: | docker run --name robomme-eval --gpus all \ @@ -361,7 +361,7 @@ jobs: lerobot-eval \ --policy.path=lerobot/smolvla_robomme \ --env.type=robomme \ - --env.task=PickXtimes,BinFill,StopCube,MoveCube,InsertPeg \ + --env.task=PickXtimes,BinFill,StopCube,MoveCube,InsertPeg,SwingXtimes,VideoUnmask,ButtonUnmask,PickHighlight,PatternLock \ --env.dataset_split=test \ --env.task_ids=[0] \ --eval.batch_size=1 \ @@ -372,7 +372,7 @@ jobs: --policy.empty_cameras=3 \ --output_dir=/tmp/eval-artifacts python scripts/ci/extract_task_descriptions.py \ - --env robomme --task PickXtimes,BinFill,StopCube,MoveCube,InsertPeg \ + --env robomme --task PickXtimes,BinFill,StopCube,MoveCube,InsertPeg,SwingXtimes,VideoUnmask,ButtonUnmask,PickHighlight,PatternLock \ --output /tmp/eval-artifacts/task_descriptions.json " @@ -389,7 +389,7 @@ jobs: python3 scripts/ci/parse_eval_metrics.py \ --artifacts-dir /tmp/robomme-artifacts \ --env robomme \ - --task PickXtimes \ + --task PickXtimes,BinFill,StopCube,MoveCube,InsertPeg,SwingXtimes,VideoUnmask,ButtonUnmask,PickHighlight,PatternLock \ --policy lerobot/smolvla_robomme - name: Upload RoboMME rollout video