From 893b10b8258f3409fc6d89bc4a80278482a6fbed Mon Sep 17 00:00:00 2001
From: Pepijn <pepijn@huggingface.co>
Date: Fri, 17 Apr 2026 13:40:42 +0100
Subject: [PATCH] ci(vlabench): smoke-eval 10 tasks instead of 1

Broader coverage on the VLABench benchmark CI job: bump the smoke eval
from 1 task to 10 (one episode each), all drawn from PRIMITIVE_TASKS.

Tasks now run: select_fruit, select_toy, select_book, select_painting,
select_drink, select_ingredient, select_mahjong, select_poker,
add_condiment, insert_flower.

Bumped the parse_eval_metrics.py `--task` label to the full comma
list so the metrics artifact reflects what was actually run.
`parse_eval_metrics.py` already reads `overall` for multi-task runs,
so no parser change is needed.

Made-with: Cursor
---
 .github/workflows/benchmark_tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmark_tests.yml b/.github/workflows/benchmark_tests.yml
index 6e89cf838..66a2fc215 100644
--- a/.github/workflows/benchmark_tests.yml
+++ b/.github/workflows/benchmark_tests.yml
@@ -357,7 +357,7 @@ jobs:
           build-args: |
             VLABENCH_ASSETS_REPO=lerobot/vlabench-assets
 
-      - name: Run VLABench smoke eval (1 episode)
+      - name: Run VLABench smoke eval (10 tasks, 1 episode each)
         if: env.HF_USER_TOKEN != ''
         run: |
           docker run --name vlabench-eval --gpus all \
@@ -372,7 +372,7 @@ jobs:
               lerobot-eval \
                 --policy.path=lerobot/smolvla_vlabench \
                 --env.type=vlabench \
-                --env.task=select_fruit \
+                --env.task=select_fruit,select_toy,select_book,select_painting,select_drink,select_ingredient,select_mahjong,select_poker,add_condiment,insert_flower \
                 --eval.batch_size=1 \
                 --eval.n_episodes=1 \
                 --eval.use_async_envs=false \
@@ -394,7 +394,7 @@ jobs:
           python3 scripts/ci/parse_eval_metrics.py \
             --artifacts-dir /tmp/vlabench-artifacts \
             --env vlabench \
-            --task select_fruit \
+            --task select_fruit,select_toy,select_book,select_painting,select_drink,select_ingredient,select_mahjong,select_poker,add_condiment,insert_flower \
             --policy lerobot/smolvla_vlabench
 
       - name: Upload VLABench rollout video