diff --git a/.github/workflows/benchmark_tests.yml b/.github/workflows/benchmark_tests.yml index 6e89cf838..66a2fc215 100644 --- a/.github/workflows/benchmark_tests.yml +++ b/.github/workflows/benchmark_tests.yml @@ -357,7 +357,7 @@ jobs: build-args: | VLABENCH_ASSETS_REPO=lerobot/vlabench-assets - - name: Run VLABench smoke eval (1 episode) + - name: Run VLABench smoke eval (10 tasks, 1 episode each) if: env.HF_USER_TOKEN != '' run: | docker run --name vlabench-eval --gpus all \ @@ -372,7 +372,7 @@ jobs: lerobot-eval \ --policy.path=lerobot/smolvla_vlabench \ --env.type=vlabench \ - --env.task=select_fruit \ + --env.task=select_fruit,select_toy,select_book,select_painting,select_drink,select_ingredient,select_mahjong,select_poker,add_condiment,insert_flower \ --eval.batch_size=1 \ --eval.n_episodes=1 \ --eval.use_async_envs=false \ @@ -394,7 +394,7 @@ jobs: python3 scripts/ci/parse_eval_metrics.py \ --artifacts-dir /tmp/vlabench-artifacts \ --env vlabench \ - --task select_fruit \ + --task select_fruit,select_toy,select_book,select_painting,select_drink,select_ingredient,select_mahjong,select_poker,add_condiment,insert_flower \ --policy lerobot/smolvla_vlabench - name: Upload VLABench rollout video