Merge feat/language-annotation-pipeline into feat/smolvla-on-steerable

Bring the authoritative annotation pipeline from the annotation branch.
The annotation surface is forced to EXACTLY match feat/language-annotation-
pipeline (the annotation branch is the source of truth for annotation
code), which also removes smolvla's stale copies:
  - deleted: steerable_pipeline/vocabulary.py, tests/annotations/test_
    vocabulary.py, prompts/module_0_vocabulary.txt, module_1_action_record
    .txt, module_3_vqa.txt, module_1_plan.txt, and the old module_* prompt
    names (now plan_*/interjections_*/vqa.txt).
  - synced: all of src/lerobot/annotations/, lerobot_annotate.py,
    examples/annotations/, tests/annotations/, datasets/language.py,
    tests/datasets/test_language.py, docs/annotation_pipeline.mdx.

Non-annotation conflicts resolved by union (keeping both branches' intent):
  - pyproject.toml: keep smolvla's pi extra (+sentencepiece) and add the
    molmoact2 extra from main.
  - policies/factory.py: keep both dataset_repo_id (pi052 FAST tokenizer)
    and dataset_meta (both are referenced); union the policy-type docstring.
  - scripts/lerobot_train.py: keep smolvla's pi052 / use_relative_actions
    processor-rebuild block.
  - uv.lock: regenerated from the merged pyproject.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

examples/annotations/run_hf_job.py

@@ -1,23 +1,37 @@
 #!/usr/bin/env python
-"""Launch ``lerobot-annotate`` on a Hugging Face job (vllm + Qwen3.6 MoE).
 
-Spawns one ``h200x4`` job that:
+# Copyright 2026 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Launch ``lerobot-annotate`` on a Hugging Face job (vllm + Qwen3.6-27B VLM).
 
-  1. installs this branch of ``lerobot`` plus the annotation extras,
-  2. boots four vllm servers (one per H200) with Qwen3.6-35B-A3B-FP8,
-  3. runs the plan + vqa modules across the dataset in free-form
-     mode — phase 0 (canonical vocabulary discovery) is disabled so
-     every episode's subtasks + memory are generated independently;
-     interjections is also disabled, which short-circuits the
-     plan_update phase that depends on it,
-  4. uploads the annotated dataset to ``--dest_repo_id`` (when set)
+Spawns one single-GPU ``h200`` job that:
+
+  1. installs ``lerobot`` from ``main`` plus the annotation extras,
+  2. boots one vllm server with Qwen3.6-27B (dense VLM),
+  3. runs the plan / interjections / vqa modules across the dataset
+     in free-form mode (each episode generates its own subtasks +
+     memory),
+  4. uploads the annotated dataset to ``--new_repo_id`` (when set)
      or back to ``--repo_id``.
 
 Usage:
 
     HF_TOKEN=hf_... uv run python examples/annotations/run_hf_job.py
 
-Adjust ``CMD`` below to point at your own dataset / target hub repo.
+Adjust ``CMD`` (dataset, model, hub repo) and ``flavor`` below for your
+run. For larger datasets, scale to ``h200x4`` and raise
+``--vlm.parallel_servers`` / ``--vlm.num_gpus`` to match.
 """
 
 import os
@@ -31,91 +45,65 @@ if not token:
 CMD = (
     "apt-get update -qq && apt-get install -y -qq git ffmpeg && "
     "pip install --no-deps "
-    "'lerobot @ git+https://github.com/huggingface/lerobot.git@feat/language-annotation-pipeline' && "
+    "'lerobot @ git+https://github.com/huggingface/lerobot.git@main' && "
     "pip install --upgrade-strategy only-if-needed "
     "datasets pyarrow av jsonlines draccus gymnasium torchcodec mergedeep pyyaml-include toml typing-inspect "
     "openai && "
     "export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 && "
     "export VLLM_VIDEO_BACKEND=pyav && "
     "lerobot-annotate "
-    "--repo_id=pepijn223/robocasa_smoke_2atomic_v3 "
-    "--dest_repo_id=pepijn223/robocasa_smoke_2atomic_v3_annotated "
+    "--repo_id=pepijn223/robocasa_pretrain_human300_v4 "
+    "--new_repo_id=pepijn223/robocasa_pretrain_human300_v4_annotated5 "
     "--push_to_hub=true "
     "--vlm.backend=openai "
-    "--vlm.model_id=Qwen/Qwen3.6-35B-A3B-FP8 "
-    "--vlm.parallel_servers=4 "
-    "--vlm.num_gpus=4 "
-    '--vlm.serve_command="vllm serve Qwen/Qwen3.6-35B-A3B-FP8 '
-    # 4× the context (32768 → 131072) so long episodes at 1 Hz fit even
-    # at full Qwen vision resolution: 90 frames @ ~700 vision tokens/frame
-    # ≈ 63 k tokens, comfortably under 131 k. On 1× H200 (144 GB) the
-    # 35B-FP8 model leaves plenty of room for the bigger KV cache.
-    "--tensor-parallel-size 1 --max-model-len 131072 "
-    '--gpu-memory-utilization 0.85 --uvicorn-log-level warning --port {port}" '
+    "--vlm.model_id=Qwen/Qwen3.6-27B "
+    "--vlm.parallel_servers=1 "
+    "--vlm.num_gpus=1 "
+    '--vlm.serve_command="vllm serve Qwen/Qwen3.6-27B '
+    "--tensor-parallel-size 1 --max-model-len 32768 "
+    '--gpu-memory-utilization 0.8 --uvicorn-log-level warning --port {port}" '
     "--vlm.serve_ready_timeout_s=1800 "
-    "--vlm.client_concurrency=256 "
+    "--vlm.client_concurrency=128 "
     "--vlm.max_new_tokens=512 "
-    # Low temperature for VQA: bbox + keypoint are coordinate-regression
-    # tasks where sampling noise directly degrades localization
-    # (overlapping boxes, drifted points). 0.2 keeps the model decisive
-    # while still letting question/label phrasing vary across frames.
-    "--vlm.temperature=0.2 "
-    "--executor.episode_parallelism=64 "
+    "--vlm.temperature=0.7 "
+    "--executor.episode_parallelism=16 "
     "--vlm.chat_template_kwargs='{\"enable_thinking\": false}' "
-    # Whole-scene agentview is the right choice for subtask reasoning +
-    # VQA on robocasa: the wrist (``robot0_eye_in_hand``) usually only
-    # sees the gripper + nearby object, which hurts "what is happening
-    # in this episode" decomposition. Override per-dataset if your
-    # cameras are named differently (inspect ``meta/info.json``).
-    "--vlm.camera_key=observation.images.robot0_agentview_left "
-    # Phase 0 — canonical vocabulary discovery DISABLED. This dataset's
-    # episodes span heterogeneous tasks/scenes, so a single shared
-    # subtask + memory vocabulary would be too narrow — each episode
-    # generates its subtasks + memory free-form instead.
-    "--vocabulary.enabled=false "
-    # Phase 1 — plan module (subtasks + plan + memory + task_aug).
-    "--plan.enabled=true "
+    "--vlm.camera_key=observation.images.robot0_agentview_right "
+    # Phase 1 — plan module (subtasks + memory).
+    # Embed decoded frames (not a file:// clip): if clip extraction fails,
+    # the video_url path silently sends no video and the VLM hallucinates.
+    "--plan.use_video_url=false "
     "--plan.frames_per_second=1.0 "
-    "--plan.use_video_url=true "
-    "--plan.use_video_url_fps=1.0 "
-    # Force coarse, composite subtasks (``pick up X`` = approach + grasp
-    # + lift in one span, not three). 3 s is large enough to host a
-    # full grasp-or-place composite at typical 20 fps robocasa speeds;
-    # any candidate span shorter than this gets merged into a neighbour
-    # by the prompt's authoring rules (see module_1_subtasks.txt).
-    "--plan.min_subtask_seconds=3.0 "
-    # Cap so the VLM can't drift into micro-segmentation. Combined with
-    # the composite-action rules in the prompt, this targets ~3-6
-    # meaningful spans per episode for typical pick-and-place demos.
-    "--plan.plan_max_steps=9 "
-    # ``off`` keeps the dataset's canonical ``record.episode_task`` as-is
-    # — no per-episode VLM "what is this video about" call. Switch to
-    # ``if_short`` (default) only if some episodes have placeholder /
-    # missing canonical tasks; ``always`` overrides every episode's task.
+    # 32 frames ≈ 8-10k vision tokens, fits the 32768 context. Don't push
+    # toward 128 — that overflows the context (BadRequestError 400).
+    "--plan.max_video_frames=32 "
+    # Window long episodes into 32s chunks (constant 1 fps density) so they
+    # get more subtasks; per-window spans are merged + stitched. 0 disables.
+    "--plan.subtask_window_seconds=32 "
+    # RoboCasa: the dataset task string is authoritative (eval uses it), so
+    # keep it driving subtasks. ``always`` would throw it away and hallucinate.
     "--plan.derive_task_from_video=off "
-    # 0 disables the task_aug pass entirely (see PlanConfig.n_task_rephrasings
-    # docstring) — no per-episode paraphrase generation, no task_aug rows.
+    # No task augmentation: eval conditions on the exact task strings, so
+    # rephrasings are unused at best and harmful when they drift.
     "--plan.n_task_rephrasings=0 "
-    # Phase 2 — interjections OFF (also skips phase 3 plan_update,
-    # see executor.py:_run_plan_update_phase guard).
-    "--interjections.enabled=false "
-    # Phase 4 — general VQA. K=1 keeps each VQA answer on its own
-    # emission frame (no temporal smear); see VqaConfig.K docstring.
-    # 3 Hz cadence: at 20 fps source, that's a VQA tick every ~7 frames.
-    # NOTE: VQA emits per-camera, so for robocasa (3 cameras) each tick
-    # produces 3 (user, assistant) row pairs — total call volume ~= 3 *
-    # 3 Hz * mean_episode_seconds * n_episodes.
-    "--vqa.enabled=true "
-    "--vqa.K=1 "
-    "--vqa.vqa_emission_hz=3.0"
+    # Keep subtask decomposition tight for atomic tasks.
+    "--plan.plan_max_steps=10 "
+    # Only subtasks + memory — skip the numbered "plan" rows. true re-enables.
+    "--plan.emit_plan=false "
+    # The describe->segment grounding pass (+1 VLM call/episode) is ON by
+    # default; pass --plan.subtask_describe_first=false to skip it.
+    # Phase 2 — interjections + speech.
+    "--interjections.max_interjections_per_episode=6 "
+    # Phase 4 — general VQA: disabled for this run.
+    "--vqa.enabled=false"
 )
 
 job = run_job(
     image="vllm/vllm-openai:latest",
     command=["bash", "-c", CMD],
-    flavor="h200x4",
+    flavor="h200",
     secrets={"HF_TOKEN": token},
-    timeout="24h",
+    timeout="2h",
 )
 print(f"Job URL: {job.url}")
 print(f"Job ID:  {job.id}")