From 51ea892a4f5b1b3bf5ec510242b53e41a419e8db Mon Sep 17 00:00:00 2001 From: CarolinePascal Date: Thu, 30 Jul 2026 15:01:11 +0200 Subject: [PATCH] feat(dataset filtering): adding support for VLM based dataset filtering following lerobot annotation pipeline style --- docs/source/annotation_pipeline.mdx | 50 +++ docs/source/using_dataset_tools.mdx | 22 ++ pyproject.toml | 1 + .../annotations/camera_curation/__init__.py | 46 +++ .../annotations/camera_curation/config.py | 80 ++++ .../annotations/camera_curation/curator.py | 349 ++++++++++++++++++ .../prompts/camera_curation.txt | 28 ++ src/lerobot/datasets/__init__.py | 2 + src/lerobot/datasets/dataset_tools.py | 249 +++++++++++++ src/lerobot/jobs/curate.py | 112 ++++++ src/lerobot/scripts/lerobot_curate_cameras.py | 244 ++++++++++++ src/lerobot/scripts/lerobot_edit_dataset.py | 55 +++ tests/annotations/test_camera_curation.py | 226 ++++++++++++ tests/datasets/test_dataset_tools.py | 164 +++++++- tests/scripts/test_lerobot_curate_cameras.py | 52 +++ 15 files changed, 1679 insertions(+), 1 deletion(-) create mode 100644 src/lerobot/annotations/camera_curation/__init__.py create mode 100644 src/lerobot/annotations/camera_curation/config.py create mode 100644 src/lerobot/annotations/camera_curation/curator.py create mode 100644 src/lerobot/annotations/camera_curation/prompts/camera_curation.txt create mode 100644 src/lerobot/jobs/curate.py create mode 100644 src/lerobot/scripts/lerobot_curate_cameras.py create mode 100644 tests/annotations/test_camera_curation.py create mode 100644 tests/scripts/test_lerobot_curate_cameras.py diff --git a/docs/source/annotation_pipeline.mdx b/docs/source/annotation_pipeline.mdx index 45396ff67..bd423121d 100644 --- a/docs/source/annotation_pipeline.mdx +++ b/docs/source/annotation_pipeline.mdx @@ -239,6 +239,56 @@ Every module is on by default and can be toggled independently (set to | `--vqa.restrict_to_default_camera` | `false` | Ground VQA only on `--vlm.camera_key` (else every camera). | | `--executor.episode_parallelism` | `16` | Episodes processed concurrently within each phase. | +## Camera-view curation + +`lerobot-curate-cameras` is a separate, lightweight command that uses the same +VLM backend for a **dataset-filtering / curation** pass. It downloads only the +**first episode**, then for each camera view asks the VLM to: + +1. **flag** whether the view is blurry / unusable, and +2. **label** the view with a canonical name from a closed vocabulary + (`top`, `wrist`, `front`, `bottom`, `left`, `right`, plus two-word combos + like `left_wrist`). + +It runs in one of two modes: + +- `--mode=report` (default) — write the labels + verdicts into `meta/` + (`meta/camera_curation.json` and a `curation` block on each camera in + `meta/info.json`). Nothing is moved; this is the cheap triage pass and works + for any dataset. +- `--mode=rename` — apply the labels by renaming each camera key to + `observation.images.