From f82713cdb257ba8ce1894fad2ab6fae2dcf1a23e Mon Sep 17 00:00:00 2001 From: CarolinePascal Date: Thu, 16 Jul 2026 17:30:10 +0200 Subject: [PATCH] migration: scope subfolder download, keep normalized joints as-is - download_subfolder: fetch only the target sub-dataset subtree instead of enumerating the whole community_dataset_v3 monorepo tree (fixes apparent hang) - normalized SO gripper (RANGE_0_100) left in native 0..100 frame, matching degrees_new datasets, instead of remapping to +/-45deg - uncalibrated normalized datasets: skip identity value rewrite, keep normalized units and flag them APPROXIMATE on the dataset card - remove --allow-uncalibrated flag and its CANON_IS_CALIBRATED side effect --- community_v3_migration/fix_dataset.py | 11 +++++- community_v3_migration/run_migration.py | 50 +++++++++++++++++-------- community_v3_migration/slurm_migrate.py | 21 ++--------- community_v3_migration/so_arm_frame.py | 23 ++++++------ 4 files changed, 58 insertions(+), 47 deletions(-) diff --git a/community_v3_migration/fix_dataset.py b/community_v3_migration/fix_dataset.py index b93007d4c..7ad58e942 100644 --- a/community_v3_migration/fix_dataset.py +++ b/community_v3_migration/fix_dataset.py @@ -7,8 +7,8 @@ from pathlib import Path import numpy as np import pandas as pd +import so_arm_frame from classify import classify -from so_arm_frame import to_degrees VALUE_COLS = ("observation.state", "action") @@ -23,7 +23,7 @@ def _rewrite_parquet(root: Path, encoding: str) -> None: changed = False for col in VALUE_COLS: if col in df.columns: - conv = to_degrees(_stack(df[col].values), encoding, n_joints_per_arm=6) + conv = so_arm_frame.to_degrees(_stack(df[col].values), encoding, n_joints_per_arm=6) df[col] = list(conv.astype(np.float32)) changed = True if changed: @@ -71,6 +71,13 @@ def fix_dataset_in_place(root) -> dict: }.get(enc, "no joint conversion applicable") return {**cls, "converted": False, "action": f"structural v2.1->v3.0 only ({reason}); joint values left unchanged"} + if enc == "normalized" and not so_arm_frame.CANON_IS_CALIBRATED: + # Without per-robot calibration the un-normalization is an identity (placeholder + # spans == 100), so rewriting is pointless. Keep the normalized values as-is and let + # the dataset card flag them APPROXIMATE instead. + return {**cls, "converted": False, + "action": "structural v2.1->v3.0 only; joint values kept in normalized units " + "(-100..100 / 0..100), NOT converted to degrees (uncalibrated -> APPROXIMATE)"} # drop stray files that would otherwise be uploaded for junk in (root / "meta").glob("info.json.bak"): junk.unlink() diff --git a/community_v3_migration/run_migration.py b/community_v3_migration/run_migration.py index 57a65b3ca..370e48a69 100644 --- a/community_v3_migration/run_migration.py +++ b/community_v3_migration/run_migration.py @@ -7,12 +7,13 @@ result under the same path into a NEW repo, then delete the local copy. Resumabl uv run python run_migration.py --dst-repo HuggingFaceVLA/community_dataset_v3_degrees \ --work-dir /big/disk/cdv3_work --manifest manifest.csv Flags: --only-classify (just write manifest), --no-push (fix+convert locally, keep output, - no upload), --folder-name A [B ...] (target specific dataset folders), --limit N, - --allow-uncalibrated (accept placeholder CANON ranges). + no upload), --folder-name A [B ...] (target specific dataset folders), --limit N. +Uncalibrated `normalized` datasets keep their normalized joint units (flagged APPROXIMATE on +the card); paste fitted CANON ranges in so_arm_frame.py to convert them to degrees instead. """ import argparse, csv, json, shutil, sys, traceback from pathlib import Path -from huggingface_hub import HfApi, snapshot_download +from huggingface_hub import HfApi import so_arm_frame from classify import classify, load_info @@ -21,6 +22,28 @@ from fix_dataset import fix_dataset_in_place SRC_REPO = "HuggingFaceVLA/community_dataset_v3" +def download_subfolder(sub: str, work_dir: str, patterns: list[str] | None = None) -> None: + """Download only ``SRC_REPO/{sub}/...`` into ``work_dir``. + + ``snapshot_download`` walks the entire repo tree (``list_repo_tree(recursive=True)`` + with no path scope) before applying ``allow_patterns``. On this 791-dataset monorepo + that whole-repo enumeration is pathologically slow and looks like a hang. Listing the + scoped ``path_in_repo=sub`` subtree and fetching its files directly avoids it. + """ + from fnmatch import fnmatch + + from huggingface_hub import hf_hub_download + from huggingface_hub.hf_api import RepoFile + + api = HfApi() + for entry in api.list_repo_tree(SRC_REPO, path_in_repo=sub, repo_type="dataset", recursive=True): + if not isinstance(entry, RepoFile): + continue + if patterns and not any(fnmatch(entry.path, pat) for pat in patterns): + continue + hf_hub_download(SRC_REPO, filename=entry.path, repo_type="dataset", local_dir=work_dir) + + def list_datasets(api: HfApi, repo: str) -> list[str]: files = api.list_repo_files(repo, repo_type="dataset") roots = {p[: -len("/meta/info.json")] for p in files if p.endswith("/meta/info.json")} @@ -69,7 +92,9 @@ def _write_dataset_card(local: Path, sub: str, result: dict) -> None: joint_actions = { "degrees_old": "per-joint offsets and axis directions corrected to the post-#777 frame (values stay in degrees)", "degrees_new": "already in the post-#777 degrees frame; values unchanged", - "normalized": "un-normalized to physical degrees using canonical joint ranges", + "normalized": ("un-normalized to physical degrees using calibrated joint ranges" + if converted_degrees else + "left in normalized units (-100..100 joints, 0..100 gripper); NOT converted to degrees"), "radians": "left unchanged (already in radians)", "unknown": "left unchanged (encoding could not be determined)", } @@ -94,8 +119,9 @@ def _write_dataset_card(local: Path, sub: str, result: dict) -> None: else: lines += ["- Joint values: not applicable (not an SO-100/101 dataset)"] if approx: - lines += ["", "> **Note:** normalized (-100..100 / 0..100) joint values were un-normalized " - "using *placeholder* canonical joint ranges (uncalibrated). These values are APPROXIMATE."] + lines += ["", "> **Note:** per-robot calibration was unavailable, so joint state/action were " + "left in their original *normalized* units (-100..100 joints, 0..100 gripper) rather " + "than converted to physical degrees. Treat these joint values as APPROXIMATE."] if result.get("ambiguous"): lines += ["", "> **Note:** joint-encoding detection was flagged ambiguous; conversion used the " "best-guess encoding above and may warrant manual review."] @@ -134,8 +160,7 @@ def migrate_one(api, dst_repo, sub, work_dir, no_upload) -> dict: local = Path(work_dir) / sub if local.parent.exists(): shutil.rmtree(local.parent, ignore_errors=True) # clean any partial - snapshot_download(SRC_REPO, repo_type="dataset", revision="main", - allow_patterns=[f"{sub}/*"], local_dir=work_dir) + download_subfolder(sub, work_dir) info = load_info(local) if info.get("codebase_version") != "v2.1": @@ -191,17 +216,11 @@ def main(): ap.add_argument("--no-push", action="store_true", help="Fix + convert locally but do NOT upload; the converted v3.0 output is " "kept under --work-dir for inspection instead of being deleted.") - ap.add_argument("--allow-uncalibrated", action="store_true", - help="Permit converting 'normalized' (-100..100) datasets using the PLACEHOLDER " - "canonical joint ranges in so_arm_frame.py. Omit this to force running " - "calibrate_canonical_ranges.py first (recommended for a real run).") args = ap.parse_args() no_upload = args.no_push if not no_upload and not args.only_classify and not args.dst_repo: ap.error("--dst-repo is required unless --no-push or --only-classify is set.") - if args.allow_uncalibrated: - so_arm_frame.CANON_IS_CALIBRATED = True api = HfApi() if args.folder_name: subs = resolve_folders(api, SRC_REPO, args.folder_name) @@ -224,8 +243,7 @@ def main(): try: if args.only_classify: # classify without full download: fetch just the meta/ of this sub - snapshot_download(SRC_REPO, repo_type="dataset", - allow_patterns=[f"{sub}/meta/*"], local_dir=args.work_dir) + download_subfolder(sub, args.work_dir, patterns=[f"{sub}/meta/*"]) row = {"root": sub, **classify(Path(args.work_dir) / sub)} shutil.rmtree(Path(args.work_dir) / sub.split("/")[0], ignore_errors=True) elif not no_upload and already_done(api, args.dst_repo, sub, dst_files): diff --git a/community_v3_migration/slurm_migrate.py b/community_v3_migration/slurm_migrate.py index 97dfad4a4..7047e4ad8 100644 --- a/community_v3_migration/slurm_migrate.py +++ b/community_v3_migration/slurm_migrate.py @@ -14,7 +14,7 @@ Example (numeric smoke test on one namespace, no SLURM): python slurm_migrate.py --slurm 0 --workers 1 \ --dst-repo HuggingFaceVLA/community_dataset_v3_degrees \ --work-dir ./cdv3_work --manifest-dir ./cdv3_manifests \ - --folder-name Beegbrain --allow-uncalibrated + --folder-name Beegbrain Full run on the cluster: python slurm_migrate.py \ @@ -24,7 +24,6 @@ Full run on the cluster: --logs-dir /fsx/$USER/logs/cdv3_migrate \ --workers 64 --partition hopper-cpu --qos normal \ --cpus-per-task 4 --mem-per-cpu 4G \ - --allow-uncalibrated \ --env-command "source /fsx/$USER/venvs/lerobot/bin/activate; export HF_TOKEN=" IMPORTANT: workers must reach the internet (HF download + upload) and have a write-scoped @@ -54,7 +53,6 @@ class MigrateShard(PipelineStep): migration_dir, no_push=False, only_classify=False, - allow_uncalibrated=False, ): super().__init__() self.subs = subs @@ -64,7 +62,6 @@ class MigrateShard(PipelineStep): self.migration_dir = migration_dir self.no_push = no_push self.only_classify = only_classify - self.allow_uncalibrated = allow_uncalibrated def run(self, data=None, rank: int = 0, world_size: int = 1): # Pickled onto the worker: keep self-contained. The migration package dir must be on @@ -80,16 +77,13 @@ class MigrateShard(PipelineStep): if self.migration_dir not in sys.path: sys.path.insert(0, self.migration_dir) - import so_arm_frame from classify import classify from huggingface_hub import HfApi - from run_migration import SRC_REPO, already_done, migrate_one + from run_migration import already_done, download_subfolder, migrate_one from lerobot.utils.utils import init_logging init_logging() - if self.allow_uncalibrated: - so_arm_frame.CANON_IS_CALIBRATED = True my_subs = self.subs[rank::world_size] if not my_subs: @@ -122,14 +116,7 @@ class MigrateShard(PipelineStep): for i, sub in enumerate(my_subs): try: if self.only_classify: - from huggingface_hub import snapshot_download - - snapshot_download( - SRC_REPO, - repo_type="dataset", - allow_patterns=[f"{sub}/meta/*"], - local_dir=work_dir, - ) + download_subfolder(sub, work_dir, patterns=[f"{sub}/meta/*"]) row = {"root": sub, **classify(Path(work_dir) / sub)} shutil.rmtree(Path(work_dir) / sub.split("/")[0], ignore_errors=True) elif not self.no_push and already_done(api, self.dst_repo, sub, dst_files): @@ -205,7 +192,6 @@ def main(): p.add_argument("--limit", type=int, default=None, help="Only the first N sub-datasets (ignored with --folder-name).") p.add_argument("--only-classify", action="store_true", help="Only classify + write manifest; no convert/upload.") p.add_argument("--no-push", action="store_true", help="Fix + convert locally, keep output, do not upload.") - p.add_argument("--allow-uncalibrated", action="store_true", help="Accept placeholder CANON ranges.") args = p.parse_args() if not args.no_push and not args.only_classify and not args.dst_repo: @@ -236,7 +222,6 @@ def main(): MIGRATION_DIR, no_push=args.no_push, only_classify=args.only_classify, - allow_uncalibrated=args.allow_uncalibrated, ) ], logs_dir=args.logs_dir, diff --git a/community_v3_migration/so_arm_frame.py b/community_v3_migration/so_arm_frame.py index dd0a6e344..3dacfe468 100644 --- a/community_v3_migration/so_arm_frame.py +++ b/community_v3_migration/so_arm_frame.py @@ -8,8 +8,9 @@ Two calibration-free branches + one that needs an assumed canonical range: * degrees_new (`*_follower` recorded with use_degrees=True, not saturated): already degrees. EXACT. * normalized (`*_follower`, -100..100 joints / 0..100 gripper, saturates at bounds): - already mid-range-zero; only the SCALE is missing (per-robot range_min/max - is not stored) -> use assumed canonical per-joint spans below. APPROXIMATE. + 5 arm joints are mid-range-zero, only the SCALE is missing (per-robot + range_min/max not stored) -> use assumed canonical spans below. APPROXIMATE. + The gripper (0..100) is kept in its native frame, matching degrees_new. * radians -> untouched. Joint order per arm: shoulder_pan, shoulder_lift, elbow_flex, wrist_flex, wrist_roll, gripper. @@ -23,13 +24,12 @@ JOINT_ORDER = ["shoulder_pan", "shoulder_lift", "elbow_flex", "wrist_flex", "wri SIGNS = np.array([1.0, -1.0, 1.0, 1.0, 1.0, 1.0], dtype=np.float64) OFFSETS_DEG = np.array([0.0, 90.0, 90.0, 0.0, 0.0, 0.0], dtype=np.float64) -# --- Canonical per-joint spans (DEGREES) used ONLY to invert the -100..100 / 0..100 -# normalization when per-robot calibration is unavailable. joints 0..4 (RANGE_M100_100): -# normalized +/-100 -> +/-HALF_RANGE. gripper (RANGE_0_100): 0..100 -> centered at 50, -# span FULL_RANGE. THESE ARE PLACEHOLDERS — run calibrate_canonical_ranges.py and paste -# the fitted values here before a production run. --- +# --- Canonical per-joint spans (DEGREES) used ONLY to invert the -100..100 normalization of +# the 5 arm joints (RANGE_M100_100) when per-robot calibration is unavailable: normalized +# +/-100 -> +/-HALF_RANGE. The gripper (RANGE_0_100) is left in its native 0..100 frame in +# every SO dataset, so it needs no canonical span. THESE ARE PLACEHOLDERS — run +# calibrate_canonical_ranges.py and paste the fitted values here before a production run. --- CANON_HALF_RANGE_DEG = np.array([100.0, 100.0, 100.0, 100.0, 100.0], dtype=np.float64) # 5 arm joints -CANON_GRIPPER_FULL_RANGE_DEG = 90.0 CANON_IS_CALIBRATED = False # flipped to True once you paste fitted values @@ -43,9 +43,10 @@ def _convert_arm(x: np.ndarray, encoding: str) -> np.ndarray: if encoding == "degrees_new": return x if encoding == "normalized": - new_deg = np.empty_like(x) + new_deg = np.array(x, dtype=np.float64) new_deg[..., :5] = (x[..., :5] / 100.0) * CANON_HALF_RANGE_DEG - new_deg[..., 5] = (x[..., 5] / 100.0 - 0.5) * CANON_GRIPPER_FULL_RANGE_DEG + # gripper is RANGE_0_100 in every SO dataset (including use_degrees=True / degrees_new), + # so it is already frame-consistent and must be left untouched, not remapped to +/-deg. return new_deg raise ValueError(f"unknown encoding: {encoding!r}") @@ -59,7 +60,7 @@ def to_degrees(arr, encoding: str, n_joints_per_arm: int = 6) -> np.ndarray: if encoding == "normalized" and not CANON_IS_CALIBRATED: raise RuntimeError( "CANON ranges are placeholders. Run calibrate_canonical_ranges.py and set " - "CANON_* + CANON_IS_CALIBRATED=True, or pass --allow-uncalibrated to accept them." + "CANON_* + CANON_IS_CALIBRATED=True before converting 'normalized' datasets to degrees." ) out = np.empty_like(arr) for a in range(d // n_joints_per_arm):