From e1da15d2431563bd34a567e39d406eef9c366f32 Mon Sep 17 00:00:00 2001 From: CarolinePascal Date: Fri, 17 Jul 2026 15:45:48 +0200 Subject: [PATCH] migration: ditch end-effector (task-space) datasets Some datasets store task-space end-effector pose (names like ee_x/ee_roll or x/y/z) instead of joint angles; the degrees mapping is meaningless there. Detect via feature names and skip them entirely (no conversion, no upload) rather than migrating a mislabeled arm. --- community_v3_migration/classify.py | 17 +++++++++++++++++ community_v3_migration/run_migration.py | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/community_v3_migration/classify.py b/community_v3_migration/classify.py index 6b21b7af1..e8e1249ee 100644 --- a/community_v3_migration/classify.py +++ b/community_v3_migration/classify.py @@ -25,6 +25,19 @@ def is_so_robot_type(rt: str) -> bool: return bool(rt) and (rt.startswith(SO_PREFIXES) or rt in SO_EXACT) and rt not in NEVER_FIX +def is_end_effector(info: dict) -> bool: + """True if action/observation.state are task-space end-effector features (e.g. ``ee_x``, + ``ee_roll``) rather than joint angles. Such datasets are out of scope for the joint fix.""" + feats = info.get("features", {}) + for key in ("action", "observation.state"): + names = [str(n).lower() for n in (feats.get(key, {}).get("names") or [])] + if any(n.startswith("ee_") or "end_effector" in n or "eef" in n for n in names): + return True + if {"x", "y", "z"} <= set(names): + return True + return False + + def load_info(root: Path) -> dict: return json.loads((Path(root) / "meta" / "info.json").read_text()) @@ -89,6 +102,10 @@ def classify(root) -> dict: out = {"root": str(root), "robot_type": rt, "action_dim": dim, "codebase_version": info.get("codebase_version"), "ambiguous": False} + if is_end_effector(info): + return {**out, "is_so": False, "encoding": "end_effector", + "note": "task-space end-effector features"} + is_so = is_so_robot_type(rt) if not is_so: return {**out, "is_so": False, "encoding": "non_so"} diff --git a/community_v3_migration/run_migration.py b/community_v3_migration/run_migration.py index 754cc3757..86ee9bcf4 100644 --- a/community_v3_migration/run_migration.py +++ b/community_v3_migration/run_migration.py @@ -16,7 +16,7 @@ from pathlib import Path from huggingface_hub import HfApi import so_arm_frame -from classify import classify, load_info +from classify import classify, is_end_effector, load_info from fix_dataset import fix_dataset_in_place SRC_REPO = "HuggingFaceVLA/community_dataset_v3" @@ -165,6 +165,9 @@ def migrate_one(api, dst_repo, sub, work_dir, no_upload) -> dict: info = load_info(local) if info.get("codebase_version") != "v2.1": return {"root": sub, "action": f"skipped: source codebase is {info.get('codebase_version')} (expected v2.1)"} + if is_end_effector(info): + return {"root": sub, "robot_type": info.get("robot_type"), + "action": "skipped: end-effector (task-space) dataset, out of scope"} result = fix_dataset_in_place(local) # SO-arm value fix (or structural_only)