migration: ditch end-effector (task-space) datasets

Some datasets store task-space end-effector pose (names like ee_x/ee_roll or
x/y/z) instead of joint angles; the degrees mapping is meaningless there. Detect
via feature names and skip them entirely (no conversion, no upload) rather than
migrating a mislabeled arm.
This commit is contained in:
CarolinePascal
2026-07-17 15:45:48 +02:00
parent 3ea347a3d9
commit e1da15d243
2 changed files with 21 additions and 1 deletions
+17
View File
@@ -25,6 +25,19 @@ def is_so_robot_type(rt: str) -> bool:
return bool(rt) and (rt.startswith(SO_PREFIXES) or rt in SO_EXACT) and rt not in NEVER_FIX return bool(rt) and (rt.startswith(SO_PREFIXES) or rt in SO_EXACT) and rt not in NEVER_FIX
def is_end_effector(info: dict) -> bool:
"""True if action/observation.state are task-space end-effector features (e.g. ``ee_x``,
``ee_roll``) rather than joint angles. Such datasets are out of scope for the joint fix."""
feats = info.get("features", {})
for key in ("action", "observation.state"):
names = [str(n).lower() for n in (feats.get(key, {}).get("names") or [])]
if any(n.startswith("ee_") or "end_effector" in n or "eef" in n for n in names):
return True
if {"x", "y", "z"} <= set(names):
return True
return False
def load_info(root: Path) -> dict: def load_info(root: Path) -> dict:
return json.loads((Path(root) / "meta" / "info.json").read_text()) return json.loads((Path(root) / "meta" / "info.json").read_text())
@@ -89,6 +102,10 @@ def classify(root) -> dict:
out = {"root": str(root), "robot_type": rt, "action_dim": dim, out = {"root": str(root), "robot_type": rt, "action_dim": dim,
"codebase_version": info.get("codebase_version"), "ambiguous": False} "codebase_version": info.get("codebase_version"), "ambiguous": False}
if is_end_effector(info):
return {**out, "is_so": False, "encoding": "end_effector",
"note": "task-space end-effector features"}
is_so = is_so_robot_type(rt) is_so = is_so_robot_type(rt)
if not is_so: if not is_so:
return {**out, "is_so": False, "encoding": "non_so"} return {**out, "is_so": False, "encoding": "non_so"}
+4 -1
View File
@@ -16,7 +16,7 @@ from pathlib import Path
from huggingface_hub import HfApi from huggingface_hub import HfApi
import so_arm_frame import so_arm_frame
from classify import classify, load_info from classify import classify, is_end_effector, load_info
from fix_dataset import fix_dataset_in_place from fix_dataset import fix_dataset_in_place
SRC_REPO = "HuggingFaceVLA/community_dataset_v3" SRC_REPO = "HuggingFaceVLA/community_dataset_v3"
@@ -165,6 +165,9 @@ def migrate_one(api, dst_repo, sub, work_dir, no_upload) -> dict:
info = load_info(local) info = load_info(local)
if info.get("codebase_version") != "v2.1": if info.get("codebase_version") != "v2.1":
return {"root": sub, "action": f"skipped: source codebase is {info.get('codebase_version')} (expected v2.1)"} return {"root": sub, "action": f"skipped: source codebase is {info.get('codebase_version')} (expected v2.1)"}
if is_end_effector(info):
return {"root": sub, "robot_type": info.get("robot_type"),
"action": "skipped: end-effector (task-space) dataset, out of scope"}
result = fix_dataset_in_place(local) # SO-arm value fix (or structural_only) result = fix_dataset_in_place(local) # SO-arm value fix (or structural_only)