diff --git a/community_v3_migration/fix_dataset.py b/community_v3_migration/fix_dataset.py index 961e5a7df..7ad58e942 100644 --- a/community_v3_migration/fix_dataset.py +++ b/community_v3_migration/fix_dataset.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd import so_arm_frame -from classify import classify, load_info +from classify import classify VALUE_COLS = ("observation.state", "action") @@ -17,13 +17,6 @@ def _stack(col_values) -> np.ndarray: return np.stack([np.asarray(v, dtype=np.float64) for v in col_values]) # (N, D) -def _set_robot_type(root: Path, robot_type: str) -> None: - info_path = root / "meta" / "info.json" - info = json.loads(info_path.read_text()) - info["robot_type"] = robot_type - info_path.write_text(json.dumps(info, indent=4)) - - def _rewrite_parquet(root: Path, encoding: str) -> None: for pq in sorted((root / "data").glob("*/*.parquet")): df = pd.read_parquet(pq) @@ -85,18 +78,6 @@ def fix_dataset_in_place(root) -> dict: return {**cls, "converted": False, "action": "structural v2.1->v3.0 only; joint values kept in normalized units " "(-100..100 / 0..100), NOT converted to degrees (uncalibrated -> APPROXIMATE)"} - feats = load_info(root).get("features", {}) - dims = [feats[c]["shape"][0] for c in VALUE_COLS if c in feats and feats[c].get("shape")] - if any(d % 6 != 0 for d in dims): - # Not a plain stack of 6-joint SO arms (e.g. a 7-joint variant): the degrees mapping - # doesn't apply. Keep the original robot_type but flag it '_nonstandard' so the SO - # lineage is preserved while making clear it isn't a canonical 6-DOF arm. - rt = cls.get("robot_type") or "so" - new_rt = rt if rt.endswith("_nonstandard") else f"{rt}_nonstandard" - _set_robot_type(root, new_rt) - return {**cls, "robot_type": new_rt, "converted": False, - "action": f"structural v2.1->v3.0 only; joint dims {dims} not a multiple of 6 " - f"(non-standard arm), robot_type set to '{new_rt}', joint values left unchanged"} # drop stray files that would otherwise be uploaded for junk in (root / "meta").glob("info.json.bak"): junk.unlink() diff --git a/community_v3_migration/run_migration.py b/community_v3_migration/run_migration.py index 86ee9bcf4..353cb0f93 100644 --- a/community_v3_migration/run_migration.py +++ b/community_v3_migration/run_migration.py @@ -16,7 +16,7 @@ from pathlib import Path from huggingface_hub import HfApi import so_arm_frame -from classify import classify, is_end_effector, load_info +from classify import classify, is_end_effector, is_so_robot_type, load_info from fix_dataset import fix_dataset_in_place SRC_REPO = "HuggingFaceVLA/community_dataset_v3" @@ -168,6 +168,14 @@ def migrate_one(api, dst_repo, sub, work_dir, no_upload) -> dict: if is_end_effector(info): return {"root": sub, "robot_type": info.get("robot_type"), "action": "skipped: end-effector (task-space) dataset, out of scope"} + feats = info.get("features", {}) + dims = [feats[c]["shape"][0] for c in ("action", "observation.state") if feats.get(c, {}).get("shape")] + if is_so_robot_type(info.get("robot_type", "") or "") and any(d % 6 != 0 for d in dims): + # We only migrate datasets usable right away by specifying joints: a clean stack of + # 6-DOF SO arms. Extra appended columns (bbox, EE pose, ...) push the dim off a + # multiple of 6 and mean the degrees mapping doesn't cleanly apply -> out of scope. + return {"root": sub, "robot_type": info.get("robot_type"), + "action": f"skipped: non-standard SO arm (joint dims {dims} not a multiple of 6), out of scope"} result = fix_dataset_in_place(local) # SO-arm value fix (or structural_only)