migration: reconcile stale meta episode count to data+video files

When the data files and video files agree on episode count N but the metadata
lists a different count, rewrite meta/episodes.jsonl, meta/episodes_stats.jsonl
and info.json to N before the v2.1->v3.0 converter runs (which otherwise raises
'Number of episodes is not the same'). Only the safe direction is handled:
trimming metadata that lists MORE episodes than exist. Non-contiguous data,
data/video disagreement, or metadata missing episodes are left untouched.
This commit is contained in:
CarolinePascal
2026-07-17 16:59:10 +02:00
parent a0ab158a83
commit a6dcb18585
2 changed files with 60 additions and 1 deletions
+5 -1
View File
@@ -17,7 +17,7 @@ from huggingface_hub import HfApi
import so_arm_frame
from classify import classify, is_end_effector, load_info
from fix_dataset import fix_dataset_in_place
from fix_dataset import fix_dataset_in_place, reconcile_episode_count
SRC_REPO = "HuggingFaceVLA/community_dataset_v3"
@@ -171,6 +171,10 @@ def migrate_one(api, dst_repo, sub, work_dir, no_upload) -> dict:
result = fix_dataset_in_place(local) # SO-arm value fix (or structural_only)
reconciled = reconcile_episode_count(local) # align stale meta counts to data+video files
if reconciled:
result["action"] = f"{result['action']}; {reconciled}"
from lerobot.scripts.convert_dataset_v21_to_v30 import convert_dataset
convert_dataset(repo_id=sub, root=str(local), push_to_hub=False) # v2.1 -> v3.0, in place