mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-24 02:06:15 +00:00
migration: compact non-contiguous episode indices before v3.0 convert
Datasets with deleted episodes (e.g. '*_clean' variants) keep gaps in
their episode numbering across data, videos, and metadata. The stock
v2.1->v3.0 converter renumbers data/videos by sorted file order (0..N-1)
but reads original gapped indices from episodes.jsonl, so it raises
'Number of episodes is not the same'. Add reindex_episodes(): when every
source agrees on the same (non-contiguous) episode set, remap it to
0..N-1 everywhere (data files + episode_index/index columns, per-camera
videos, episodes.jsonl, episodes_stats.jsonl, info.json) so conversion
succeeds. Verified end-to-end on danaaubakirova/svla_so100_task4_v3_clean
(gaps {20,37,38,39} -> 0..49).
This commit is contained in:
@@ -17,7 +17,12 @@ from huggingface_hub import HfApi
|
||||
|
||||
import so_arm_frame
|
||||
from classify import classify, is_end_effector, load_info
|
||||
from fix_dataset import data_video_episode_mismatch, fix_dataset_in_place, reconcile_episode_count
|
||||
from fix_dataset import (
|
||||
data_video_episode_mismatch,
|
||||
fix_dataset_in_place,
|
||||
reconcile_episode_count,
|
||||
reindex_episodes,
|
||||
)
|
||||
|
||||
SRC_REPO = "HuggingFaceVLA/community_dataset_v3"
|
||||
|
||||
@@ -179,6 +184,10 @@ def migrate_one(api, dst_repo, sub, work_dir, no_upload) -> dict:
|
||||
if reconciled:
|
||||
result["action"] = f"{result['action']}; {reconciled}"
|
||||
|
||||
reindexed = reindex_episodes(local) # compact non-contiguous episode indices to 0..N-1
|
||||
if reindexed:
|
||||
result["action"] = f"{result['action']}; {reindexed}"
|
||||
|
||||
from lerobot.scripts.convert_dataset_v21_to_v30 import convert_dataset
|
||||
convert_dataset(repo_id=sub, root=str(local), push_to_hub=False) # v2.1 -> v3.0, in place
|
||||
|
||||
|
||||
Reference in New Issue
Block a user