test(datasets): support the multi-file video layout in dataset fixtures

Add `episodes_per_video_file` to `episodes_factory`: episode `i` goes to
`file_index = i // episodes_per_video_file` and `from_timestamp` restarts at 0 on
each rollover. `create_videos` encodes one .mp4 per file, and
`mock_snapshot_download` lists and creates every video file rather than only
file-000. Without the option the fixtures emit the same single-file layout as
before.

Add streaming tests over that layout, on the plain and the delta path.
This commit is contained in:
dongmao.zhang
2026-07-16 14:11:33 -07:00
committed by CarolinePascal
parent e0226b23c8
commit e2804c9fbd
3 changed files with 228 additions and 19 deletions
+8 -2
View File
@@ -29,6 +29,7 @@ from lerobot.datasets.utils import (
STATS_PATH,
)
from tests.fixtures.constants import LEROBOT_TEST_DIR
from tests.fixtures.dataset_factories import video_file_frames
@pytest.fixture(scope="session")
@@ -99,7 +100,12 @@ def mock_snapshot_download_factory(
video_keys = [key for key, feats in info.features.items() if feats["dtype"] == "video"]
for key in video_keys:
all_files.append(DEFAULT_VIDEO_PATH.format(video_key=key, chunk_index=0, file_index=0))
# A video key spans one file per `videos/<key>/file_index` in the episodes
# metadata — several of them once v3.0 rolls a video over.
for file_index in video_file_frames(episodes, key, info.total_frames):
all_files.append(
DEFAULT_VIDEO_PATH.format(video_key=key, chunk_index=0, file_index=file_index)
)
allowed_files = filter_repo_objects(
all_files, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns
@@ -138,7 +144,7 @@ def mock_snapshot_download_factory(
if request_data:
create_hf_dataset(local_dir, hf_dataset, data_files_size_in_mb, chunks_size)
if request_videos:
create_videos(root=local_dir, info=info)
create_videos(root=local_dir, info=info, episodes=episodes)
return str(local_dir)